#include "ImageTailor.h" #include #include "OptimizeCal.h" #include "FusionAndTailor.cuh" std::mutex g_CurFrameMutex; std::mutex g_PanoMutex; ImageTailor::ImageTailor() { pPanoImageBuffer = NULL; pTailorThread = NULL; this->bTailorThreadOut = true; StartTailorThread(); bTailorThreadWork = false; } ImageTailor::~ImageTailor() { this->bTailorThreadOut = false; if (pTailorThread->joinable()) pTailorThread->join(); if (pTailorThread!=NULL) { delete pTailorThread; pTailorThread = NULL; } if (pPanoImageBuffer != NULL) { delete[] pPanoImageBuffer; cudaFree(pPanoImageBufferInGpu); pPanoImageBuffer = NULL; } } void ImageTailor::Init(unsigned char* pImage, int nPanoWidth, int nPanoHeight, cv::Point cvLeftTop) { //get the pointer pPanoImageBuffer = pImage; //get the width this->nPanoWidth = nPanoWidth; //get the height this->nPanoHeight = nPanoHeight; //get the lefttop point this->LeftTop = cvLeftTop; //get the panoimage pitch this->nPanoPitch = (nPanoWidth*3 + 3) / 4 * 4; } void ImageTailor::Init_Gpu(unsigned char* pImage, unsigned char* pImageGpu, int nPanoWidth, int nPanoHeight, cv::Point cvLeftTop) { //get the pointer pPanoImageBuffer = pImage; //get the width this->nPanoWidth = nPanoWidth; //get the height this->nPanoHeight = nPanoHeight; //get the lefttop point this->LeftTop = cvLeftTop; //get the panoimage pitch this->nPanoPitch = (nPanoWidth * 3 + 3) / 4 * 4; pPanoImageBufferInGpu = pImageGpu; } int ImageTailor::StitchingOnPano(cv::Point LeftTop, cv::Mat* FrameImage,std::vector>* pvvOverlap) { //in the paon image ,the lefttop point is (0,0),stitching the image on the pano image //get the lefttop point cv::Point cvLeftTop = LeftTop; //get the width int nFrameWidth = FrameImage->cols; //get the height int nFrameHeight = FrameImage->rows; //get the pitch int nFramePitch = FrameImage->step[0]; //set a pointer to pano image stitching target unsigned char* pPanoImageTarget = pPanoImageBuffer + cvLeftTop.y * nPanoPitch + cvLeftTop.x * 3; cv::Mat PanoImage = cv::Mat(nPanoHeight, nPanoWidth, CV_8UC3, pPanoImageBuffer); //stitching the image for (int i = 0; i < nFrameHeight; i++) { //get this line overlap std::vector& vOverlap = (*pvvOverlap)[i]; //copy the image FastCopy_Any(FrameImage->ptr(i) + vOverlap[0] * 3 , pPanoImageTarget + nPanoPitch * i + vOverlap[0] * 3 , (vOverlap[1]- vOverlap[0]) * 3); } return 0; } int ImageTailor::StitchingOnPanoYUV(cv::Point LeftTop, unsigned char* pImage, int nWidth, int nHeight, int nPitch, std::vector>* pvvOverlap) { //get Y,U,V pointer in pano image unsigned char* pPanoImageY = pPanoImageBuffer; unsigned char* pPanoImageU = pPanoImageBuffer + nPanoPitch * nPanoHeight; unsigned char* pPanoImageV = pPanoImageBuffer + nPanoPitch * nPanoHeight * 5 / 4; cv::Mat panoImage(nPanoHeight * 3 / 2, nPanoWidth, CV_8U, pPanoImageBuffer); //get Y,U,V pointer in frame image unsigned char* pFrameImageY = pImage; unsigned char* pFrameImageU = pImage + nPitch * nHeight; unsigned char* pFrameImageV = pImage + nPitch * nHeight * 5 / 4; //Disassembly the pvvOverlap in rect to calculate the planepaste std::vector vPastePart; int nPanoImageHeight = int(pvvOverlap->size()); cv::Rect rectPastePart; bool bChangeFlags = true; for (int i = 0; i < nPanoImageHeight; i++) { std::vector& vOverlap = (*pvvOverlap)[i]; if (rectPastePart.x != vOverlap[0] && rectPastePart.y != vOverlap[1]) bChangeFlags = true; if (bChangeFlags) { if (i != 0) vPastePart.push_back(rectPastePart); rectPastePart.x = vOverlap[0]; rectPastePart.width = vOverlap[1] - vOverlap[0]; rectPastePart.y = i; rectPastePart.height = 0; bChangeFlags = false; } rectPastePart.height++; } vPastePart.push_back(rectPastePart); int nPartNum = (int)vPastePart.size(); for (int i = 0; i < nPartNum; i++) { cv::Rect& CurPart = vPastePart[i]; //panoImage pointer unsigned char* pPanoImageTargetY = pPanoImageY + (CurPart.y + LeftTop.y) * nPanoPitch + (CurPart.x + LeftTop.x); unsigned char* pPanoImageTargetU = pPanoImageU + (CurPart.y + LeftTop.y) * nPanoPitch / 4 + (CurPart.x + LeftTop.x) / 2; unsigned char* pPanoImageTargetV = pPanoImageV + (CurPart.y + LeftTop.y) * nPanoPitch / 4 + (CurPart.x + LeftTop.x) / 2; //frameImage pointer unsigned char* pFrameImageTargetY = pFrameImageY + CurPart.y * nPitch + CurPart.x; unsigned char* pFrameImageTargetU = pFrameImageU + CurPart.y * nPitch / 4 + CurPart.x / 2; unsigned char* pFrameImageTargetV = pFrameImageV + CurPart.y * nPitch / 4 + CurPart.x / 2; //copy the Y plane libyuv::CopyPlane( pPanoImageTargetY, nPanoPitch, pFrameImageTargetY, nPitch, CurPart.width, CurPart.height ); //copy the U plane libyuv::CopyPlane( pPanoImageTargetU, nPanoPitch / 2, pFrameImageTargetU, nPitch / 2, CurPart.width / 2, CurPart.height / 2 ); //copy the V plane libyuv::CopyPlane( pPanoImageTargetV, nPanoPitch / 2, pFrameImageTargetV, nPitch / 2, CurPart.width / 2, CurPart.height / 2 ); } return 0; } int ImageTailor::StitchingOnPanoYUV(cv::Point LeftTop, cv::Mat* FrameImage, std::vector>* pvvOverlap) { //get Y,U,V pointer in pano image unsigned char* pPanoImageY = pPanoImageBuffer; unsigned char* pPanoImageU = pPanoImageBuffer + ALIGN_4(nPanoWidth) * nPanoHeight; unsigned char* pPanoImageV = pPanoImageBuffer + ALIGN_4(nPanoWidth) * nPanoHeight * 5 / 4; cv::Mat panoImage(nPanoHeight * 3 / 2, nPanoWidth, CV_8U, pPanoImageBuffer); //get Y,U,V pointer in frame image unsigned char* &pFrameImageY = FrameImage->data; unsigned char* pFrameImageU = pFrameImageY + FrameImage->step * FrameImage->rows; unsigned char* pFrameImageV = pFrameImageY + FrameImage->step * FrameImage->rows * 5 / 4; //Disassembly the pvvOverlap in rect to calculate the planepaste std::vector vPastePart; int nPanoImageHeight = int(pvvOverlap->size()); cv::Rect rectPastePart; bool bChangeFlags = true; for (int i = 0; i < nPanoImageHeight; i++) { std::vector& vOverlap = (*pvvOverlap)[i]; if (rectPastePart.x != vOverlap[0] && rectPastePart.y != vOverlap[1]) bChangeFlags = true; if (bChangeFlags) { if (i != 0) vPastePart.push_back(rectPastePart); rectPastePart.x = vOverlap[0]; rectPastePart.width = vOverlap[1] - vOverlap[0]; rectPastePart.y = i; rectPastePart.height = 0; bChangeFlags = false; } rectPastePart.height++; } vPastePart.push_back(rectPastePart); int nPartNum = (int)vPastePart.size(); for (int i = 0; i < nPartNum; i++) { cv::Rect& CurPart = vPastePart[i]; //panoImage pointer unsigned char* pPanoImageTargetY = pPanoImageY + (CurPart.y + LeftTop.y) * ALIGN_4(nPanoWidth) + (CurPart.x + LeftTop.x); unsigned char* pPanoImageTargetU = pPanoImageU + (CurPart.y + LeftTop.y) * ALIGN_4(nPanoWidth) / 4 + (CurPart.x + LeftTop.x) / 2; unsigned char* pPanoImageTargetV = pPanoImageV + (CurPart.y + LeftTop.y) * ALIGN_4(nPanoWidth) / 4 + (CurPart.x + LeftTop.x) / 2; //frameImage pointer unsigned char* pFrameImageTargetY = pFrameImageY + CurPart.y * FrameImage->step + CurPart.x; unsigned char* pFrameImageTargetU = pFrameImageU + CurPart.y * FrameImage->step / 4 + CurPart.x / 2; unsigned char* pFrameImageTargetV = pFrameImageV + CurPart.y * FrameImage->step / 4 + CurPart.x / 2; //copy the Y plane libyuv::CopyPlane( pFrameImageTargetY, FrameImage->step, pPanoImageTargetY, ALIGN_4(nPanoWidth), CurPart.width, CurPart.height ); //copy the U plane libyuv::CopyPlane( pFrameImageTargetU, FrameImage->step / 2, pPanoImageTargetU, ALIGN_4(nPanoWidth) / 2, CurPart.width / 2, CurPart.height / 2 ); //copy the V plane libyuv::CopyPlane( pFrameImageTargetV, FrameImage->step / 2, pPanoImageTargetV, ALIGN_4(nPanoWidth) / 2, CurPart.width / 2, CurPart.height / 2 ); } return 0; } void ImageTailor::TailorWordInThread(ImageTailor* pClass) { while (pClass->bTailorThreadOut) { g_CurFrameMutex.lock(); g_PanoMutex.lock(); if (pClass->qWorkQueue.size()<=0 || pClass->qWorkQueue.size() >10000) { g_CurFrameMutex.unlock(); g_PanoMutex.unlock(); continue; //pClass->ClearWorkOrder(); } pClass->bTailorThreadWork = true; //WAIT_TAILOR& WorkOrder = pClass->qWorkQueue.front(); if (!pClass->bUseYUV) pClass->StitchingOnPano(pClass->qWorkQueue.front().ptPosition, pClass->qWorkQueue.front().pROI, pClass->qWorkQueue.front().pvvOverlap); else pClass->StitchingOnPanoYUVByGPU( pClass->qWorkQueue.front().ptPosition, pClass->qWorkQueue.front().pImage, pClass->qWorkQueue.front().nWidth, pClass->qWorkQueue.front().nHeight, ALIGN_4(pClass->qWorkQueue.front().nPitch), pClass->qWorkQueue.front().pvvOverlap ); g_CurFrameMutex.unlock(); g_PanoMutex.unlock(); pClass->qWorkQueue.pop(); if (pClass->qWorkQueue.size() > 25) pClass->qWorkQueue.pop(); if( pClass->qWorkQueue.size() > 50) pClass->qWorkQueue.pop(); pClass->bTailorThreadWork = false; } } /// /// Stitchings the on pano. /// /// The left top. /// The frame image. /// Width of the n. /// Height of the n. /// The n pitch. /// The PVV overlap. /// int ImageTailor::StitchingOnPanoYUVByGPU(cv::Point LeftTop, unsigned char* FrameImage, int nWidth, int nHeight, int nPitch, std::vector>* pvvOverlap) { //get Y,U,V pointer in pano image unsigned char* pPanoImageY = pPanoImageBufferInGpu + LeftTop.y * ALIGN_4(nPanoWidth) + LeftTop.x; unsigned char* pPanoImageU = pPanoImageBufferInGpu + ALIGN_4(nPanoWidth) * nPanoHeight + LeftTop.y/2 * (ALIGN_4(nPanoWidth)/2) + LeftTop.x/2; unsigned char* pPanoImageV = pPanoImageBufferInGpu + ALIGN_4(nPanoWidth) * nPanoHeight * 5 / 4 + LeftTop.y / 2 * (ALIGN_4(nPanoWidth) / 2) + LeftTop.x / 2; //cv::Mat panoImage(nPanoHeight * 3 / 2, nPanoWidth, CV_8U, pPanoImageBuffer); unsigned char*& pFrameImageY = FrameImage; unsigned char* pFrameImageU = pFrameImageY + nPitch * nHeight; unsigned char* pFrameImageV = pFrameImageY + nPitch * nHeight * 5 / 4; //Disassembly the pvvOverlap in rect to calculate the planepaste std::vector vPastePart; int nPanoImageHeight = int(pvvOverlap->size()); cv::Rect rectPastePart; bool bChangeFlags = true; for (int i = 0; i < nPanoImageHeight; i++) { std::vector& vOverlap = (*pvvOverlap)[i]; if (rectPastePart.x != vOverlap[0] && rectPastePart.y != vOverlap[1]) bChangeFlags = true; if (bChangeFlags) { if (i != 0) vPastePart.push_back(rectPastePart); rectPastePart.x = vOverlap[0]; rectPastePart.width = vOverlap[1] - vOverlap[0]; rectPastePart.y = i; rectPastePart.height = 0; bChangeFlags = false; } rectPastePart.height++; } vPastePart.push_back(rectPastePart); int nBlockCount =int(vPastePart.size()); for (int i = 0; i < nBlockCount; i++) { cv::Rect CurPart = vPastePart[i]; int nPasteWidth = CurPart.width; int nPasteHeight = CurPart.height; int nPasteLeft = CurPart.x; int nPasteTop = CurPart.y; unsigned char* pPanoImageTargetY = pPanoImageY;//; +(CurPart.y) * ALIGN_4(nPanoWidth) + (CurPart.x); unsigned char* pPanoImageTargetU = pPanoImageU;// +(CurPart.y) * ALIGN_4(nPanoWidth) / 4 + (CurPart.x) / 2; unsigned char* pPanoImageTargetV = pPanoImageV;// +(CurPart.y) * ALIGN_4(nPanoWidth) / 4 + (CurPart.x) / 2; //frameImage pointer unsigned char* pFrameImageTargetY = pFrameImageY + CurPart.y * nPitch + CurPart.x; unsigned char* pFrameImageTargetU = pFrameImageU + CurPart.y * nPitch / 4 + CurPart.x / 2; unsigned char* pFrameImageTargetV = pFrameImageV + CurPart.y * nPitch / 4 + CurPart.x / 2; YUVTailorAndBlender::CopyPlane( pFrameImageTargetY, nWidth, nHeight, nPitch, pPanoImageTargetY, nPanoWidth, nPanoHeight, ALIGN_4(nPanoWidth), nPasteWidth, nPasteHeight, nPasteLeft, nPasteTop, 1, NULL ); YUVTailorAndBlender::CopyPlane( pFrameImageTargetU, nWidth / 2, nHeight / 2, nPitch / 2, pPanoImageTargetU, nPanoWidth / 2, nPanoHeight / 2, ALIGN_4(nPanoWidth) / 2, nPasteWidth / 2, nPasteHeight / 2, nPasteLeft / 2, nPasteTop / 2, 1, NULL ); YUVTailorAndBlender::CopyPlane( pFrameImageTargetV, nWidth / 2, nHeight / 2, nPitch / 2, pPanoImageTargetV, nPanoWidth / 2, nPanoHeight / 2, ALIGN_4(nPanoWidth) / 2, nPasteWidth / 2, nPasteHeight / 2, nPasteLeft / 2, nPasteTop / 2, 1, NULL ); cudaError Error = cudaDeviceSynchronize(); if (Error == 700) int x = 10; } return 0; } int ImageTailor::Stitching(int nPostion, cv::Mat& ROI, std::vector>* pvvOverlap) { return 0; } void ImageTailor::SetOverlap(std::vector>* pvvOverlap) { vvOverlap = *pvvOverlap; } bool ImageTailor::StartTailorThread() { pTailorThread = new std::thread(TailorWordInThread, this); return false; } bool ImageTailor::EndTailorThread() { if (pTailorThread->joinable()) { this->bTailorThreadOut = false; pTailorThread->join(); return true; } else { return false; } } void ImageTailor::AddWorkOrderInqueue(cv::Point LeftTop, cv::Mat* FrameImage, std::vector>* pvvOverlap) { WAIT_TAILOR WorkOrder; WorkOrder.ptPosition = LeftTop; WorkOrder.pROI = FrameImage; WorkOrder.pvvOverlap = pvvOverlap; qWorkQueue.push(WorkOrder); } void ImageTailor::ClearWorkOrder() { std::queue().swap(qWorkQueue); /*while (!qWorkQueue.empty()) { qWorkQueue.pop(); }*/ } void ImageTailor::AddWorkOrderInQueueGpu(cv::Point LeftTop, unsigned char* pData, int nWidth, int nHeight, int nPitch, std::vector>* pvvOverlap) { WAIT_TAILOR WorkOrder; WorkOrder.ptPosition = LeftTop; WorkOrder.pImage = pData; WorkOrder.nWidth = nWidth; WorkOrder.nHeight = nHeight; WorkOrder.nPitch = nPitch; WorkOrder.pvvOverlap = pvvOverlap; qWorkQueue.push(WorkOrder); }