找出了rga导致系统崩溃的问题所在，并且修复，目前进度将单个图像工作流程推进至AI处理，正在检查后处理部分代码

.gitignore

@@ -4064,3 +4064,4 @@ build/.cmake/api/v1/reply/target-UVC_Demo-Debug-407eeda93e396b562ca5.json
 build/CMakeFiles/CMakeError.log
 build/CMakeFiles/UVC_Demo.dir/GrpcTransfer/ImageService.grpc.pb.cc.o
 build/CMakeFiles/UVC_Demo.dir/GrpcTransfer/ImageService.pb.cc.o
+build/output.png

AIManager/PPYOLOE.cpp

@@ -94,7 +94,7 @@ bool PPYOLOE::initialize(const std::string &model_path)
 /// @param input_image 输入图像的格式为BGR
 /// @param output_image 直接将结果绘制到输出图像
 /// @return
-bool PPYOLOE::infer(int index, const cv::Mat &input_image, cv::Mat &output_image)
+bool PPYOLOE::infer(int index, unsigned char *input_data, int input_width, int intput_height)
 {
     rknn_input inputs[io_num.n_input];
     rknn_output outputs[io_num.n_output];
@@ -109,25 +109,7 @@ bool PPYOLOE::infer(int index, const cv::Mat &input_image, cv::Mat &output_image
     inputs[0].size = width * height * channel;
     inputs[0].fmt = RKNN_TENSOR_NHWC;
     inputs[0].pass_through = 0;
-
-    if (input_image.cols != width || input_image.rows != height)
-    {
-        resize_buf = malloc(height * width * channel);
-        memset(resize_buf, 0x00, height * width * channel);
-
-        rga_buffer_t src = wrapbuffer_virtualaddr((void *)input_image.data, input_image.cols, input_image.rows, RK_FORMAT_RGB_888);
-        rga_buffer_t dst = wrapbuffer_virtualaddr((void *)resize_buf, width, height, RK_FORMAT_RGB_888);
-        im_rect src_rect, dst_rect;
-        IM_STATUS status = imresize(src, dst);
-        if (status != IM_STATUS_SUCCESS)
-            return false;
-
-        inputs[0].buf = resize_buf;
-    }
-    else
-    {
-        inputs[0].buf = (void *)input_image.data;
-    }
+    inputs[0].buf = (void *)input_data;
 
     int ret = rknn_inputs_set(ctx, io_num.n_input, inputs);
     if (ret < 0)

AIManager/PPYOLOE.hpp

@@ -16,7 +16,7 @@ public:
 
     int get_index() const { return index; }
     bool initialize(const std::string &model_path);
-    bool infer(int index, const cv::Mat &input_image, cv::Mat &output_image);
+    bool infer(int index, unsigned char *input_data, int width, int height);
     void release();
 
     int width, height, channel;

AIManager/RKNNManager.cpp

@@ -3,15 +3,17 @@
 #include "../DataManager/DataManager.h"
 #include "../DataManager/DataPackage.h"
 
-void RKNNManager::addRknnTask(const char *modelData)
+void RKNNManager::addRknnTask(std::string modelData)
 {
     m_threads.emplace_back(&RKNNManager::taskThread, this, modelData);
+    m_threads.emplace_back(&RKNNManager::taskThread, this, modelData);
+    m_threads.emplace_back(&RKNNManager::taskThread, this, modelData);
 }
 
-void RKNNManager::taskThread(const char *modelData)
+void RKNNManager::taskThread(std::string modelpath)
 {
     PPYOLOE infer;
-    if (!infer.initialize(modelData))
+    if (!infer.initialize(modelpath))
     {
         std::cerr << "initialize rknn infer failed" << std::endl;
         return;
@@ -24,11 +26,13 @@ void RKNNManager::taskThread(const char *modelData)
 
     while (m_threadSwitch)
     {
-        if (DataManager::getInstance().popData(pipeName, dataPackage))
+        if (DataManager::getInstance().popData("resized" + std::to_string(index), dataPackage))
         {
-            // infer.infer(dataPackage);
-            // auto resultPipe = DataManager::getInstance().getDataPipe<DataPackage>(pipeName);
-            // resultPipe->pushData(dataPackage);
+            infer.infer(index, (unsigned char *)dataPackage->pResizeData, dataPackage->nResizeWidth, dataPackage->nResizeHeight);
+
+            continue;
         }
+
+        std::this_thread::sleep_for(std::chrono::milliseconds(20));
     }
 }

AIManager/RKNNManager.h

@@ -17,13 +17,13 @@ public:
     RKNNManager(const RKNNManager &) = delete;
     RKNNManager &operator=(const RKNNManager &) = delete;
 
-    void addRknnTask(const char *modelData);
+    void addRknnTask(std::string modelData);
 
 private:
     RKNNManager() = default;
     ~RKNNManager();
 
-    void taskThread(const char *modelData);
+    void taskThread(std::string modelData);
 
 private:
     std::vector<std::thread> m_threads;

AIManager/postprocess.cpp

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-
 #include <math.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -85,7 +84,6 @@ static int readLines(const char *fileName, char *lines[], int max_line)
     return i;
 }
 
-
 static float CalculateOverlap(float xmin0, float ymin0, float xmax0, float ymax0, float xmin1, float ymin1, float xmax1,
                               float ymax1)
 {
@@ -195,18 +193,22 @@ static float deqnt_affine_to_f32(int8_t qnt, int32_t zp, float scale) { return (
 
 static float deqnt_affine_u8_to_f32(uint8_t qnt, int32_t zp, float scale) { return ((float)qnt - (float)zp) * scale; }
 
-static void compute_dfl(float* tensor, int dfl_len, float* box){
-    for (int b=0; b<4; b++){
+static void compute_dfl(float *tensor, int dfl_len, float *box)
+{
+    for (int b = 0; b < 4; b++)
+    {
         float exp_t[dfl_len];
-        float exp_sum=0;
-        float acc_sum=0;
-        for (int i=0; i< dfl_len; i++){
-            exp_t[i] = exp(tensor[i+b*dfl_len]);
+        float exp_sum = 0;
+        float acc_sum = 0;
+        for (int i = 0; i < dfl_len; i++)
+        {
+            exp_t[i] = exp(tensor[i + b * dfl_len]);
             exp_sum += exp_t[i];
         }
-        
-        for (int i=0; i< dfl_len; i++){
-            acc_sum += exp_t[i]/exp_sum *i;
+
+        for (int i = 0; i < dfl_len; i++)
+        {
+            acc_sum += exp_t[i] / exp_sum * i;
         }
         box[b] = acc_sum;
     }
@@ -219,7 +221,7 @@ static int process_u8(uint8_t *box_tensor, int32_t box_zp, float box_scale,
                       std::vector<float> &boxes,
                       std::vector<float> &objProbs,
                       std::vector<int> &classId,
-                      float threshold,int OBJ_CLASS_NUM)
+                      float threshold, int OBJ_CLASS_NUM)
 {
     int validCount = 0;
     int grid_len = grid_h * grid_w;
@@ -291,10 +293,10 @@ static int process_i8(int8_t *box_tensor, int32_t box_zp, float box_scale,
                       int8_t *score_tensor, int32_t score_zp, float score_scale,
                       int8_t *score_sum_tensor, int32_t score_sum_zp, float score_sum_scale,
                       int grid_h, int grid_w, int stride, int dfl_len,
-                      std::vector<float> &boxes, 
-                      std::vector<float> &objProbs, 
-                      std::vector<int> &classId, 
-                      float threshold,int OBJ_CLASS_NUM)
+                      std::vector<float> &boxes,
+                      std::vector<float> &objProbs,
+                      std::vector<int> &classId,
+                      float threshold, int OBJ_CLASS_NUM)
 {
     int validCount = 0;
     int grid_len = grid_h * grid_w;
@@ -305,18 +307,21 @@ static int process_i8(int8_t *box_tensor, int32_t box_zp, float box_scale,
     {
         for (int j = 0; j < grid_w; j++)
         {
-            int offset = i* grid_w + j;
+            int offset = i * grid_w + j;
             int max_class_id = -1;
 
             // 通过 score sum 起到快速过滤的作用
-            if (score_sum_tensor != nullptr){
-                if (score_sum_tensor[offset] < score_sum_thres_i8){
+            if (score_sum_tensor != nullptr)
+            {
+                if (score_sum_tensor[offset] < score_sum_thres_i8)
+                {
                     continue;
                 }
             }
 
             int8_t max_score = -score_zp;
-            for (int c= 0; c< OBJ_CLASS_NUM; c++){
+            for (int c = 0; c < OBJ_CLASS_NUM; c++)
+            {
                 if ((score_tensor[offset] > score_thres_i8) && (score_tensor[offset] > max_score))
                 {
                     max_score = score_tensor[offset];
@@ -326,21 +331,23 @@ static int process_i8(int8_t *box_tensor, int32_t box_zp, float box_scale,
             }
 
             // compute box
-            if (max_score> score_thres_i8){
-                offset = i* grid_w + j;
+            if (max_score > score_thres_i8)
+            {
+                offset = i * grid_w + j;
                 float box[4];
-                float before_dfl[dfl_len*4];
-                for (int k=0; k< dfl_len*4; k++){
+                float before_dfl[dfl_len * 4];
+                for (int k = 0; k < dfl_len * 4; k++)
+                {
                     before_dfl[k] = deqnt_affine_to_f32(box_tensor[offset], box_zp, box_scale);
                     offset += grid_len;
                 }
                 compute_dfl(before_dfl, dfl_len, box);
 
-                float x1,y1,x2,y2,w,h;
-                x1 = (-box[0] + j + 0.5)*stride;
-                y1 = (-box[1] + i + 0.5)*stride;
-                x2 = (box[2] + j + 0.5)*stride;
-                y2 = (box[3] + i + 0.5)*stride;
+                float x1, y1, x2, y2, w, h;
+                x1 = (-box[0] + j + 0.5) * stride;
+                y1 = (-box[1] + i + 0.5) * stride;
+                x2 = (box[2] + j + 0.5) * stride;
+                y2 = (box[3] + i + 0.5) * stride;
                 w = x2 - x1;
                 h = y2 - y1;
                 boxes.push_back(x1);
@@ -350,19 +357,19 @@ static int process_i8(int8_t *box_tensor, int32_t box_zp, float box_scale,
 
                 objProbs.push_back(deqnt_affine_to_f32(max_score, score_zp, score_scale));
                 classId.push_back(max_class_id);
-                validCount ++;
+                validCount++;
             }
         }
     }
     return validCount;
 }
 
-static int process_fp32(float *box_tensor, float *score_tensor, float *score_sum_tensor, 
+static int process_fp32(float *box_tensor, float *score_tensor, float *score_sum_tensor,
                         int grid_h, int grid_w, int stride, int dfl_len,
-                        std::vector<float> &boxes, 
-                        std::vector<float> &objProbs, 
-                        std::vector<int> &classId, 
-                        float threshold,int OBJ_CLASS_NUM)
+                        std::vector<float> &boxes,
+                        std::vector<float> &objProbs,
+                        std::vector<int> &classId,
+                        float threshold, int OBJ_CLASS_NUM)
 {
     int validCount = 0;
     int grid_len = grid_h * grid_w;
@@ -370,18 +377,21 @@ static int process_fp32(float *box_tensor, float *score_tensor, float *score_sum
     {
         for (int j = 0; j < grid_w; j++)
         {
-            int offset = i* grid_w + j;
+            int offset = i * grid_w + j;
             int max_class_id = -1;
 
             // 通过 score sum 起到快速过滤的作用
-            if (score_sum_tensor != nullptr){
-                if (score_sum_tensor[offset] < threshold){
+            if (score_sum_tensor != nullptr)
+            {
+                if (score_sum_tensor[offset] < threshold)
+                {
                     continue;
                 }
             }
 
             float max_score = 0;
-            for (int c= 0; c< OBJ_CLASS_NUM; c++){
+            for (int c = 0; c < OBJ_CLASS_NUM; c++)
+            {
                 if ((score_tensor[offset] > threshold) && (score_tensor[offset] > max_score))
                 {
                     max_score = score_tensor[offset];
@@ -391,21 +401,23 @@ static int process_fp32(float *box_tensor, float *score_tensor, float *score_sum
             }
 
             // compute box
-            if (max_score> threshold){
-                offset = i* grid_w + j;
+            if (max_score > threshold)
+            {
+                offset = i * grid_w + j;
                 float box[4];
-                float before_dfl[dfl_len*4];
-                for (int k=0; k< dfl_len*4; k++){
+                float before_dfl[dfl_len * 4];
+                for (int k = 0; k < dfl_len * 4; k++)
+                {
                     before_dfl[k] = box_tensor[offset];
                     offset += grid_len;
                 }
                 compute_dfl(before_dfl, dfl_len, box);
 
-                float x1,y1,x2,y2,w,h;
-                x1 = (-box[0] + j + 0.5)*stride;
-                y1 = (-box[1] + i + 0.5)*stride;
-                x2 = (box[2] + j + 0.5)*stride;
-                y2 = (box[3] + i + 0.5)*stride;
+                float x1, y1, x2, y2, w, h;
+                x1 = (-box[0] + j + 0.5) * stride;
+                y1 = (-box[1] + i + 0.5) * stride;
+                x2 = (box[2] + j + 0.5) * stride;
+                y2 = (box[3] + i + 0.5) * stride;
                 w = x2 - x1;
                 h = y2 - y1;
                 boxes.push_back(x1);
@@ -415,17 +427,16 @@ static int process_fp32(float *box_tensor, float *score_tensor, float *score_sum
 
                 objProbs.push_back(max_score);
                 classId.push_back(max_class_id);
-                validCount ++;
+                validCount++;
             }
         }
     }
     return validCount;
 }
 
-
-int post_process(PPYOLOE *app_ctx, rknn_output *outputs, float conf_threshold, float nms_threshold, object_detect_result_list *od_results,int OBJ_CLASS_NUM)
+int post_process(PPYOLOE *app_ctx, rknn_output *outputs, float conf_threshold, float nms_threshold, object_detect_result_list *od_results, int OBJ_CLASS_NUM)
 {
-    
+
     std::vector<float> filterBoxes;
     std::vector<float> objProbs;
     std::vector<int> classId;
@@ -442,7 +453,7 @@ int post_process(PPYOLOE *app_ctx, rknn_output *outputs, float conf_threshold, f
 #ifdef RKNPU1
     int dfl_len = app_ctx->output_attrs[0].dims[2] / 4;
 #else
-    int dfl_len = app_ctx->output_attrs[0].dims[1] /4;
+    int dfl_len = app_ctx->output_attrs[0].dims[1] / 4;
 #endif
     int output_per_branch = app_ctx->io_num.n_output / 3;
     for (int i = 0; i < 3; i++)
@@ -451,13 +462,14 @@ int post_process(PPYOLOE *app_ctx, rknn_output *outputs, float conf_threshold, f
         void *score_sum = nullptr;
         int32_t score_sum_zp = 0;
         float score_sum_scale = 1.0;
-        if (output_per_branch == 3){
-            score_sum = outputs[i*output_per_branch + 2].buf;
-            score_sum_zp = app_ctx->output_attrs[i*output_per_branch + 2].zp;
-            score_sum_scale = app_ctx->output_attrs[i*output_per_branch + 2].scale;
+        if (output_per_branch == 3)
+        {
+            score_sum = outputs[i * output_per_branch + 2].buf;
+            score_sum_zp = app_ctx->output_attrs[i * output_per_branch + 2].zp;
+            score_sum_scale = app_ctx->output_attrs[i * output_per_branch + 2].scale;
         }
-        int box_idx = i*output_per_branch;
-        int score_idx = i*output_per_branch + 1;
+        int box_idx = i * output_per_branch;
+        int score_idx = i * output_per_branch + 1;
 
 #ifdef RKNPU1
         grid_h = app_ctx->output_attrs[box_idx].dims[1];
@@ -480,17 +492,16 @@ int post_process(PPYOLOE *app_ctx, rknn_output *outputs, float conf_threshold, f
             validCount += process_i8((int8_t *)outputs[box_idx].buf, app_ctx->output_attrs[box_idx].zp, app_ctx->output_attrs[box_idx].scale,
                                      (int8_t *)outputs[score_idx].buf, app_ctx->output_attrs[score_idx].zp, app_ctx->output_attrs[score_idx].scale,
                                      (int8_t *)score_sum, score_sum_zp, score_sum_scale,
-                                     grid_h, grid_w, stride, dfl_len, 
-                                     filterBoxes, objProbs, classId, conf_threshold,OBJ_CLASS_NUM);
+                                     grid_h, grid_w, stride, dfl_len,
+                                     filterBoxes, objProbs, classId, conf_threshold, OBJ_CLASS_NUM);
 #endif
         }
         else
         {
             validCount += process_fp32((float *)outputs[box_idx].buf, (float *)outputs[score_idx].buf, (float *)score_sum,
-                                       grid_h, grid_w, stride, dfl_len, 
-                                       filterBoxes, objProbs, classId, conf_threshold,OBJ_CLASS_NUM);
+                                       grid_h, grid_w, stride, dfl_len,
+                                       filterBoxes, objProbs, classId, conf_threshold, OBJ_CLASS_NUM);
         }
-
     }
 
     // no object detect
@@ -515,7 +526,6 @@ int post_process(PPYOLOE *app_ctx, rknn_output *outputs, float conf_threshold, f
     int last_count = 0;
     od_results->count = 0;
 
-
     /* box valid detect target */
     for (int i = 0; i < validCount; ++i)
     {
@@ -532,7 +542,6 @@ int post_process(PPYOLOE *app_ctx, rknn_output *outputs, float conf_threshold, f
         int id = classId[n];
         float obj_conf = objProbs[i];
 
-
         od_results->results[last_count].box.left = (int)(clamp(x1, 0, model_in_w));
         od_results->results[last_count].box.top = (int)(clamp(y1, 0, model_in_h));
         od_results->results[last_count].box.right = (int)(clamp(x2, 0, model_in_w));

ImageTest/ImageTest.h

@@ -0,0 +1,17 @@
+#ifndef IMAGE_TEST_H
+#define IMAGE_TEST_H
+
+#include <opencv4/opencv2/opencv.hpp>
+#include <string>
+
+namespace ImageTest
+{
+
+    void saveImageFromData(unsigned char *data, int width, int height, const std::string &filename = "./output.png")
+    {
+        cv::Mat image(height, width, CV_8UC3, data);
+        cv::imwrite(filename, image);
+    }
+
+}
+#endif // IMAGE_TEST_H

MppDecoder/MppDecoder.cpp

@@ -40,7 +40,7 @@ bool MppDecoder::init()
         return false;
     }
 
-    MppFrameFormat fmt = MPP_FMT_YUV420SP;
+    MppFrameFormat fmt = MPP_FMT_RGB888;
     param = &fmt;
     ret = m_mpi->control(m_ctx, MPP_DEC_SET_OUTPUT_FORMAT, param);
     if (ret)
@@ -155,10 +155,9 @@ bool MppDecoder::decodeJpegToRgb(const char *jpegData, size_t jpegSize, unsigned
             buffer = mpp_frame_get_buffer(Output);
             if (buffer)
             {
-                *rgbData = (unsigned char *)buffer;
+                *rgbData = (unsigned char *)mpp_buffer_get_ptr(buffer);
                 *width = mpp_frame_get_width(Output);
                 *height = mpp_frame_get_height(Output);
-
             }
         }
     }

MppDecoder/MppManager.cpp

@@ -3,6 +3,7 @@
 #include "../DataManager/DataPackage.h"
 #include "../RGAColorTransfer/RgaColorTransfer.h"
 #include <thread>
+#include "../ImageTest/ImageTest.h"
 // ...existing code...
 
 MppManager::~MppManager()
@@ -20,8 +21,8 @@ MppManager::~MppManager()
 void MppManager::addMppDecode()
 {
     m_threads.emplace_back(&MppManager::decodeThread, this);
-    // m_threads.emplace_back(&MppManager::decodeThread, this);
-    // m_threads.emplace_back(&MppManager::decodeThread, this);
+    m_threads.emplace_back(&MppManager::decodeThread, this);
+    m_threads.emplace_back(&MppManager::decodeThread, this);
 }
 
 void MppManager::decodeThread()
@@ -33,21 +34,22 @@ void MppManager::decodeThread()
     int width = 0, height = 0;
     m_threadSwitch = true;
     int index = decoder.getDecodeIndex();
-    DataManager::getInstance().addDataPipe<DataPackagePtr>("mpp" + std::to_string(index));
+    DataManager::getInstance().addDataPipe<DataPackagePtr>("resized" + std::to_string(index));
     while (m_threadSwitch)
     {
         DataPackagePtr dataPackage = nullptr;
-        if (DataManager::getInstance().popData("mpp" + std::to_string(index), dataPackage))
+        if (DataManager::getInstance().popData("uvc" + std::to_string(index), dataPackage))
         {
             decoder.decodeJpegToRgb((const char *)dataPackage->pJpegData, dataPackage->nJpegSize, (unsigned char **)&dataPackage->pRGBData, &width, &height);
             if (width != dataPackage->nWidth || height != dataPackage->nHeight)
             {
                 std::cerr << "the decode result is different from defined !!" << std::endl;
             }
+
             // color transfer
-            // colorTransfer.setSrc(dataPackage->pRGBData,width,height,RK_FORMAT_RGB_888);
-            // colorTransfer.setDst(dataPackage->pResizeData,dataPackage->nResizeWidth,dataPackage->nResizeHeight,RK_FORMAT_RGB_888);
-            // colorTransfer.resizeImage(cv::Rect(0,0,width,height),cv::Rect(0,0,dataPackage->nResizeWidth,dataPackage->nResizeHeight));
+            resize_image(dataPackage->pRGBData, width, height, dataPackage->pResizeData, dataPackage->nResizeWidth, dataPackage->nResizeHeight);
+
+            // ImageTest::saveImageFromData((unsigned char *)dataPackage->pResizeData, dataPackage->nResizeWidth, dataPackage->nResizeHeight);
 
             // push data to the next pipe
             DataManager::getInstance().pushData("resized" + std::to_string(index), dataPackage);

RGAColorTransfer/RgaColorTransfer.cpp

@@ -102,3 +102,33 @@ void RgaColorTransfer::ReleaseResource()
         m_Dst = nullptr;
     }
 }
+
+int resize_image(void * src_buf, int src_width, int src_height, void * dst_buf, int dst_width, int dst_height)
+{
+    int ret;
+    rga_info_t src_info;
+    rga_info_t dst_info;
+
+    // Initialize source image information
+    memset(&src_info, 0, sizeof(rga_info_t));
+    src_info.fd = -1;
+    src_info.virAddr = src_buf;
+    src_info.mmuFlag = 1;
+    rga_set_rect(&src_info.rect, 0, 0, src_width, src_height, src_width, src_height, RK_FORMAT_RGB_888);
+
+    // Initialize destination image information
+    memset(&dst_info, 0, sizeof(rga_info_t));
+    dst_info.fd = -1;
+    dst_info.virAddr = dst_buf;
+    dst_info.mmuFlag = 1;
+    rga_set_rect(&dst_info.rect, 0, 0, dst_width, dst_height, dst_width, dst_height, RK_FORMAT_RGB_888);
+
+    // Perform the resize operation
+    ret = c_RkRgaBlit(&src_info, &dst_info, NULL);
+    if (ret) {
+        printf("RGA resize failed: %d\n", ret);
+        return ret;
+    }
+
+    return 0;
+}

RGAColorTransfer/RgaColorTransfer.h

@@ -1,3 +1,6 @@
+#ifndef RGA_COLOR_TRANSFER_H
+#define RGA_COLOR_TRANSFER_H
+
 #include <rga/RgaApi.h>
 #include <rga/im2d.hpp>
 #include <rga/RgaUtils.h>
@@ -95,3 +98,8 @@ private:
     int m_releaseFenceFd;
     int m_accFenceFd;
 };
+
+int resize_image(void *src_buf, int src_width, int src_height, void *dst_buf, int dst_width, int dst_height);
+
+
+#endif // RGA_COLOR_TRANSFER_H