OpenVINO2023异步回调流水线提升推理吞吐率

while(true) {
  // capture frame
  // populate CURRENT InferRequest
  // Infer CURRENT InferRequest
  //this call is synchronous
  // display CURRENT result
}

// 创建IE插件, 查询支持硬件设备
ov::Core core;
std::string model_onnx = "D:/python/yolov5-7.0/yolov5s.onnx";
auto model = core.read_model(model_onnx);
ov::CompiledModel cmodel = core.compile_model(model, "CPU");


// create infer request
auto request = cmodel.create_infer_request();
cv::Mat frame;
while (true) {
  bool ret = cap.read(frame);
  if (frame.empty()) {
    break;
  }
  image_detect(frame, request);
  char c = cv::waitKey(1);
  if (c == 27) { // ESC
    break;
  }
}

// 前处理
// 开启同步
request.infer();
// 后处理

while(true) {
  // capture frame
  // populate NEXT InferRequest
  // start NEXT InferRequest
  // this call is async and returns immediately
  // wait for the CURRENT InferRequest
  // display CURRENT result
  // swap CURRENT and NEXT InferRequests
}

// 创建IE插件, 查询支持硬件设备
ov::Core core;
std::string model_onnx = "D:/python/yolov5-7.0/yolov5s.onnx";
auto model = core.read_model(model_onnx);
ov::CompiledModel cmodel = core.compile_model(model, "AUTO");

// create infer request
auto request = cmodel.create_infer_request();
auto next_request = cmodel.create_infer_request();
std::exception_ptr exception_var;
request.set_callback([&](std::exception_ptr ex) {
    if (ex) {
        exception_var = ex;
        return;
    }
    det_boxes.clear();
    det_ids.clear();
    ov::Tensor output = request.get_output_tensor();
    const float* prob = (float*)output.data();
    const ov::Shape outputDims = output.get_shape();
    size_t numRows = outputDims[1];
    size_t numCols = outputDims[2];

    // 后处理, 1x25200x85
    std::vector<cv::Rect> boxes;
    std::vector<int> classIds;
    std::vector<float> confidences;
    cv::Mat det_output(numRows, numCols, CV_32F, (float*)prob);
    for (int i = 0; i < det_output.rows; i++) {
        float confidence = det_output.at<float>(i, 4);
        if (confidence < 0.45) {
            continue;
        }
        cv::Mat classes_scores = det_output.row(i).colRange(5, numCols);
        cv::Point classIdPoint;
        double score;
        minMaxLoc(classes_scores, 0, &score, 0, &classIdPoint);

        // 置信度 0～1之间
        if (score > 0.25)
        {
            float cx = det_output.at<float>(i, 0);
            float cy = det_output.at<float>(i, 1);
            float ow = det_output.at<float>(i, 2);
            float oh = det_output.at<float>(i, 3);
            int x = static_cast<int>((cx - 0.5 * ow) * x_factor);
            int y = static_cast<int>((cy - 0.5 * oh) * y_factor);
            int width = static_cast<int>(ow * x_factor);
            int height = static_cast<int>(oh * y_factor);
            cv::Rect box;
            box.x = x;
            box.y = y;
            box.width = width;
            box.height = height;

            boxes.push_back(box);
            classIds.push_back(classIdPoint.x);
            confidences.push_back(score);
        }
    }

    // NMS
    std::vector<int> indexes;
    cv::dnn::NMSBoxes(boxes, confidences, 0.25, 0.45, indexes);
    for (size_t i = 0; i < indexes.size(); i++) {
        int index = indexes[i];
        det_ids.emplace_back(classIds[index]);
        det_boxes.emplace_back(boxes[index]);
    }
});

cv::Mat frame, next_frame;
// do first frame
cap.read(frame);
async_image_detect(frame, request);
std::chrono::milliseconds tout{ 50 };
int cnt = 0;
while (true) {
    bool ret = cap.read(next_frame);
    if (next_frame.empty()) {
        break;
    }

    int64 start = cv::getTickCount();
    // 继续异步
    if (cnt % 2 == 0) {
        async_image_detect(next_frame, next_request);
        request.wait_for(tout);
    }
    if (cnt % 2 == 1) {
        async_image_detect(next_frame, request);
        next_request.wait_for(tout);
    }
    for (size_t t = 0; t < det_boxes.size(); t++) {
        int idx = det_ids[t];
        cv::rectangle(frame, det_boxes[t], colors_table[idx % 6], 2, 8, 0);
        putText(frame, classNames[idx].c_str(), det_boxes[t].tl(), cv::FONT_HERSHEY_PLAIN, 1.0, cv::Scalar(255, 0, 0), 1, 8);
    }

    // 计算FPS render it
    float t = (cv::getTickCount() - start) / static_cast<float>(cv::getTickFrequency());
    putText(frame, cv::format("FPS: %.2f", 1.0 / t), cv::Point(20, 40), cv::FONT_HERSHEY_PLAIN, 2.0, cv::Scalar(255, 0, 0), 2, 8);
    cv::imshow("OpenVINO2023 - YOLOv5 7.0 异步推理", frame);
    char c = cv::waitKey(1);
    if (c == 27) { // ESC
        break;
    }
    next_frame.copyTo(frame);
    cnt++;
}
cv::waitKey(0);
cv::destroyAllWindows();
return 0;

preprocess(frame)
// 开启异步
request.start_async();