在ELF 1开发板上实现读取摄像头视频进行目标检测

/*命名为 squeezenetssd_thread.cpp */
#include "net.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <vector>
#include <chrono>

/*增加多线程代码*/
#include <thread>
#include <mutex>
#include <queue>
#include <condition_variable>

/*队列通信，全局变量*/
std::queue<cv::Mat> frameQueue;
std::mutex queueMutex;
std::condition_variable queueCondVar;
bool finished = false;
const size_t MAX_QUEUE_SIZE = 2; // 设为两个，因为检测速度实在太慢，多了意义不大

struct Object {
    cv::Rect_<float> rect;
    int label;
    float prob;
};

ncnn::Net squeezenet;
int client_sock;


static int detect_squeezenet(const cv::Mat& bgr, std::vector<Object>& objects)
{

    const int target_size = 300;

    int img_w = bgr.cols;
    int img_h = bgr.rows;

    ncnn::Mat in = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR, bgr.cols, bgr.rows, target_size, target_size);

    const float mean_vals[3] = {104.f, 117.f, 123.f};
    in.substract_mean_normalize(mean_vals, 0);

    ncnn::Extractor ex = squeezenet.create_extractor();

    ex.input("data", in);

    ncnn::Mat out;
    ex.extract("detection_out", out);

    //     printf("%d %d %dn", out.w, out.h, out.c);
    objects.clear();
    for (int i = 0; i < out.h; i++)
    {
        const float* values = out.row(i);

        Object object;
        object.label = values[0];
        object.prob = values[1];
        object.rect.x = values[2] * img_w;
        object.rect.y = values[3] * img_h;
        object.rect.width = values[4] * img_w - object.rect.x;
        object.rect.height = values[5] * img_h - object.rect.y;

        objects.push_back(object);
    }

    return 0;
}

static void draw_objects(cv::Mat& bgr, const std::vector<Object>& objects)
{
    static const char* class_names[] = {"background",
                                        "aeroplane", "bicycle", "bird", "boat",
                                        "bottle", "bus", "car", "cat", "chair",
                                        "cow", "diningtable", "dog", "horse",
                                        "motorbike", "person", "pottedplant",
                                        "sheep", "sofa", "train", "tvmonitor"
                                       };

    //cv::Mat image = bgr.clone();
    //cv::Mat& image = bgr;
    for (size_t i = 0; i < objects.size(); i++)
    {
        const Object& obj = objects[i];

        fprintf(stderr, "%d = %.5f at %.2f %.2f %.2f x %.2fn", obj.label, obj.prob,
                obj.rect.x, obj.rect.y, obj.rect.width, obj.rect.height);

        cv::rectangle(bgr, obj.rect, cv::Scalar(255, 0, 0));

        char text[256];
        sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100);

        int baseLine = 0;
        cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);

        int x = obj.rect.x;
        int y = obj.rect.y - label_size.height - baseLine;
        if (y < 0)
            y = 0;
        if (x + label_size.width > bgr.cols)
            x = bgr.cols - label_size.width;

        cv::rectangle(bgr, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)),
                      cv::Scalar(255, 255, 255), -1);

        cv::putText(bgr, text, cv::Point(x, y + label_size.height),
                    cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0));
        
    }

   // cv::imshow("image", image);
   // cv::waitKey(0);
}

void send_to_client(const cv::Mat& image, int client_sock) {
    std::vector<uchar> buffer;
    std::vector<int> params = {cv::IMWRITE_JPEG_QUALITY, 80};
    cv::imencode(".jpg", image, buffer, params);

    uint32_t len = htonl(buffer.size());
    send(client_sock, &len, sizeof(len), 0);
    send(client_sock, buffer.data(), buffer.size(), 0);
}
/*线程1工作函数，此线程是用来采集相机图像的*/
static void captureThreadFunction(cv::VideoCapture& cap) {
    while (true) {
        cv::Mat frame;
        cap >> frame;
        if (frame.empty()) {
            finished = true;
            queueCondVar.notify_all();
            break;
        }
        cv::rotate(frame, frame, cv::ROTATE_90_COUNTERCLOCKWISE); // 图像旋转90度
        std::unique_lock<std::mutex> lock(queueMutex);
        if (frameQueue.size() >= MAX_QUEUE_SIZE) {
            frameQueue.pop(); // 丢弃最旧的帧
        }
        frameQueue.push(frame);
        queueCondVar.notify_one();
    }
}

/* 线程2工作函数，此线程是用来检测图像*/
void processThreadFunction() {
    int frameCount = 0;
    while (true) {
        cv::Mat frame;
        {
            std::unique_lock<std::mutex> lock(queueMutex);
            queueCondVar.wait(lock, []{ return !frameQueue.empty() || finished; });

            if (finished && frameQueue.empty()) {
                // 退出前释放锁
                return; // 使用 return 替代 break 来确保在持有锁时不退出循环
            }

            frame = frameQueue.front();
            frameQueue.pop();
        } // 锁在这里被释放
    
        // 检测代码...
        std::vector<Object> objects;
        // if (++frameCount % 5 == 0) {
     
        //     detect_squeezenet(frame, objects);
        //     frameCount = 0;
     
        // }
        detect_squeezenet(frame,objects);
        draw_objects(frame, objects);
        send_to_client(frame, client_sock);
        if (cv::waitKey(1) >= 0) {
            break;
        }
    }
}

int main() {

    int server_sock = socket(AF_INET, SOCK_STREAM, 0);
    if (server_sock < 0) {
        perror("socket 创建失败");
        return -1;
    }

    struct sockaddr_in server_addr;
    server_addr.sin_family = AF_INET;
    server_addr.sin_port = htons(12345);
    server_addr.sin_addr.s_addr = INADDR_ANY;

    if (bind(server_sock, (struct sockaddr*)&server_addr, sizeof(server_addr)) < 0) {
        perror("bind 失败");
        close(server_sock);
        return -1;
    }

    if (listen(server_sock, 1) < 0) {
        perror("listen 失败");
        close(server_sock);
        return -1;
    }

    printf("等待客户端连接...n");
    struct sockaddr_in client_addr;
    socklen_t client_len = sizeof(client_addr);
    client_sock = accept(server_sock, (struct sockaddr*)&client_addr, &client_len);
    if (client_sock < 0) {
        perror("accept 失败");
        close(server_sock);
        return -1;
    }
    printf("客户端已连接n");

    // ... [模型加载和初始化代码] ...

    cv::VideoCapture cap("/dev/video2");
    if (!cap.isOpened()) {
        fprintf(stderr, "摄像头打开失败n");
        return -1;
    }

    squeezenet.opt.use_vulkan_compute = true;

    // original pretrained model from https://github.com/chuanqi305/SqueezeNet-SSD
    // squeezenet_ssd_voc_deploy.prototxt
    // https://drive.google.com/open?id=0B3gersZ2cHIxdGpyZlZnbEQ5Snc
    // the ncnn model https://github.com/nihui/ncnn-assets/tree/master/models
    if (squeezenet.load_param("squeezenet_ssd_voc.param"))
        exit(-1);
    if (squeezenet.load_model("squeezenet_ssd_voc.bin"))
        exit(-1);

    std::thread captureThread(captureThreadFunction, std::ref(cap));
    std::thread processThread(processThreadFunction);

    captureThread.join();
    processThread.join();
    cap.release();

    close(client_sock);
    close(server_sock);
    return 0;
}