RGB Encoding & MobilenetSSD

1#!/usr/bin/env python3
2
3from pathlib import Path
4import sys
5import cv2
6import depthai as dai
7import numpy as np
8
9# Get argument first
10nnPath = str((Path(__file__).parent / Path('../models/mobilenet-ssd_openvino_2021.4_6shave.blob')).resolve().absolute())
11if len(sys.argv) > 1:
12    nnPath = sys.argv[1]
13
14if not Path(nnPath).exists():
15    import sys
16    raise FileNotFoundError(f'Required file/s not found, please run "{sys.executable} install_requirements.py"')
17
18# MobilenetSSD label texts
19labelMap = ["background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow",
20            "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"]
21
22# Create pipeline
23pipeline = dai.Pipeline()
24
25# Define sources and outputs
26camRgb = pipeline.create(dai.node.ColorCamera)
27videoEncoder = pipeline.create(dai.node.VideoEncoder)
28nn = pipeline.create(dai.node.MobileNetDetectionNetwork)
29
30xoutRgb = pipeline.create(dai.node.XLinkOut)
31videoOut = pipeline.create(dai.node.XLinkOut)
32nnOut = pipeline.create(dai.node.XLinkOut)
33
34xoutRgb.setStreamName("rgb")
35videoOut.setStreamName("h265")
36nnOut.setStreamName("nn")
37
38# Properties
39camRgb.setBoardSocket(dai.CameraBoardSocket.CAM_A)
40camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
41camRgb.setPreviewSize(300, 300)
42camRgb.setInterleaved(False)
43
44videoEncoder.setDefaultProfilePreset(30, dai.VideoEncoderProperties.Profile.H265_MAIN)
45
46nn.setConfidenceThreshold(0.5)
47nn.setBlobPath(nnPath)
48nn.setNumInferenceThreads(2)
49nn.input.setBlocking(False)
50
51# Linking
52camRgb.video.link(videoEncoder.input)
53camRgb.preview.link(xoutRgb.input)
54camRgb.preview.link(nn.input)
55videoEncoder.bitstream.link(videoOut.input)
56nn.out.link(nnOut.input)
57
58# Connect to device and start pipeline
59with dai.Device(pipeline) as device, open('video.h265', 'wb') as videoFile:
60
61    # Queues
62    queue_size = 8
63    qRgb = device.getOutputQueue("rgb", queue_size)
64    qDet = device.getOutputQueue("nn", queue_size)
65    qRgbEnc = device.getOutputQueue('h265', maxSize=30, blocking=True)
66
67    frame = None
68    detections = []
69
70    def frameNorm(frame, bbox):
71        normVals = np.full(len(bbox), frame.shape[0])
72        normVals[::2] = frame.shape[1]
73        return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)
74
75    def displayFrame(name, frame):
76        color = (255, 0, 0)
77        for detection in detections:
78            bbox = frameNorm(frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax))
79            cv2.putText(frame, labelMap[detection.label], (bbox[0] + 10, bbox[1] + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)
80            cv2.putText(frame, f"{int(detection.confidence * 100)}%", (bbox[0] + 10, bbox[1] + 40), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)
81            cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color, 2)
82        # Show the frame
83        cv2.imshow(name, frame)
84
85    while True:
86        inRgb = qRgb.tryGet()
87        inDet = qDet.tryGet()
88
89        while qRgbEnc.has():
90            qRgbEnc.get().getData().tofile(videoFile)
91
92        if inRgb is not None:
93            frame = inRgb.getCvFrame()
94
95        if inDet is not None:
96            detections = inDet.detections
97
98        if frame is not None:
99            displayFrame("rgb", frame)
100
101        if cv2.waitKey(1) == ord('q'):
102            break
103
104print("To view the encoded data, convert the stream file (.h265) into a video file (.mp4), using a command below:")
105print("ffmpeg -framerate 30 -i video.h265 -c copy video.mp4")

1#include <iostream>
2
3// Includes common necessary includes for development using depthai library
4#include "depthai/depthai.hpp"
5
6// MobilenetSSD label texts
7static const std::vector<std::string> labelMap = {"background", "aeroplane", "bicycle",     "bird",  "boat",        "bottle", "bus",
8                                                  "car",        "cat",       "chair",       "cow",   "diningtable", "dog",    "horse",
9                                                  "motorbike",  "person",    "pottedplant", "sheep", "sofa",        "train",  "tvmonitor"};
10
11int main(int argc, char** argv) {
12    using namespace std;
13    // Default blob path provided by Hunter private data download
14    // Applicable for easier example usage only
15    std::string nnPath(BLOB_PATH);
16
17    // If path to blob specified, use that
18    if(argc > 1) {
19        nnPath = std::string(argv[1]);
20    }
21
22    // Print which blob we are using
23    printf("Using blob at path: %s\n", nnPath.c_str());
24
25    // Create pipeline
26    dai::Pipeline pipeline;
27
28    // Define sources and outputs
29    auto camRgb = pipeline.create<dai::node::ColorCamera>();
30    auto videoEncoder = pipeline.create<dai::node::VideoEncoder>();
31    auto nn = pipeline.create<dai::node::MobileNetDetectionNetwork>();
32
33    auto xoutRgb = pipeline.create<dai::node::XLinkOut>();
34    auto videoOut = pipeline.create<dai::node::XLinkOut>();
35    auto nnOut = pipeline.create<dai::node::XLinkOut>();
36
37    xoutRgb->setStreamName("rgb");
38    videoOut->setStreamName("h265");
39    nnOut->setStreamName("nn");
40
41    // Properties
42    camRgb->setBoardSocket(dai::CameraBoardSocket::CAM_A);
43    camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P);
44    camRgb->setPreviewSize(300, 300);
45    camRgb->setInterleaved(false);
46
47    videoEncoder->setDefaultProfilePreset(30, dai::VideoEncoderProperties::Profile::H265_MAIN);
48
49    nn->setConfidenceThreshold(0.5);
50    nn->setBlobPath(nnPath);
51    nn->setNumInferenceThreads(2);
52    nn->input.setBlocking(false);
53
54    // Linking
55    camRgb->video.link(videoEncoder->input);
56    camRgb->preview.link(xoutRgb->input);
57    camRgb->preview.link(nn->input);
58    videoEncoder->bitstream.link(videoOut->input);
59    nn->out.link(nnOut->input);
60
61    // Connect to device and start pipeline
62    dai::Device device(pipeline);
63
64    // Queues
65    int queueSize = 8;
66    auto qRgb = device.getOutputQueue("rgb", queueSize);
67    auto qDet = device.getOutputQueue("nn", queueSize);
68    auto qRgbEnc = device.getOutputQueue("h265", 30, true);
69
70    cv::Mat frame;
71    std::vector<dai::ImgDetection> detections;
72
73    // Add bounding boxes and text to the frame and show it to the user
74    auto displayFrame = [](std::string name, cv::Mat frame, std::vector<dai::ImgDetection>& detections) {
75        auto color = cv::Scalar(255, 0, 0);
76        // nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height
77        for(auto& detection : detections) {
78            int x1 = detection.xmin * frame.cols;
79            int y1 = detection.ymin * frame.rows;
80            int x2 = detection.xmax * frame.cols;
81            int y2 = detection.ymax * frame.rows;
82
83            uint32_t labelIndex = detection.label;
84            std::string labelStr = to_string(labelIndex);
85            if(labelIndex < labelMap.size()) {
86                labelStr = labelMap[labelIndex];
87            }
88            cv::putText(frame, labelStr, cv::Point(x1 + 10, y1 + 20), cv::FONT_HERSHEY_TRIPLEX, 0.5, color);
89            std::stringstream confStr;
90            confStr << std::fixed << std::setprecision(2) << detection.confidence * 100;
91            cv::putText(frame, confStr.str(), cv::Point(x1 + 10, y1 + 40), cv::FONT_HERSHEY_TRIPLEX, 0.5, color);
92            cv::rectangle(frame, cv::Rect(cv::Point(x1, y1), cv::Point(x2, y2)), color, cv::FONT_HERSHEY_SIMPLEX);
93        }
94        // Show the frame
95        cv::imshow(name, frame);
96    };
97
98    auto videoFile = std::ofstream("video.h264", std::ios::binary);
99
100    while(true) {
101        auto inRgb = qRgb->tryGet<dai::ImgFrame>();
102        auto inDet = qDet->tryGet<dai::ImgDetections>();
103
104        auto out = qRgbEnc->get<dai::ImgFrame>();
105        videoFile.write((char*)out->getData().data(), out->getData().size());
106
107        if(inRgb) {
108            frame = inRgb->getCvFrame();
109        }
110
111        if(inDet) {
112            detections = inDet->detections;
113        }
114
115        if(!frame.empty()) {
116            displayFrame("rgb", frame, detections);
117        }
118
119        int key = cv::waitKey(1);
120        if(key == 'q' || key == 'Q') {
121            break;
122        }
123    }
124    cout << "To view the encoded data, convert the stream file (.h265) into a video file (.mp4), using a command below:" << endl;
125    cout << "ffmpeg -framerate 30 -i video.h264 -c copy video.mp4" << endl;
126    return 0;
127}