RGB & MobileNetSSD @ 4K

View as Markdown

1#!/usr/bin/env python3
2
3from pathlib import Path
4import sys
5import cv2
6import depthai as dai
7import numpy as np
8
9# Get argument first
10nnPath = str((Path(__file__).parent / Path('../models/mobilenet-ssd_openvino_2021.4_5shave.blob')).resolve().absolute())
11if len(sys.argv) > 1:
12    nnPath = sys.argv[1]
13
14if not Path(nnPath).exists():
15    import sys
16    raise FileNotFoundError(f'Required file/s not found, please run "{sys.executable} install_requirements.py"')
17
18# MobilenetSSD label texts
19labelMap = ["background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow",
20            "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"]
21
22# Create pipeline
23pipeline = dai.Pipeline()
24
25# Define sources and outputs
26camRgb = pipeline.create(dai.node.ColorCamera)
27nn = pipeline.create(dai.node.MobileNetDetectionNetwork)
28
29xoutVideo = pipeline.create(dai.node.XLinkOut)
30xoutPreview = pipeline.create(dai.node.XLinkOut)
31nnOut = pipeline.create(dai.node.XLinkOut)
32
33xoutVideo.setStreamName("video")
34xoutPreview.setStreamName("preview")
35nnOut.setStreamName("nn")
36
37# Properties
38camRgb.setPreviewSize(300, 300)    # NN input
39camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_4_K)
40camRgb.setInterleaved(False)
41camRgb.setPreviewKeepAspectRatio(False)
42# Define a neural network that will make predictions based on the source frames
43nn.setConfidenceThreshold(0.5)
44nn.setBlobPath(nnPath)
45nn.setNumInferenceThreads(2)
46nn.input.setBlocking(False)
47
48# Linking
49camRgb.video.link(xoutVideo.input)
50camRgb.preview.link(xoutPreview.input)
51camRgb.preview.link(nn.input)
52nn.out.link(nnOut.input)
53
54# Connect to device and start pipeline
55with dai.Device(pipeline) as device:
56
57    # Output queues will be used to get the frames and nn data from the outputs defined above
58    qVideo = device.getOutputQueue(name="video", maxSize=4, blocking=False)
59    qPreview = device.getOutputQueue(name="preview", maxSize=4, blocking=False)
60    qDet = device.getOutputQueue(name="nn", maxSize=4, blocking=False)
61
62    previewFrame = None
63    videoFrame = None
64    detections = []
65
66    # nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height
67    def frameNorm(frame, bbox):
68        normVals = np.full(len(bbox), frame.shape[0])
69        normVals[::2] = frame.shape[1]
70        return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)
71
72    def displayFrame(name, frame):
73        color = (255, 0, 0)
74        for detection in detections:
75            bbox = frameNorm(frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax))
76            cv2.putText(frame, labelMap[detection.label], (bbox[0] + 10, bbox[1] + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)
77            cv2.putText(frame, f"{int(detection.confidence * 100)}%", (bbox[0] + 10, bbox[1] + 40), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)
78            cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color, 2)
79        # Show the frame
80        cv2.imshow(name, frame)
81
82    cv2.namedWindow("video", cv2.WINDOW_NORMAL)
83    cv2.resizeWindow("video", 1280, 720)
84    print("Resize video window with mouse drag!")
85
86    while True:
87        # Instead of get (blocking), we use tryGet (non-blocking) which will return the available data or None otherwise
88        inVideo = qVideo.tryGet()
89        inPreview = qPreview.tryGet()
90        inDet = qDet.tryGet()
91
92        if inVideo is not None:
93            videoFrame = inVideo.getCvFrame()
94
95        if inPreview is not None:
96            previewFrame = inPreview.getCvFrame()
97
98        if inDet is not None:
99            detections = inDet.detections
100
101        if videoFrame is not None:
102            displayFrame("video", videoFrame)
103
104        if previewFrame is not None:
105            displayFrame("preview", previewFrame)
106
107        if cv2.waitKey(1) == ord('q'):
108            break

1#include <iostream>
2
3// Includes common necessary includes for development using depthai library
4#include "depthai/depthai.hpp"
5
6// MobilenetSSD label texts
7static const std::vector<std::string> labelMap = {"background", "aeroplane", "bicycle",     "bird",  "boat",        "bottle", "bus",
8                                                  "car",        "cat",       "chair",       "cow",   "diningtable", "dog",    "horse",
9                                                  "motorbike",  "person",    "pottedplant", "sheep", "sofa",        "train",  "tvmonitor"};
10
11int main(int argc, char** argv) {
12    using namespace std;
13    // Default blob path provided by Hunter private data download
14    // Applicable for easier example usage only
15    std::string nnPath(BLOB_PATH);
16
17    // If path to blob specified, use that
18    if(argc > 1) {
19        nnPath = std::string(argv[1]);
20    }
21
22    // Print which blob we are using
23    printf("Using blob at path: %s\n", nnPath.c_str());
24
25    // Create pipeline
26    dai::Pipeline pipeline;
27
28    // Define sources and outputs
29    auto camRgb = pipeline.create<dai::node::ColorCamera>();
30    auto nn = pipeline.create<dai::node::MobileNetDetectionNetwork>();
31
32    auto xoutVideo = pipeline.create<dai::node::XLinkOut>();
33    auto xoutPreview = pipeline.create<dai::node::XLinkOut>();
34    auto nnOut = pipeline.create<dai::node::XLinkOut>();
35
36    xoutVideo->setStreamName("video");
37    xoutPreview->setStreamName("preview");
38    nnOut->setStreamName("nn");
39
40    // Properties
41    camRgb->setPreviewSize(300, 300);  // NN input
42    camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_4_K);
43    camRgb->setInterleaved(false);
44    camRgb->setPreviewKeepAspectRatio(false);
45    // Define a neural network that will make predictions based on the source frames
46    nn->setConfidenceThreshold(0.5);
47    nn->setBlobPath(nnPath);
48    nn->setNumInferenceThreads(2);
49    nn->input.setBlocking(false);
50
51    // Linking
52    camRgb->video.link(xoutVideo->input);
53    camRgb->preview.link(xoutPreview->input);
54    camRgb->preview.link(nn->input);
55    nn->out.link(nnOut->input);
56
57    // Connect to device and start pipeline
58    dai::Device device(pipeline);
59
60    // Output queues will be used to get the frames and nn data from the outputs defined above
61    auto qVideo = device.getOutputQueue("video", 4, false);
62    auto qPreview = device.getOutputQueue("preview", 4, false);
63    auto qDet = device.getOutputQueue("nn", 4, false);
64
65    cv::Mat previewFrame;
66    cv::Mat videoFrame;
67    std::vector<dai::ImgDetection> detections;
68
69    // Add bounding boxes and text to the frame and show it to the user
70    auto displayFrame = [](std::string name, cv::Mat frame, std::vector<dai::ImgDetection>& detections) {
71        auto color = cv::Scalar(255, 0, 0);
72        // nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height
73        for(auto& detection : detections) {
74            int x1 = detection.xmin * frame.cols;
75            int y1 = detection.ymin * frame.rows;
76            int x2 = detection.xmax * frame.cols;
77            int y2 = detection.ymax * frame.rows;
78
79            uint32_t labelIndex = detection.label;
80            std::string labelStr = to_string(labelIndex);
81            if(labelIndex < labelMap.size()) {
82                labelStr = labelMap[labelIndex];
83            }
84            cv::putText(frame, labelStr, cv::Point(x1 + 10, y1 + 20), cv::FONT_HERSHEY_TRIPLEX, 0.5, color);
85            std::stringstream confStr;
86            confStr << std::fixed << std::setprecision(2) << detection.confidence * 100;
87            cv::putText(frame, confStr.str(), cv::Point(x1 + 10, y1 + 40), cv::FONT_HERSHEY_TRIPLEX, 0.5, color);
88            cv::rectangle(frame, cv::Rect(cv::Point(x1, y1), cv::Point(x2, y2)), color, cv::FONT_HERSHEY_SIMPLEX);
89        }
90        // Show the frame
91        cv::imshow(name, frame);
92    };
93
94    cv::namedWindow("video", cv::WINDOW_NORMAL);
95    cv::resizeWindow("video", 1280, 720);
96    cout << "Resize video window with mouse drag!" << endl;
97
98    while(true) {
99        // Instead of get (blocking), we use tryGet (non-blocking) which will return the available data or None otherwise
100        auto inVideo = qVideo->tryGet<dai::ImgFrame>();
101        auto inPreview = qPreview->tryGet<dai::ImgFrame>();
102        auto inDet = qDet->tryGet<dai::ImgDetections>();
103
104        if(inVideo) {
105            videoFrame = inVideo->getCvFrame();
106        }
107
108        if(inPreview) {
109            previewFrame = inPreview->getCvFrame();
110        }
111
112        if(inDet) {
113            detections = inDet->detections;
114        }
115
116        if(!videoFrame.empty()) {
117            displayFrame("video", videoFrame, detections);
118        }
119
120        if(!previewFrame.empty()) {
121            displayFrame("preview", previewFrame, detections);
122        }
123
124        int key = cv::waitKey(1);
125        if(key == 'q' || key == 'Q') {
126            return 0;
127        }
128    }
129    return 0;
130}