# Video & MobilenetSSD This example shows how to MobileNetv2SSD on the RGB input frame, which is read from the specified file, and not from the RGB camera, and how to display both the RGB frame and the metadata results from the MobileNetv2SSD on the frame. DepthAI is used here only as a processing unit ### Similar samples: * [RGB & MobilenetSSD](https://docs.luxonis.com/software/depthai/examples/rgb_mobilenet.md) * [RGB & MobilenetSSD @ 4K](https://docs.luxonis.com/software/depthai/examples/rgb_mobilenet_4k.md) * [Mono & MobilenetSSD](https://docs.luxonis.com/software/depthai/examples/mono_depth_mobilenetssd.md) * [Mono & MobilenetSSD & Depth](https://docs.luxonis.com/software/depthai/examples/mono_depth_mobilenetssd.md) ## Demo ## Setup Please run the [install script](https://github.com/luxonis/depthai-python/blob/main/examples/install_requirements.py) to download all required dependencies. Please note that this script must be ran from git context, so you have to download the [depthai-python](https://github.com/luxonis/depthai-python) repository first and then run the script ```bash git clone https://github.com/luxonis/depthai-python.git cd depthai-python/examples python3 install_requirements.py ``` For additional information, please follow the [installation guide](https://docs.luxonis.com/software/depthai/manual-install.md). ## Source code #### Python ```python #!/usr/bin/env python3 from pathlib import Path import sys import cv2 import depthai as dai import numpy as np from time import monotonic # Get argument first parentDir = Path(__file__).parent nnPath = str((parentDir / Path('../models/mobilenet-ssd_openvino_2021.4_8shave.blob')).resolve().absolute()) videoPath = str((parentDir / Path('../models/construction_vest.mp4')).resolve().absolute()) if len(sys.argv) > 2: nnPath = sys.argv[1] videoPath = sys.argv[2] if not Path(nnPath).exists() or not Path(videoPath).exists(): import sys raise FileNotFoundError(f'Required file/s not found, please run "{sys.executable} install_requirements.py"') # MobilenetSSD label texts labelMap = ["background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"] # Create pipeline pipeline = dai.Pipeline() # Define sources and outputs nn = pipeline.create(dai.node.MobileNetDetectionNetwork) xinFrame = pipeline.create(dai.node.XLinkIn) nnOut = pipeline.create(dai.node.XLinkOut) xinFrame.setStreamName("inFrame") nnOut.setStreamName("nn") # Properties nn.setConfidenceThreshold(0.5) nn.setBlobPath(nnPath) nn.setNumInferenceThreads(2) nn.input.setBlocking(False) # Linking xinFrame.out.link(nn.input) nn.out.link(nnOut.input) # Connect to device and start pipeline with dai.Device(pipeline) as device: # Input queue will be used to send video frames to the device. qIn = device.getInputQueue(name="inFrame") # Output queue will be used to get nn data from the video frames. qDet = device.getOutputQueue(name="nn", maxSize=4, blocking=False) frame = None detections = [] # nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height def frameNorm(frame, bbox): normVals = np.full(len(bbox), frame.shape[0]) normVals[::2] = frame.shape[1] return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int) def to_planar(arr: np.ndarray, shape: tuple) -> np.ndarray: return cv2.resize(arr, shape).transpose(2, 0, 1).flatten() def displayFrame(name, frame): for detection in detections: bbox = frameNorm(frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax)) cv2.putText(frame, labelMap[detection.label], (bbox[0] + 10, bbox[1] + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255) cv2.putText(frame, f"{int(detection.confidence * 100)}%", (bbox[0] + 10, bbox[1] + 40), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255) cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (255, 0, 0), 2) # Show the frame cv2.imshow(name, frame) cap = cv2.VideoCapture(videoPath) while cap.isOpened(): read_correctly, frame = cap.read() if not read_correctly: break img = dai.ImgFrame() img.setData(to_planar(frame, (300, 300))) img.setTimestamp(monotonic()) img.setWidth(300) img.setHeight(300) qIn.send(img) inDet = qDet.tryGet() if inDet is not None: detections = inDet.detections if frame is not None: displayFrame("rgb", frame) if cv2.waitKey(1) == ord('q'): break ``` #### C++ ```cpp #include #include #include "utility.hpp" // Includes common necessary includes for development using depthai library #include "depthai/depthai.hpp" // MobilenetSSD label texts static const std::vector labelMap = {"background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"}; int main(int argc, char** argv) { using namespace std; using namespace std::chrono; // Default blob path provided by Hunter private data download // Applicable for easier example usage only std::string nnPath(BLOB_PATH); std::string videoPath(VIDEO_PATH); // If path to blob specified, use that if(argc > 2) { nnPath = std::string(argv[1]); videoPath = std::string(argv[2]); } // Print which blob we are using printf("Using blob at path: %s\n", nnPath.c_str()); printf("Using video at path: %s\n", videoPath.c_str()); // Create pipeline dai::Pipeline pipeline; // Define source and outputs auto nn = pipeline.create(); auto xinFrame = pipeline.create(); auto nnOut = pipeline.create(); xinFrame->setStreamName("inFrame"); nnOut->setStreamName("nn"); // Properties nn->setConfidenceThreshold(0.5); nn->setBlobPath(nnPath); nn->setNumInferenceThreads(2); nn->input.setBlocking(false); // Linking xinFrame->out.link(nn->input); nn->out.link(nnOut->input); // Connect to device and start pipeline dai::Device device(pipeline); // Input queue will be used to send video frames to the device. auto qIn = device.getInputQueue("inFrame"); // Output queue will be used to get nn data from the video frames. auto qDet = device.getOutputQueue("nn", 4, false); // Add bounding boxes and text to the frame and show it to the user auto displayFrame = [](std::string name, cv::Mat frame, std::vector& detections) { auto color = cv::Scalar(255, 0, 0); // nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height for(auto& detection : detections) { int x1 = detection.xmin * frame.cols; int y1 = detection.ymin * frame.rows; int x2 = detection.xmax * frame.cols; int y2 = detection.ymax * frame.rows; uint32_t labelIndex = detection.label; std::string labelStr = to_string(labelIndex); if(labelIndex < labelMap.size()) { labelStr = labelMap[labelIndex]; } cv::putText(frame, labelStr, cv::Point(x1 + 10, y1 + 20), cv::FONT_HERSHEY_TRIPLEX, 0.5, color); std::stringstream confStr; confStr << std::fixed << std::setprecision(2) << detection.confidence * 100; cv::putText(frame, confStr.str(), cv::Point(x1 + 10, y1 + 40), cv::FONT_HERSHEY_TRIPLEX, 0.5, color); cv::rectangle(frame, cv::Rect(cv::Point(x1, y1), cv::Point(x2, y2)), color, cv::FONT_HERSHEY_SIMPLEX); } // Show the frame cv::imshow(name, frame); }; cv::Mat frame; cv::VideoCapture cap(videoPath); cv::namedWindow("inFrame", cv::WINDOW_NORMAL); cv::resizeWindow("inFrame", 1280, 720); std::cout << "Resize video window with mouse drag!" << std::endl; while(cap.isOpened()) { // Read frame from video cap >> frame; if(frame.empty()) break; auto img = std::make_shared(); frame = resizeKeepAspectRatio(frame, cv::Size(300, 300), cv::Scalar(0)); toPlanar(frame, img->getData()); img->setTimestamp(steady_clock::now()); img->setWidth(300); img->setHeight(300); qIn->send(img); auto inDet = qDet->get(); auto detections = inDet->detections; displayFrame("inFrame", frame, detections); int key = cv::waitKey(1); if(key == 'q' || key == 'Q') return 0; } return 0; } ``` ## Pipeline ### examples/video_mobilenet.pipeline.json ```json { "pipeline": { "connections": [ { "node1Id": 1, "node1Output": "out", "node1OutputGroup": "", "node2Id": 0, "node2Input": "in", "node2InputGroup": "" }, { "node1Id": 0, "node1Output": "out", "node1OutputGroup": "", "node2Id": 2, "node2Input": "in", "node2InputGroup": "" } ], "globalProperties": { "calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1 }, "nodes": [ [ 0, { "id": 0, "ioInfo": [ [ [ "", "in" ], { "blocking": false, "group": "", "id": 1, "name": "in", "queueSize": 5, "type": 3, "waitForMessage": true } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 2, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "passthrough" ], { "blocking": false, "group": "", "id": 3, "name": "passthrough", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "DetectionNetwork", "properties": { "blobSize": 14493888, "blobUri": "asset:__blob", "numFrames": 8, "numNCEPerThread": 0, "numThreads": 2, "parser": { "anchorMasks": {}, "anchors": [], "classes": 0, "confidenceThreshold": 0.5, "coordinates": 0, "iouThreshold": 0.0, "nnFamily": 1 } } } ], [ 1, { "id": 1, "ioInfo": [ [ [ "", "out" ], { "blocking": false, "group": "", "id": 4, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "XLinkIn", "properties": { "maxDataSize": 5242880, "numFrames": 8, "streamName": "inFrame" } } ], [ 2, { "id": 2, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 5, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "nn" } } ] ] } } ``` ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.