# RGB & MobilenetSSD This example shows how to run MobileNetv2SSD on the RGB input frame, and how to display both the RGB preview and the metadata results from the MobileNetv2SSD on the preview. ### Similar samples: * [# RGB & MobilenetSSD @ 4K](https://docs.luxonis.com/software/depthai/examples/rgb_mobilenet_4k.md) * [Mono & MobilenetSSD](https://docs.luxonis.com/software/depthai/examples/mono_depth_mobilenetssd.md) * [Video & MobilenetSSD](https://docs.luxonis.com/software/depthai/examples/video_mobilenet.md) * [Mono & MobilenetSSD & Depth](https://docs.luxonis.com/software/depthai/examples/mono_depth_mobilenetssd.md) ## Demo ## Setup Please run the [install script](https://github.com/luxonis/depthai-python/blob/main/examples/install_requirements.py) to download all required dependencies. Please note that this script must be ran from git context, so you have to download the [depthai-python](https://github.com/luxonis/depthai-python) repository first and then run the script ```bash git clone https://github.com/luxonis/depthai-python.git cd depthai-python/examples python3 install_requirements.py ``` For additional information, please follow the [installation guide](https://docs.luxonis.com/software/depthai/manual-install.md). ## Source code #### Python ```python #!/usr/bin/env python3 from pathlib import Path import cv2 import depthai as dai import numpy as np import time import argparse nnPathDefault = str((Path(__file__).parent / Path('../models/mobilenet-ssd_openvino_2021.4_6shave.blob')).resolve().absolute()) parser = argparse.ArgumentParser() parser.add_argument('nnPath', nargs='?', help="Path to mobilenet detection network blob", default=nnPathDefault) parser.add_argument('-s', '--sync', action="store_true", help="Sync RGB output with NN output", default=False) args = parser.parse_args() if not Path(nnPathDefault).exists(): import sys raise FileNotFoundError(f'Required file/s not found, please run "{sys.executable} install_requirements.py"') # MobilenetSSD label texts labelMap = ["background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"] # Create pipeline pipeline = dai.Pipeline() # Define sources and outputs camRgb = pipeline.create(dai.node.ColorCamera) nn = pipeline.create(dai.node.MobileNetDetectionNetwork) xoutRgb = pipeline.create(dai.node.XLinkOut) nnOut = pipeline.create(dai.node.XLinkOut) nnNetworkOut = pipeline.create(dai.node.XLinkOut) xoutRgb.setStreamName("rgb") nnOut.setStreamName("nn") nnNetworkOut.setStreamName("nnNetwork"); # Properties camRgb.setPreviewSize(300, 300) camRgb.setInterleaved(False) camRgb.setFps(40) # Define a neural network that will make predictions based on the source frames nn.setConfidenceThreshold(0.5) nn.setBlobPath(args.nnPath) nn.setNumInferenceThreads(2) nn.input.setBlocking(False) # Linking if args.sync: nn.passthrough.link(xoutRgb.input) else: camRgb.preview.link(xoutRgb.input) camRgb.preview.link(nn.input) nn.out.link(nnOut.input) nn.outNetwork.link(nnNetworkOut.input); # Connect to device and start pipeline with dai.Device(pipeline) as device: # Output queues will be used to get the rgb frames and nn data from the outputs defined above qRgb = device.getOutputQueue(name="rgb", maxSize=4, blocking=False) qDet = device.getOutputQueue(name="nn", maxSize=4, blocking=False) qNN = device.getOutputQueue(name="nnNetwork", maxSize=4, blocking=False); frame = None detections = [] startTime = time.monotonic() counter = 0 color2 = (255, 255, 255) # nn data (bounding box locations) are in <0..1> range - they need to be normalized with frame width/height def frameNorm(frame, bbox): normVals = np.full(len(bbox), frame.shape[0]) normVals[::2] = frame.shape[1] return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int) def displayFrame(name, frame): color = (255, 0, 0) for detection in detections: bbox = frameNorm(frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax)) cv2.putText(frame, labelMap[detection.label], (bbox[0] + 10, bbox[1] + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color) cv2.putText(frame, f"{int(detection.confidence * 100)}%", (bbox[0] + 10, bbox[1] + 40), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color) cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color, 2) # Show the frame cv2.imshow(name, frame) printOutputLayersOnce = True while True: if args.sync: # Use blocking get() call to catch frame and inference result synced inRgb = qRgb.get() inDet = qDet.get() inNN = qNN.get() else: # Instead of get (blocking), we use tryGet (non-blocking) which will return the available data or None otherwise inRgb = qRgb.tryGet() inDet = qDet.tryGet() inNN = qNN.tryGet() if inRgb is not None: frame = inRgb.getCvFrame() cv2.putText(frame, "NN fps: {:.2f}".format(counter / (time.monotonic() - startTime)), (2, frame.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.4, color2) if inDet is not None: detections = inDet.detections counter += 1 if printOutputLayersOnce and inNN is not None: toPrint = 'Output layer names:' for ten in inNN.getAllLayerNames(): toPrint = f'{toPrint} {ten},' print(toPrint) printOutputLayersOnce = False; # If the frame is available, draw bounding boxes on it and show the frame if frame is not None: displayFrame("rgb", frame) if cv2.waitKey(1) == ord('q'): break ``` #### C++ ```cpp #include #include #include #include "utility.hpp" // Includes common necessary includes for development using depthai library #include "depthai/depthai.hpp" // MobilenetSSD label texts static const std::vector labelMap = {"background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"}; static std::atomic syncNN{true}; int main(int argc, char** argv) { using namespace std; using namespace std::chrono; // Default blob path provided by Hunter private data download // Applicable for easier example usage only std::string nnPath(BLOB_PATH); // If path to blob specified, use that if(argc > 1) { nnPath = std::string(argv[1]); } // Print which blob we are using printf("Using blob at path: %s\n", nnPath.c_str()); // Create pipeline dai::Pipeline pipeline; // Define sources and outputs auto camRgb = pipeline.create(); auto nn = pipeline.create(); auto xoutRgb = pipeline.create(); auto nnOut = pipeline.create(); auto nnNetworkOut = pipeline.create(); xoutRgb->setStreamName("rgb"); nnOut->setStreamName("nn"); nnNetworkOut->setStreamName("nnNetwork"); // Properties camRgb->setPreviewSize(300, 300); // NN input camRgb->setInterleaved(false); camRgb->setFps(40); // Define a neural network that will make predictions based on the source frames nn->setConfidenceThreshold(0.5); nn->setBlobPath(nnPath); nn->setNumInferenceThreads(2); nn->input.setBlocking(false); // Linking if(syncNN) { nn->passthrough.link(xoutRgb->input); } else { camRgb->preview.link(xoutRgb->input); } camRgb->preview.link(nn->input); nn->out.link(nnOut->input); nn->outNetwork.link(nnNetworkOut->input); // Connect to device and start pipeline dai::Device device(pipeline); // Output queues will be used to get the rgb frames and nn data from the outputs defined above auto qRgb = device.getOutputQueue("rgb", 4, false); auto qDet = device.getOutputQueue("nn", 4, false); auto qNN = device.getOutputQueue("nnNetwork", 4, false); cv::Mat frame; std::vector detections; auto startTime = steady_clock::now(); int counter = 0; float fps = 0; auto color2 = cv::Scalar(255, 255, 255); // Add bounding boxes and text to the frame and show it to the user auto displayFrame = [](std::string name, cv::Mat frame, std::vector& detections) { auto color = cv::Scalar(255, 0, 0); // nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height for(auto& detection : detections) { int x1 = detection.xmin * frame.cols; int y1 = detection.ymin * frame.rows; int x2 = detection.xmax * frame.cols; int y2 = detection.ymax * frame.rows; uint32_t labelIndex = detection.label; std::string labelStr = to_string(labelIndex); if(labelIndex < labelMap.size()) { labelStr = labelMap[labelIndex]; } cv::putText(frame, labelStr, cv::Point(x1 + 10, y1 + 20), cv::FONT_HERSHEY_TRIPLEX, 0.5, color); std::stringstream confStr; confStr << std::fixed << std::setprecision(2) << detection.confidence * 100; cv::putText(frame, confStr.str(), cv::Point(x1 + 10, y1 + 40), cv::FONT_HERSHEY_TRIPLEX, 0.5, color); cv::rectangle(frame, cv::Rect(cv::Point(x1, y1), cv::Point(x2, y2)), color, cv::FONT_HERSHEY_SIMPLEX); } // Show the frame cv::imshow(name, frame); }; bool printOutputLayersOnce = true; while(true) { std::shared_ptr inRgb; std::shared_ptr inDet; std::shared_ptr inNN; if(syncNN) { inRgb = qRgb->get(); inDet = qDet->get(); inNN = qNN->get(); } else { inRgb = qRgb->tryGet(); inDet = qDet->tryGet(); inNN = qNN->tryGet(); } counter++; auto currentTime = steady_clock::now(); auto elapsed = duration_cast>(currentTime - startTime); if(elapsed > seconds(1)) { fps = counter / elapsed.count(); counter = 0; startTime = currentTime; } if(inRgb) { frame = inRgb->getCvFrame(); std::stringstream fpsStr; fpsStr << "NN fps: " << std::fixed << std::setprecision(2) << fps; cv::putText(frame, fpsStr.str(), cv::Point(2, inRgb->getHeight() - 4), cv::FONT_HERSHEY_TRIPLEX, 0.4, color2); } if(inDet) { detections = inDet->detections; } if(printOutputLayersOnce && inNN) { std::cout << "Output layer names: "; for(const auto& ten : inNN->getAllLayerNames()) { std::cout << ten << ", "; } std::cout << std::endl; printOutputLayersOnce = false; } if(!frame.empty()) { displayFrame("video", frame, detections); } int key = cv::waitKey(1); if(key == 'q' || key == 'Q') { return 0; } } return 0; } ``` ## Pipeline ### examples/rgb_mobilenet.pipeline.json ```json { "pipeline": { "connections": [ { "node1Id": 0, "node1Output": "preview", "node1OutputGroup": "", "node2Id": 2, "node2Input": "in", "node2InputGroup": "" }, { "node1Id": 0, "node1Output": "preview", "node1OutputGroup": "", "node2Id": 1, "node2Input": "in", "node2InputGroup": "" }, { "node1Id": 1, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "in", "node2InputGroup": "" }, { "node1Id": 1, "node1Output": "outNetwork", "node1OutputGroup": "", "node2Id": 4, "node2Input": "in", "node2InputGroup": "" } ], "globalProperties": { "calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1 }, "nodes": [ [ 0, { "id": 0, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 1, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 6, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "still" ], { "blocking": false, "group": "", "id": 7, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 2, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "video" ], { "blocking": false, "group": "", "id": 3, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "isp" ], { "blocking": false, "group": "", "id": 4, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "preview" ], { "blocking": false, "group": "", "id": 5, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 8, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ColorCamera", "properties": { "boardSocket": -1, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 40.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": false, "isp3aFps": 0, "ispScale": { "horizDenominator": 0, "horizNumerator": 0, "vertDenominator": 0, "vertNumerator": 0 }, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 300, "previewKeepAspectRatio": true, "previewWidth": 300, "rawPacked": null, "resolution": 0, "sensorCropX": -1.0, "sensorCropY": -1.0, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1 } } ], [ 1, { "id": 1, "ioInfo": [ [ [ "", "in" ], { "blocking": false, "group": "", "id": 9, "name": "in", "queueSize": 5, "type": 3, "waitForMessage": true } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 10, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "passthrough" ], { "blocking": false, "group": "", "id": 11, "name": "passthrough", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "DetectionNetwork", "properties": { "blobSize": 14499200, "blobUri": "asset:__blob", "numFrames": 8, "numNCEPerThread": 0, "numThreads": 2, "parser": { "anchorMasks": {}, "anchors": [], "classes": 0, "confidenceThreshold": 0.5, "coordinates": 0, "iouThreshold": 0.0, "nnFamily": 1 } } } ], [ 2, { "id": 2, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 12, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "rgb" } } ], [ 3, { "id": 3, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 13, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "nn" } } ], [ 4, { "id": 4, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 14, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "nnNetwork" } } ] ] } } ``` ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.