DepthAI Tutorials
DepthAI API References

ON THIS PAGE

  • Mono & MobilenetSSD & Depth
  • Similar samples:
  • Demo
  • Setup
  • Source code

Mono & MobilenetSSD & Depth

This example shows how to run MobileNetv2SSD on the left grayscale camera in parallel with running the disparity depth results, displaying both the depth map and the right grayscale stream, with the bounding box from the neural network overlaid. It's a combination of Depth Preview and Mono & MobilenetSSD.

Similar samples:

Demo

Setup

Please run the install script to download all required dependencies. Please note that this script must be ran from git context, so you have to download the depthai-python repository first and then run the script
Command Line
1git clone https://github.com/luxonis/depthai-python.git
2cd depthai-python/examples
3python3 install_requirements.py
For additional information, please follow the installation guide.

Source code

Python
C++
Python
GitHub
1#!/usr/bin/env python3
2
3from pathlib import Path
4import sys
5import cv2
6import depthai as dai
7import numpy as np
8
9# Get argument first
10nnPath = str((Path(__file__).parent / Path('../models/mobilenet-ssd_openvino_2021.4_6shave.blob')).resolve().absolute())
11if len(sys.argv) > 1:
12    nnPath = sys.argv[1]
13
14if not Path(nnPath).exists():
15    import sys
16    raise FileNotFoundError(f'Required file/s not found, please run "{sys.executable} install_requirements.py"')
17
18# MobilenetSSD label nnLabels
19labelMap = ["background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow",
20            "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"]
21
22# Create pipeline
23pipeline = dai.Pipeline()
24
25# Define sources and outputs
26monoRight = pipeline.create(dai.node.MonoCamera)
27monoLeft = pipeline.create(dai.node.MonoCamera)
28stereo = pipeline.create(dai.node.StereoDepth)
29manip = pipeline.create(dai.node.ImageManip)
30nn = pipeline.create(dai.node.MobileNetDetectionNetwork)
31
32nnOut = pipeline.create(dai.node.XLinkOut)
33disparityOut = pipeline.create(dai.node.XLinkOut)
34xoutRight = pipeline.create(dai.node.XLinkOut)
35
36disparityOut.setStreamName("disparity")
37xoutRight.setStreamName("rectifiedRight")
38nnOut.setStreamName("nn")
39
40# Properties
41monoRight.setCamera("right")
42monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
43monoLeft.setCamera("left")
44monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
45
46# Produce the depth map (using disparity output as it's easier to visualize depth this way)
47stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)
48stereo.setRectifyEdgeFillColor(0)  # Black, to better see the cutout from rectification (black stripe on the edges)
49# Convert the grayscale frame into the nn-acceptable form
50manip.initialConfig.setResize(300, 300)
51# The NN model expects BGR input. By default ImageManip output type would be same as input (gray in this case)
52manip.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
53
54# Define a neural network that will make predictions based on the source frames
55nn.setConfidenceThreshold(0.5)
56nn.setBlobPath(nnPath)
57nn.setNumInferenceThreads(2)
58nn.input.setBlocking(False)
59
60# Linking
61monoRight.out.link(stereo.right)
62monoLeft.out.link(stereo.left)
63stereo.rectifiedRight.link(manip.inputImage)
64stereo.disparity.link(disparityOut.input)
65manip.out.link(nn.input)
66manip.out.link(xoutRight.input)
67nn.out.link(nnOut.input)
68
69# Connect to device and start pipeline
70with dai.Device(pipeline) as device:
71
72    # Output queues will be used to get the grayscale / depth frames and nn data from the outputs defined above
73    qRight = device.getOutputQueue("rectifiedRight", maxSize=4, blocking=False)
74    qDisparity = device.getOutputQueue("disparity", maxSize=4, blocking=False)
75    qDet = device.getOutputQueue("nn", maxSize=4, blocking=False)
76
77    rightFrame = None
78    disparityFrame = None
79    detections = []
80
81    # nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height
82    def frameNorm(frame, bbox):
83        normVals = np.full(len(bbox), frame.shape[0])
84        normVals[::2] = frame.shape[1]
85        return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)
86
87    # Add bounding boxes and text to the frame and show it to the user
88    def show(name, frame):
89        color = (255, 0, 0)
90        for detection in detections:
91            bbox = frameNorm(frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax))
92            cv2.putText(frame, labelMap[detection.label], (bbox[0] + 10, bbox[1] + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)
93            cv2.putText(frame, f"{int(detection.confidence * 100)}%", (bbox[0] + 10, bbox[1] + 40), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)
94            cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color, 2)
95        # Show the frame
96        cv2.imshow(name, frame)
97
98    disparityMultiplier = 255 / stereo.initialConfig.getMaxDisparity()
99
100    while True:
101        # Instead of get (blocking), we use tryGet (non-blocking) which will return the available data or None otherwise
102        if qDet.has():
103            detections = qDet.get().detections
104
105        if qRight.has():
106            rightFrame = qRight.get().getCvFrame()
107
108        if qDisparity.has():
109            # Frame is transformed, normalized, and color map will be applied to highlight the depth info
110            disparityFrame = qDisparity.get().getFrame()
111            disparityFrame = (disparityFrame*disparityMultiplier).astype(np.uint8)
112            # Available color maps: https://docs.opencv.org/3.4/d3/d50/group__imgproc__colormap.html
113            disparityFrame = cv2.applyColorMap(disparityFrame, cv2.COLORMAP_JET)
114            show("disparity", disparityFrame)
115
116        if rightFrame is not None:
117            show("rectified right", rightFrame)
118
119        if cv2.waitKey(1) == ord('q'):
120            break

Need assistance?

Head over to Discussion Forum for technical support or any other questions you might have.