Mono & MobilenetSSD & Depth

1#!/usr/bin/env python3
2
3from pathlib import Path
4import sys
5import cv2
6import depthai as dai
7import numpy as np
8
9# Get argument first
10nnPath = str((Path(__file__).parent / Path('../models/mobilenet-ssd_openvino_2021.4_6shave.blob')).resolve().absolute())
11if len(sys.argv) > 1:
12    nnPath = sys.argv[1]
13
14if not Path(nnPath).exists():
15    import sys
16    raise FileNotFoundError(f'Required file/s not found, please run "{sys.executable} install_requirements.py"')
17
18# MobilenetSSD label nnLabels
19labelMap = ["background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow",
20            "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"]
21
22# Create pipeline
23pipeline = dai.Pipeline()
24
25# Define sources and outputs
26monoRight = pipeline.create(dai.node.MonoCamera)
27monoLeft = pipeline.create(dai.node.MonoCamera)
28stereo = pipeline.create(dai.node.StereoDepth)
29manip = pipeline.create(dai.node.ImageManip)
30nn = pipeline.create(dai.node.MobileNetDetectionNetwork)
31
32nnOut = pipeline.create(dai.node.XLinkOut)
33disparityOut = pipeline.create(dai.node.XLinkOut)
34xoutRight = pipeline.create(dai.node.XLinkOut)
35
36disparityOut.setStreamName("disparity")
37xoutRight.setStreamName("rectifiedRight")
38nnOut.setStreamName("nn")
39
40# Properties
41monoRight.setCamera("right")
42monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
43monoLeft.setCamera("left")
44monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
45
46# Produce the depth map (using disparity output as it's easier to visualize depth this way)
47stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)
48stereo.setRectifyEdgeFillColor(0)  # Black, to better see the cutout from rectification (black stripe on the edges)
49# Convert the grayscale frame into the nn-acceptable form
50manip.initialConfig.setResize(300, 300)
51# The NN model expects BGR input. By default ImageManip output type would be same as input (gray in this case)
52manip.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
53
54# Define a neural network that will make predictions based on the source frames
55nn.setConfidenceThreshold(0.5)
56nn.setBlobPath(nnPath)
57nn.setNumInferenceThreads(2)
58nn.input.setBlocking(False)
59
60# Linking
61monoRight.out.link(stereo.right)
62monoLeft.out.link(stereo.left)
63stereo.rectifiedRight.link(manip.inputImage)
64stereo.disparity.link(disparityOut.input)
65manip.out.link(nn.input)
66manip.out.link(xoutRight.input)
67nn.out.link(nnOut.input)
68
69# Connect to device and start pipeline
70with dai.Device(pipeline) as device:
71
72    # Output queues will be used to get the grayscale / depth frames and nn data from the outputs defined above
73    qRight = device.getOutputQueue("rectifiedRight", maxSize=4, blocking=False)
74    qDisparity = device.getOutputQueue("disparity", maxSize=4, blocking=False)
75    qDet = device.getOutputQueue("nn", maxSize=4, blocking=False)
76
77    rightFrame = None
78    disparityFrame = None
79    detections = []
80
81    # nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height
82    def frameNorm(frame, bbox):
83        normVals = np.full(len(bbox), frame.shape[0])
84        normVals[::2] = frame.shape[1]
85        return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)
86
87    # Add bounding boxes and text to the frame and show it to the user
88    def show(name, frame):
89        color = (255, 0, 0)
90        for detection in detections:
91            bbox = frameNorm(frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax))
92            cv2.putText(frame, labelMap[detection.label], (bbox[0] + 10, bbox[1] + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)
93            cv2.putText(frame, f"{int(detection.confidence * 100)}%", (bbox[0] + 10, bbox[1] + 40), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)
94            cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color, 2)
95        # Show the frame
96        cv2.imshow(name, frame)
97
98    disparityMultiplier = 255 / stereo.initialConfig.getMaxDisparity()
99
100    while True:
101        # Instead of get (blocking), we use tryGet (non-blocking) which will return the available data or None otherwise
102        if qDet.has():
103            detections = qDet.get().detections
104
105        if qRight.has():
106            rightFrame = qRight.get().getCvFrame()
107
108        if qDisparity.has():
109            # Frame is transformed, normalized, and color map will be applied to highlight the depth info
110            disparityFrame = qDisparity.get().getFrame()
111            disparityFrame = (disparityFrame*disparityMultiplier).astype(np.uint8)
112            # Available color maps: https://docs.opencv.org/3.4/d3/d50/group__imgproc__colormap.html
113            disparityFrame = cv2.applyColorMap(disparityFrame, cv2.COLORMAP_JET)
114            show("disparity", disparityFrame)
115
116        if rightFrame is not None:
117            show("rectified right", rightFrame)
118
119        if cv2.waitKey(1) == ord('q'):
120            break