# Spatial detections on rotated OAK This example is very similar to [RGB & MobilenetSSD with spatial data](https://docs.luxonis.com/software/depthai-components/nodes/mobilenet_spatial_detection_network.md) - it only assumes we have OAK rotated by 180° (upside down) [ColorCamera](https://docs.luxonis.com/software/depthai-components/nodes/color_camera.md) frames are rotated on the sensor itself, by setting `camRgb.setImageOrientation(dai.CameraImageOrientation.ROTATE_180_DEG)`. This means all outputs from the node (still/isp/video/preview) will already be rotated. We rotate `depth` frames after the [StereoDepth](https://docs.luxonis.com/software/depthai-components/nodes/stereo_depth.md) creates them. One might try rotating mono frames before sending them to the [StereoDepth](https://docs.luxonis.com/software/depthai-components/nodes/stereo_depth.md) node, but this wouldn't work as stereo calibration would need to reflect such changes. So we use the [ImageManip](https://docs.luxonis.com/software/depthai-components/nodes/image_manip.md) node to rotate `depth` (code below) and then send it to the [MobileNetSpatialDetectionNetwork](https://docs.luxonis.com/software/depthai-components/nodes/mobilenet_spatial_detection_network.md). ```bash manip = pipeline.createImageManip() # Vertical + Horizontal flip == rotate frame for 180° manip.initialConfig.setVerticalFlip(True) manip.initialConfig.setHorizontalFlip(True) manip.setFrameType(dai.ImgFrame.Type.RAW16) stereo.depth.link(manip.inputImage) ``` [MobileNetSpatialDetectionNetwork](https://docs.luxonis.com/software/depthai-components/nodes/mobilenet_spatial_detection_network.md) node then receives correctly rotated color and depth frame, which results in correct spatial detection output. ### Similar samples: * [RGB & MobilenetSSD with spatial data](https://docs.luxonis.com/software/depthai-components/nodes/mobilenet_spatial_detection_network.md) * [Spatial object tracker on RGB](https://docs.luxonis.com/software/depthai/examples/spatial_object_tracker.md) * [Mono & MobilenetSSD with spatial data](https://docs.luxonis.com/software/depthai/examples/mono_depth_mobilenetssd.md) * [RGB & TinyYolo with spatial data](https://docs.luxonis.com/software/depthai-components/nodes/yolo_spatial_detection_network.md) ## Setup Please run the [install script](https://github.com/luxonis/depthai-python/blob/main/examples/install_requirements.py) to download all required dependencies. Please note that this script must be ran from git context, so you have to download the [depthai-python](https://github.com/luxonis/depthai-python) repository first and then run the script ```bash git clone https://github.com/luxonis/depthai-python.git cd depthai-python/examples python3 install_requirements.py ``` For additional information, please follow the [installation guide](https://docs.luxonis.com/software/depthai/manual-install.md). ## Source code #### Python ```python #!/usr/bin/env python3 from pathlib import Path import sys import cv2 import depthai as dai import numpy as np ''' Spatial object detections demo for 180° rotated OAK camera. ''' # Get argument first nnBlobPath = str((Path(__file__).parent / Path('../models/mobilenet-ssd_openvino_2021.4_6shave.blob')).resolve().absolute()) if len(sys.argv) > 1: nnBlobPath = sys.argv[1] if not Path(nnBlobPath).exists(): import sys raise FileNotFoundError(f'Required file/s not found, please run "{sys.executable} install_requirements.py"') # MobilenetSSD label texts labelMap = ["background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"] syncNN = True # Create pipeline pipeline = dai.Pipeline() # Define sources and outputs camRgb = pipeline.createColorCamera() spatialDetectionNetwork = pipeline.create(dai.node.MobileNetSpatialDetectionNetwork) monoLeft = pipeline.create(dai.node.MonoCamera) monoRight = pipeline.create(dai.node.MonoCamera) stereo = pipeline.create(dai.node.StereoDepth) xoutRgb = pipeline.create(dai.node.XLinkOut) xoutNN = pipeline.create(dai.node.XLinkOut) xoutDepth = pipeline.create(dai.node.XLinkOut) xoutRgb.setStreamName("rgb") xoutNN.setStreamName("detections") xoutDepth.setStreamName("depth") # Properties camRgb.setPreviewSize(300, 300) camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P) camRgb.setInterleaved(False) camRgb.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR) camRgb.setImageOrientation(dai.CameraImageOrientation.ROTATE_180_DEG) monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P) monoLeft.setCamera("left") monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P) monoRight.setCamera("right") # Setting node configs stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY) # Align depth map to the perspective of RGB camera, on which inference is done stereo.setDepthAlign(dai.CameraBoardSocket.CAM_A) stereo.setSubpixel(True) stereo.setOutputSize(monoLeft.getResolutionWidth(), monoLeft.getResolutionHeight()) rotate_stereo_manip = pipeline.createImageManip() rotate_stereo_manip.initialConfig.setVerticalFlip(True) rotate_stereo_manip.initialConfig.setHorizontalFlip(True) rotate_stereo_manip.setFrameType(dai.ImgFrame.Type.RAW16) stereo.depth.link(rotate_stereo_manip.inputImage) spatialDetectionNetwork.setBlobPath(nnBlobPath) spatialDetectionNetwork.setConfidenceThreshold(0.5) spatialDetectionNetwork.input.setBlocking(False) spatialDetectionNetwork.setBoundingBoxScaleFactor(0.5) spatialDetectionNetwork.setDepthLowerThreshold(100) spatialDetectionNetwork.setDepthUpperThreshold(5000) # Linking monoLeft.out.link(stereo.left) monoRight.out.link(stereo.right) camRgb.preview.link(spatialDetectionNetwork.input) if syncNN: spatialDetectionNetwork.passthrough.link(xoutRgb.input) else: camRgb.preview.link(xoutRgb.input) spatialDetectionNetwork.out.link(xoutNN.input) rotate_stereo_manip.out.link(spatialDetectionNetwork.inputDepth) spatialDetectionNetwork.passthroughDepth.link(xoutDepth.input) color = (255, 0, 0) # Connect to device and start pipeline with dai.Device(pipeline) as device: # Output queues will be used to get the rgb frames and nn data from the outputs defined above previewQueue = device.getOutputQueue(name="rgb", maxSize=4, blocking=False) detectionNNQueue = device.getOutputQueue(name="detections", maxSize=4, blocking=False) depthQueue = device.getOutputQueue(name="depth", maxSize=4, blocking=False) while True: inPreview = previewQueue.get() inDet = detectionNNQueue.get() depth = depthQueue.get() frame = inPreview.getCvFrame() depthFrame = depth.getFrame() # depthFrame values are in millimeters depth_downscaled = depthFrame[::4] if np.all(depth_downscaled == 0): min_depth = 0 # Set a default minimum depth value when all elements are zero else: min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) max_depth = np.percentile(depth_downscaled, 99) depthFrameColor = np.interp(depthFrame, (min_depth, max_depth), (0, 255)).astype(np.uint8) depthFrameColor = cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_HOT) detections = inDet.detections # If the frame is available, draw bounding boxes on it and show the frame height = frame.shape[0] width = frame.shape[1] for detection in detections: roiData = detection.boundingBoxMapping roi = roiData.roi roi = roi.denormalize(depthFrameColor.shape[1], depthFrameColor.shape[0]) topLeft = roi.topLeft() bottomRight = roi.bottomRight() xmin = int(topLeft.x) ymin = int(topLeft.y) xmax = int(bottomRight.x) ymax = int(bottomRight.y) cv2.rectangle(depthFrameColor, (xmin, ymin), (xmax, ymax), color, cv2.FONT_HERSHEY_SCRIPT_SIMPLEX) # Denormalize bounding box x1 = int(detection.xmin * width) x2 = int(detection.xmax * width) y1 = int(detection.ymin * height) y2 = int(detection.ymax * height) try: label = labelMap[detection.label] except: label = detection.label cv2.putText(frame, str(label), (x1 + 10, y1 + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255) cv2.putText(frame, "{:.2f}".format(detection.confidence*100), (x1 + 10, y1 + 35), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255) cv2.putText(frame, f"X: {int(detection.spatialCoordinates.x)} mm", (x1 + 10, y1 + 50), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255) cv2.putText(frame, f"Y: {int(detection.spatialCoordinates.y)} mm", (x1 + 10, y1 + 65), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255) cv2.putText(frame, f"Z: {int(detection.spatialCoordinates.z)} mm", (x1 + 10, y1 + 80), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255) cv2.rectangle(frame, (x1, y1), (x2, y2), (255, 0, 0), cv2.FONT_HERSHEY_SIMPLEX) cv2.imshow("depth", depthFrameColor) cv2.imshow("preview", frame) if cv2.waitKey(1) == ord('q'): break ``` #### C++ WIP ## Pipeline ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.