RGB-Left Align

This example demonstrates how to align depth information from a left camera to an RGB camera. This is particularly useful for applications where you need to overlay or compare depth and color data. An OpenCV window is created to display the blended image of the RGB and aligned depth data. Trackbars are provided to adjust the blending ratio and the static depth plane.

Demo

Setup

Please run the install script to download all required dependencies. Please note that this script must be ran from git context, so you have to download the depthai-python repository first and then run the script

Command Line

1git clone https://github.com/luxonis/depthai-python.git
2cd depthai-python/examples
3python3 install_requirements.py

For additional information, please follow the installation guide.

Source code

Python

C++

Python

GitHub

1import cv2
2import depthai as dai
3from datetime import timedelta
4
5# This is an interactive example that shows how two frame sources without depth information.
6FPS = 30.0
7
8RGB_SOCKET = dai.CameraBoardSocket.CAM_A
9LEFT_SOCKET = dai.CameraBoardSocket.CAM_B
10ALIGN_SOCKET = LEFT_SOCKET
11
12COLOR_RESOLUTION = dai.ColorCameraProperties.SensorResolution.THE_1080_P
13LEFT_RIGHT_RESOLUTION = dai.MonoCameraProperties.SensorResolution.THE_720_P
14
15device = dai.Device()
16pipeline = dai.Pipeline()
17
18# Define sources and outputs
19camRgb = pipeline.create(dai.node.ColorCamera)
20left = pipeline.create(dai.node.MonoCamera)
21sync = pipeline.create(dai.node.Sync)
22out = pipeline.create(dai.node.XLinkOut)
23align = pipeline.create(dai.node.ImageAlign)
24cfgIn = pipeline.create(dai.node.XLinkIn)
25
26left.setResolution(LEFT_RIGHT_RESOLUTION)
27left.setBoardSocket(LEFT_SOCKET)
28left.setFps(FPS)
29
30camRgb.setBoardSocket(RGB_SOCKET)
31camRgb.setResolution(COLOR_RESOLUTION)
32camRgb.setFps(FPS)
33camRgb.setIspScale(1, 3)
34
35out.setStreamName("out")
36
37sync.setSyncThreshold(timedelta(seconds=(1 / FPS) * 0.5))
38
39cfgIn.setStreamName("config")
40
41cfg = align.initialConfig.get()
42staticDepthPlane = cfg.staticDepthPlane
43
44# Linking
45align.outputAligned.link(sync.inputs["aligned"])
46camRgb.isp.link(sync.inputs["rgb"])
47camRgb.isp.link(align.inputAlignTo)
48left.out.link(align.input)
49sync.out.link(out.input)
50cfgIn.out.link(align.inputConfig)
51
52
53rgbWeight = 0.4
54leftWeight = 0.6
55
56
57def updateBlendWeights(percentRgb):
58    """
59    Update the rgb and left weights used to blend rgb/left image
60
61    @param[in] percent_rgb The rgb weight expressed as a percentage (0..100)
62    """
63    global leftWeight
64    global rgbWeight
65    rgbWeight = float(percentRgb) / 100.0
66    leftWeight = 1.0 - rgbWeight
67
68def updateDepthPlane(depth):
69    global staticDepthPlane
70    staticDepthPlane = depth
71
72# Connect to device and start pipeline
73with device:
74    device.startPipeline(pipeline)
75    queue = device.getOutputQueue("out", 8, False)
76    cfgQ = device.getInputQueue("config")
77
78    # Configure windows; trackbar adjusts blending ratio of rgb/depth
79    windowName = "rgb-left"
80
81    # Set the window to be resizable and the initial size
82    cv2.namedWindow(windowName, cv2.WINDOW_NORMAL)
83    cv2.resizeWindow(windowName, 1280, 720)
84    cv2.createTrackbar(
85        "RGB Weight %",
86        windowName,
87        int(rgbWeight * 100),
88        100,
89        updateBlendWeights,
90    )
91    cv2.createTrackbar(
92        "Static Depth Plane [mm]",
93        windowName,
94        0,
95        2000,
96        updateDepthPlane,
97    )
98    while True:
99        messageGroup = queue.get()
100        assert isinstance(messageGroup, dai.MessageGroup)
101        frameRgb = messageGroup["rgb"]
102        assert isinstance(frameRgb, dai.ImgFrame)
103        leftAligned = messageGroup["aligned"]
104        assert isinstance(leftAligned, dai.ImgFrame)
105
106        frameRgbCv = frameRgb.getCvFrame()
107        # Colorize the aligned depth
108        leftCv = leftAligned.getCvFrame()
109
110        if len(leftCv.shape) == 2:
111            leftCv = cv2.cvtColor(leftCv, cv2.COLOR_GRAY2BGR)
112        if leftCv.shape != frameRgbCv.shape:
113            leftCv = cv2.resize(leftCv, (frameRgbCv.shape[1], frameRgbCv.shape[0]))
114
115        blended = cv2.addWeighted(frameRgbCv, rgbWeight, leftCv, leftWeight, 0)
116        cv2.imshow(windowName, blended)
117
118        key = cv2.waitKey(1)
119        if key == ord("q"):
120            break
121
122        cfg.staticDepthPlane = staticDepthPlane
123        cfgQ.send(cfg)

Need assistance?

Head over to Discussion Forum for technical support or any other questions you might have.

ON THIS PAGE

RGB-Left Align

Demo

Setup

Source code

Need assistance?