# RGB-Left Align This example demonstrates how to align depth information from a left camera to an RGB camera. This is particularly useful for applications where you need to overlay or compare depth and color data. An OpenCV window is created to display the blended image of the RGB and aligned depth data. Trackbars are provided to adjust the blending ratio and the static depth plane. ## Demo ## Setup Please run the [install script](https://github.com/luxonis/depthai-python/blob/main/examples/install_requirements.py) to download all required dependencies. Please note that this script must be ran from git context, so you have to download the [depthai-python](https://github.com/luxonis/depthai-python) repository first and then run the script ```bash git clone https://github.com/luxonis/depthai-python.git cd depthai-python/examples python3 install_requirements.py ``` For additional information, please follow the [installation guide](https://docs.luxonis.com/software/depthai/manual-install.md). ## Source code #### Python ```python #!/usr/bin/env python3 import cv2 import depthai as dai from datetime import timedelta import numpy as np # This is an interactive example that shows how two frame sources without depth information. FPS = 30.0 RGB_SOCKET = dai.CameraBoardSocket.CAM_A LEFT_SOCKET = dai.CameraBoardSocket.CAM_B ALIGN_SOCKET = LEFT_SOCKET COLOR_RESOLUTION = dai.ColorCameraProperties.SensorResolution.THE_1080_P LEFT_RIGHT_RESOLUTION = dai.MonoCameraProperties.SensorResolution.THE_720_P ISP_SCALE = 3 device = dai.Device() calibrationHandler = device.readCalibration() rgbDistortion = calibrationHandler.getDistortionCoefficients(RGB_SOCKET) distortionModel = calibrationHandler.getDistortionModel(RGB_SOCKET) if distortionModel != dai.CameraModel.Perspective: raise RuntimeError("Unsupported distortion model for RGB camera. This example supports only Perspective model.") pipeline = dai.Pipeline() # Define sources and outputs camRgb = pipeline.create(dai.node.ColorCamera) left = pipeline.create(dai.node.MonoCamera) sync = pipeline.create(dai.node.Sync) out = pipeline.create(dai.node.XLinkOut) align = pipeline.create(dai.node.ImageAlign) cfgIn = pipeline.create(dai.node.XLinkIn) left.setResolution(LEFT_RIGHT_RESOLUTION) left.setBoardSocket(LEFT_SOCKET) left.setFps(FPS) camRgb.setBoardSocket(RGB_SOCKET) camRgb.setResolution(COLOR_RESOLUTION) camRgb.setFps(FPS) camRgb.setIspScale(1, ISP_SCALE) out.setStreamName("out") sync.setSyncThreshold(timedelta(seconds=0.5 / FPS)) cfgIn.setStreamName("config") cfg = align.initialConfig.get() staticDepthPlane = cfg.staticDepthPlane # Linking align.outputAligned.link(sync.inputs["aligned"]) camRgb.isp.link(sync.inputs["rgb"]) camRgb.isp.link(align.inputAlignTo) left.out.link(align.input) sync.out.link(out.input) cfgIn.out.link(align.inputConfig) rgbWeight = 0.4 leftWeight = 0.6 def updateBlendWeights(percentRgb): """ Update the rgb and left weights used to blend rgb/left image @param[in] percent_rgb The rgb weight expressed as a percentage (0..100) """ global leftWeight global rgbWeight rgbWeight = float(percentRgb) / 100.0 leftWeight = 1.0 - rgbWeight def updateDepthPlane(depth): global staticDepthPlane staticDepthPlane = depth # Connect to device and start pipeline with device: device.startPipeline(pipeline) queue = device.getOutputQueue("out", 8, False) cfgQ = device.getInputQueue("config") # Configure windows; trackbar adjusts blending ratio of rgb/depth windowName = "rgb-left" # Set the window to be resizable and the initial size cv2.namedWindow(windowName, cv2.WINDOW_NORMAL) cv2.resizeWindow(windowName, 1280, 720) cv2.createTrackbar( "RGB Weight %", windowName, int(rgbWeight * 100), 100, updateBlendWeights, ) cv2.createTrackbar( "Static Depth Plane [mm]", windowName, 0, 2000, updateDepthPlane, ) while True: messageGroup = queue.get() assert isinstance(messageGroup, dai.MessageGroup) frameRgb = messageGroup["rgb"] assert isinstance(frameRgb, dai.ImgFrame) leftAligned = messageGroup["aligned"] assert isinstance(leftAligned, dai.ImgFrame) frameRgbCv = frameRgb.getCvFrame() # Colorize the aligned depth leftCv = leftAligned.getCvFrame() rgbIntrinsics = calibrationHandler.getCameraIntrinsics(RGB_SOCKET, int(frameRgbCv.shape[1]), int(frameRgbCv.shape[0])) cvFrameUndistorted = cv2.undistort( frameRgbCv, np.array(rgbIntrinsics), np.array(rgbDistortion), ) if len(leftCv.shape) == 2: leftCv = cv2.cvtColor(leftCv, cv2.COLOR_GRAY2BGR) if leftCv.shape != cvFrameUndistorted.shape: leftCv = cv2.resize(leftCv, (cvFrameUndistorted.shape[1], cvFrameUndistorted.shape[0])) blended = cv2.addWeighted(cvFrameUndistorted, rgbWeight, leftCv, leftWeight, 0) cv2.imshow(windowName, blended) key = cv2.waitKey(1) if key == ord("q"): break cfg.staticDepthPlane = staticDepthPlane cfgQ.send(cfg) ``` #### C++ ```cpp #include #include #include #include "depthai/depthai.hpp" constexpr auto FPS = 30.0; constexpr auto RGB_SOCKET = dai::CameraBoardSocket::CAM_A; constexpr auto LEFT_SOCKET = dai::CameraBoardSocket::CAM_B; constexpr auto ALIGN_SOCKET = LEFT_SOCKET; constexpr auto COLOR_RESOLUTION = dai::ColorCameraProperties::SensorResolution::THE_1080_P; constexpr auto LEFT_RIGHT_RESOLUTION = dai::MonoCameraProperties::SensorResolution::THE_720_P; class FPSCounter { public: void tick() { auto now = std::chrono::steady_clock::now(); frameTimes.push(now); if(frameTimes.size() > 100) { frameTimes.pop(); } } double getFps() { if(frameTimes.size() <= 1) return 0; auto duration = std::chrono::duration_cast(frameTimes.back() - frameTimes.front()).count(); return (frameTimes.size() - 1) * 1000.0 / duration; } private: std::queue frameTimes; }; double rgbWeight = 0.4; double leftWeight = 0.6; int staticDepthPlane = 0; void updateBlendWeights(int percentRgb, void*) { rgbWeight = static_cast(percentRgb) / 100.0; leftWeight = 1.0 - rgbWeight; } void updateDepthPlane(int depth, void*) { staticDepthPlane = depth; } int main() { dai::Pipeline pipeline; auto camRgb = pipeline.create(); auto left = pipeline.create(); auto sync = pipeline.create(); auto out = pipeline.create(); auto align = pipeline.create(); auto cfgIn = pipeline.create(); left->setResolution(LEFT_RIGHT_RESOLUTION); left->setBoardSocket(LEFT_SOCKET); left->setFps(FPS); camRgb->setBoardSocket(RGB_SOCKET); camRgb->setResolution(COLOR_RESOLUTION); camRgb->setFps(FPS); camRgb->setIspScale(1, 3); out->setStreamName("out"); sync->setSyncThreshold(std::chrono::milliseconds(static_cast((1 / FPS) * 1000.0 * 0.5))); cfgIn->setStreamName("config"); align->outputAligned.link(sync->inputs["aligned"]); camRgb->isp.link(sync->inputs["rgb"]); camRgb->isp.link(align->inputAlignTo); left->out.link(align->input); sync->out.link(out->input); cfgIn->out.link(align->inputConfig); dai::Device device(pipeline); auto queue = device.getOutputQueue("out", 8, false); auto cfgQ = device.getInputQueue("config"); FPSCounter fpsCounter; std::string windowName = "rgb-left"; cv::namedWindow(windowName, cv::WINDOW_NORMAL); cv::resizeWindow(windowName, 1280, 720); cv::createTrackbar("RGB Weight %", windowName, nullptr, 100, updateBlendWeights); cv::createTrackbar("Static Depth Plane [mm]", windowName, nullptr, 2000, updateDepthPlane); while(true) { auto messageGroup = queue->get(); fpsCounter.tick(); auto frameRgb = messageGroup->get("rgb"); auto leftAligned = messageGroup->get("aligned"); if(frameRgb && leftAligned) { auto frameRgbCv = frameRgb->getCvFrame(); auto leftCv = leftAligned->getCvFrame(); if(leftCv.channels() == 1) { cv::cvtColor(leftCv, leftCv, cv::COLOR_GRAY2BGR); } if(leftCv.size() != frameRgbCv.size()) { cv::resize(leftCv, leftCv, frameRgbCv.size()); } cv::Mat blended; cv::addWeighted(frameRgbCv, rgbWeight, leftCv, leftWeight, 0, blended); cv::imshow(windowName, blended); } int key = cv::waitKey(1); if(key == 'q' || key == 'Q') { break; } auto cfg = align->initialConfig.get(); cfg.staticDepthPlane = staticDepthPlane; auto alignConfig = std::make_shared(); alignConfig->set(cfg); cfgQ->send(alignConfig); } return 0; } ``` ## Pipeline ### examples/image_align.pipeline.json ```json { "pipeline": { "connections": [ { "node1Id": 4, "node1Output": "outputAligned", "node1OutputGroup": "", "node2Id": 2, "node2Input": "aligned", "node2InputGroup": "inputs" }, { "node1Id": 0, "node1Output": "isp", "node1OutputGroup": "", "node2Id": 2, "node2Input": "rgb", "node2InputGroup": "inputs" }, { "node1Id": 5, "node1Output": "out", "node1OutputGroup": "", "node2Id": 4, "node2Input": "inputConfig", "node2InputGroup": "" }, { "node1Id": 0, "node1Output": "isp", "node1OutputGroup": "", "node2Id": 4, "node2Input": "inputAlignTo", "node2InputGroup": "" }, { "node1Id": 1, "node1Output": "out", "node1OutputGroup": "", "node2Id": 4, "node2Input": "input", "node2InputGroup": "" }, { "node1Id": 2, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "in", "node2InputGroup": "" } ], "globalProperties": { "calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1 }, "nodes": [ [ 0, { "id": 0, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 1, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 6, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "still" ], { "blocking": false, "group": "", "id": 7, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 2, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "video" ], { "blocking": false, "group": "", "id": 3, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "isp" ], { "blocking": false, "group": "", "id": 4, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "preview" ], { "blocking": false, "group": "", "id": 5, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 8, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ColorCamera", "properties": { "boardSocket": 0, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 30.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": true, "isp3aFps": 0, "ispScale": { "horizDenominator": 3, "horizNumerator": 1, "vertDenominator": 3, "vertNumerator": 1 }, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 300, "previewKeepAspectRatio": true, "previewWidth": 300, "rawPacked": null, "resolution": 0, "sensorCropX": -1.0, "sensorCropY": -1.0, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1 } } ], [ 1, { "id": 1, "ioInfo": [ [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 9, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 10, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 11, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 12, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "MonoCamera", "properties": { "boardSocket": 1, "cameraName": "", "fps": 30.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "isp3aFps": 0, "numFramesPool": 3, "numFramesPoolRaw": 3, "rawPacked": null, "resolution": 0 } } ], [ 2, { "id": 2, "ioInfo": [ [ [ "inputs", "aligned" ], { "blocking": true, "group": "inputs", "id": 13, "name": "aligned", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "inputs", "rgb" ], { "blocking": true, "group": "inputs", "id": 14, "name": "rgb", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 15, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "Sync", "properties": { "syncAttempts": -1, "syncThresholdNs": 16667000 } } ], [ 3, { "id": 3, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 16, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "out" } } ], [ 4, { "id": 4, "ioInfo": [ [ [ "", "inputAlignTo" ], { "blocking": false, "group": "", "id": 17, "name": "inputAlignTo", "queueSize": 1, "type": 3, "waitForMessage": true } ], [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 18, "name": "inputConfig", "queueSize": 4, "type": 3, "waitForMessage": false } ], [ [ "", "input" ], { "blocking": false, "group": "", "id": 19, "name": "input", "queueSize": 4, "type": 3, "waitForMessage": true } ], [ [ "", "outputAligned" ], { "blocking": false, "group": "", "id": 20, "name": "outputAligned", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "passthroughInput" ], { "blocking": false, "group": "", "id": 21, "name": "passthroughInput", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ImageAlign", "properties": { "alignHeight": 0, "alignWidth": 0, "initialConfig": { "staticDepthPlane": 0 }, "interpolation": -1, "numFramesPool": 4, "numShaves": 2, "outKeepAspectRatio": true, "warpHwIds": [] } } ], [ 5, { "id": 5, "ioInfo": [ [ [ "", "out" ], { "blocking": false, "group": "", "id": 22, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "XLinkIn", "properties": { "maxDataSize": 5242880, "numFrames": 8, "streamName": "config" } } ] ] } } ``` ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.