# Depth Align This example shows how to align the stereo depth map to the RGB frame to get RGB-D frames. ## Demo This example requires the DepthAI v3 API, see [installation instructions](https://docs.luxonis.com/software-v3/depthai.md). ## Pipeline ### examples/stereo_depth_align.pipeline.json ```json { "pipeline": { "connections": [ { "node1Id": 5, "node1Output": "outputAligned", "node1OutputGroup": "", "node2Id": 4, "node2Input": "depth_aligned", "node2InputGroup": "inputs" }, { "node1Id": 3, "node1Output": "depth", "node1OutputGroup": "", "node2Id": 5, "node2Input": "input", "node2InputGroup": "" }, { "node1Id": 2, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 3, "node2Input": "right", "node2InputGroup": "" }, { "node1Id": 1, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 3, "node2Input": "left", "node2InputGroup": "" }, { "node1Id": 0, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 5, "node2Input": "inputAlignTo", "node2InputGroup": "" }, { "node1Id": 0, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 4, "node2Input": "rgb", "node2InputGroup": "inputs" } ], "globalProperties": { "calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1 }, "nodes": [ [ 5, { "alias": "", "id": 5, "ioInfo": [ [ [ "", "passthroughInput" ], { "blocking": false, "group": "", "id": 34, "name": "passthroughInput", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 30, "name": "inputConfig", "queueSize": 4, "type": 3, "waitForMessage": false } ], [ [ "", "input" ], { "blocking": false, "group": "", "id": 31, "name": "input", "queueSize": 4, "type": 3, "waitForMessage": false } ], [ [ "", "outputAligned" ], { "blocking": false, "group": "", "id": 33, "name": "outputAligned", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputAlignTo" ], { "blocking": false, "group": "", "id": 32, "name": "inputAlignTo", "queueSize": 1, "type": 3, "waitForMessage": true } ] ], "name": "ImageAlign", "parentId": -1, "properties": { "alignHeight": 0, "alignWidth": 0, "initialConfig": { "staticDepthPlane": 0 }, "interpolation": -1, "numFramesPool": 4, "numShaves": 2, "outKeepAspectRatio": true, "warpHwIds": [] } } ], [ 4, { "alias": "", "id": 4, "ioInfo": [ [ [ "inputs", "rgb" ], { "blocking": false, "group": "inputs", "id": 28, "name": "rgb", "queueSize": 10, "type": 3, "waitForMessage": false } ], [ [ "inputs", "depth_aligned" ], { "blocking": false, "group": "inputs", "id": 27, "name": "depth_aligned", "queueSize": 10, "type": 3, "waitForMessage": false } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 29, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "Sync", "parentId": -1, "properties": { "syncAttempts": -1, "syncThresholdNs": 16667000 } } ], [ 3, { "alias": "", "id": 3, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": true, "group": "", "id": 6, "name": "inputConfig", "queueSize": 3, "type": 3, "waitForMessage": false } ], [ [ "", "inputAlignTo" ], { "blocking": false, "group": "", "id": 7, "name": "inputAlignTo", "queueSize": 1, "type": 3, "waitForMessage": true } ], [ [ "", "debugExtDispLrCheckIt2" ], { "blocking": false, "group": "", "id": 22, "name": "debugExtDispLrCheckIt2", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "outConfig" ], { "blocking": false, "group": "", "id": 18, "name": "outConfig", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "left" ], { "blocking": true, "group": "", "id": 8, "name": "left", "queueSize": 3, "type": 3, "waitForMessage": false } ], [ [ "", "right" ], { "blocking": true, "group": "", "id": 9, "name": "right", "queueSize": 3, "type": 3, "waitForMessage": false } ], [ [ "", "pixelDescriptorsLeft" ], { "blocking": false, "group": "", "id": 25, "name": "pixelDescriptorsLeft", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "depth" ], { "blocking": false, "group": "", "id": 12, "name": "depth", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputRightPixelDescriptor" ], { "blocking": true, "group": "", "id": 11, "name": "inputRightPixelDescriptor", "queueSize": 3, "type": 3, "waitForMessage": false } ], [ [ "", "inputLeftPixelDescriptor" ], { "blocking": true, "group": "", "id": 10, "name": "inputLeftPixelDescriptor", "queueSize": 3, "type": 3, "waitForMessage": false } ], [ [ "", "pixelDescriptorsRight" ], { "blocking": false, "group": "", "id": 26, "name": "pixelDescriptorsRight", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "disparity" ], { "blocking": false, "group": "", "id": 13, "name": "disparity", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "syncedLeft" ], { "blocking": false, "group": "", "id": 14, "name": "syncedLeft", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "syncedRight" ], { "blocking": false, "group": "", "id": 15, "name": "syncedRight", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "confidenceMap" ], { "blocking": false, "group": "", "id": 24, "name": "confidenceMap", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "debugDispCostDump" ], { "blocking": false, "group": "", "id": 23, "name": "debugDispCostDump", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "rectifiedLeft" ], { "blocking": false, "group": "", "id": 16, "name": "rectifiedLeft", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "debugDispLrCheckIt2" ], { "blocking": false, "group": "", "id": 20, "name": "debugDispLrCheckIt2", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "rectifiedRight" ], { "blocking": false, "group": "", "id": 17, "name": "rectifiedRight", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "debugDispLrCheckIt1" ], { "blocking": false, "group": "", "id": 19, "name": "debugDispLrCheckIt1", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "debugExtDispLrCheckIt1" ], { "blocking": false, "group": "", "id": 21, "name": "debugExtDispLrCheckIt1", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "StereoDepth", "parentId": -1, "properties": { "alphaScaling": null, "baseline": null, "customPixelDescriptors": false, "depthAlignCamera": -1, "depthAlignmentUseSpecTranslation": null, "disparityToDepthUseSpecTranslation": null, "enableFrameSync": true, "enableRectification": true, "enableRuntimeStereoModeSwitch": false, "focalLength": null, "focalLengthFromCalibration": true, "height": null, "initialConfig": { "algorithmControl": { "centerAlignmentShiftFactor": null, "customDepthUnitMultiplier": 1000.0, "depthAlign": 1, "depthUnit": 2, "disparityShift": 0, "enableExtended": true, "enableLeftRightCheck": true, "enableSubpixel": true, "leftRightCheckThreshold": 10, "numInvalidateEdgePixels": 0, "subpixelFractionalBits": 5 }, "censusTransform": { "enableMeanMode": true, "kernelMask": 0, "kernelSize": -1, "noiseThresholdOffset": 1, "noiseThresholdScale": 1, "threshold": 0 }, "confidenceMetrics": { "flatnessConfidenceThreshold": 2, "flatnessConfidenceWeight": 8, "flatnessOverride": false, "motionVectorConfidenceThreshold": 1, "motionVectorConfidenceWeight": 4, "occlusionConfidenceWeight": 20 }, "costAggregation": { "divisionFactor": 1, "horizontalPenaltyCostP1": 250, "horizontalPenaltyCostP2": 500, "p1Config": { "defaultValue": 11, "edgeThreshold": 15, "edgeValue": 10, "enableAdaptive": true, "smoothThreshold": 5, "smoothValue": 22 }, "p2Config": { "defaultValue": 33, "edgeValue": 22, "enableAdaptive": true, "smoothValue": 63 }, "verticalPenaltyCostP1": 250, "verticalPenaltyCostP2": 500 }, "costMatching": { "confidenceThreshold": 55, "disparityWidth": 1, "enableCompanding": false, "invalidDisparityValue": 0, "linearEquationParameters": { "alpha": 0, "beta": 2, "threshold": 127 } }, "postProcessing": { "adaptiveMedianFilter": { "confidenceThreshold": 200, "enable": true }, "bilateralSigmaValue": 0, "brightnessFilter": { "maxBrightness": 256, "minBrightness": 0 }, "decimationFilter": { "decimationFactor": 1, "decimationMode": 0 }, "filteringOrder": [ 3, 1, 2, 4, 5 ], "holeFilling": { "enable": true, "fillConfidenceThreshold": 200, "highConfidenceThreshold": 210, "invalidateDisparities": true, "minValidDisparity": 1 }, "median": 0, "spatialFilter": { "alpha": 0.5, "delta": 0, "enable": false, "holeFillingRadius": 2, "numIterations": 1 }, "speckleFilter": { "differenceThreshold": 2, "enable": false, "speckleRange": 50 }, "temporalFilter": { "alpha": 0.4000000059604645, "delta": 0, "enable": false, "persistencyMode": 3 }, "thresholdFilter": { "maxRange": 65535, "minRange": 0 } } }, "mesh": { "meshLeftUri": "", "meshRightUri": "", "meshSize": null, "stepHeight": 16, "stepWidth": 16 }, "numFramesPool": 3, "numPostProcessingMemorySlices": -1, "numPostProcessingShaves": -1, "outHeight": null, "outKeepAspectRatio": true, "outWidth": null, "rectificationUseSpecTranslation": null, "rectifyEdgeFillColor": 0, "useHomographyRectification": null, "verticalStereo": false, "width": null } } ], [ 2, { "alias": "", "id": 2, "ioInfo": [ [ [ "dynamicOutputs", "0" ], { "blocking": false, "group": "dynamicOutputs", "id": 5, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 4, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false } ] ], "name": "Camera", "parentId": -1, "properties": { "boardSocket": 2, "calibAlpha": null, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 30.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "miscControls": [], "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": true, "isp3aFps": 0, "ispScale": { "horizDenominator": 0, "horizNumerator": 0, "vertDenominator": 0, "vertNumerator": 0 }, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [ { "enableUndistortion": null, "fps": { "value": { "index": 0, "value": 30.0 } }, "resizeMode": 0, "size": { "value": { "index": 0, "value": [ 640, 400 ] } }, "type": null } ], "previewHeight": 300, "previewKeepAspectRatio": false, "previewWidth": 300, "rawPacked": null, "resolutionHeight": -1, "resolutionWidth": -1, "sensorCropX": -1.0, "sensorCropY": -1.0, "sensorType": -1, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1, "warpMeshHeight": 0, "warpMeshSource": -1, "warpMeshStepHeight": 32, "warpMeshStepWidth": 32, "warpMeshUri": "", "warpMeshWidth": 0 } } ], [ 1, { "alias": "", "id": 1, "ioInfo": [ [ [ "dynamicOutputs", "0" ], { "blocking": false, "group": "dynamicOutputs", "id": 3, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 2, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false } ] ], "name": "Camera", "parentId": -1, "properties": { "boardSocket": 1, "calibAlpha": null, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 30.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "miscControls": [], "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": true, "isp3aFps": 0, "ispScale": { "horizDenominator": 0, "horizNumerator": 0, "vertDenominator": 0, "vertNumerator": 0 }, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [ { "enableUndistortion": null, "fps": { "value": { "index": 0, "value": 30.0 } }, "resizeMode": 0, "size": { "value": { "index": 0, "value": [ 640, 400 ] } }, "type": null } ], "previewHeight": 300, "previewKeepAspectRatio": false, "previewWidth": 300, "rawPacked": null, "resolutionHeight": -1, "resolutionWidth": -1, "sensorCropX": -1.0, "sensorCropY": -1.0, "sensorType": -1, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1, "warpMeshHeight": 0, "warpMeshSource": -1, "warpMeshStepHeight": 32, "warpMeshStepWidth": 32, "warpMeshUri": "", "warpMeshWidth": 0 } } ], [ 0, { "alias": "", "id": 0, "ioInfo": [ [ [ "dynamicOutputs", "0" ], { "blocking": false, "group": "dynamicOutputs", "id": 1, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 0, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false } ] ], "name": "Camera", "parentId": -1, "properties": { "boardSocket": 0, "calibAlpha": null, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 30.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 1764586528, "aeRegion": { "height": 0, "priority": 1764580784, "width": 1, "x": 35840, "y": 26864 }, "afRegion": { "height": 2331, "priority": 1, "width": 5168, "x": 1, "y": 0 }, "antiBandingMode": 70, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 240, "brightness": 0, "captureIntent": 2, "chromaDenoise": 1, "cmdMask": 0, "contrast": -80, "controlMode": 1, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 1, "frameDurationUs": 1, "sensitivityIso": 37211424 }, "frameSyncMode": 0, "lensPosAutoInfinity": 208, "lensPosAutoMacro": 165, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 80, "lowPowerNumFramesDiscard": 164, "lumaDenoise": 2, "miscControls": [], "saturation": -128, "sceneMode": 163, "sharpness": 54, "strobeConfig": { "activeLevel": 48, "enable": 0, "gpioNumber": 20 }, "strobeTimings": { "durationUs": 1, "exposureBeginOffsetUs": 1, "exposureEndOffsetUs": 152769584 }, "wbColorTemp": 0 }, "interleaved": true, "isp3aFps": 0, "ispScale": { "horizDenominator": 0, "horizNumerator": 0, "vertDenominator": 0, "vertNumerator": 0 }, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [ { "enableUndistortion": null, "fps": { "value": { "index": 0, "value": 30.0 } }, "resizeMode": 0, "size": { "value": { "index": 0, "value": [ 1280, 960 ] } }, "type": null } ], "previewHeight": 300, "previewKeepAspectRatio": false, "previewWidth": 300, "rawPacked": null, "resolutionHeight": -1, "resolutionWidth": -1, "sensorCropX": -1.0, "sensorCropY": -1.0, "sensorType": -1, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1, "warpMeshHeight": 0, "warpMeshSource": -1, "warpMeshStepHeight": 32, "warpMeshStepWidth": 32, "warpMeshUri": "", "warpMeshWidth": 0 } } ] ] } } ``` ## Source code #### Python ```python #!/usr/bin/env python3 import numpy as np import cv2 import depthai as dai import time from datetime import timedelta FPS = 25.0 RGB_SOCKET = dai.CameraBoardSocket.CAM_A LEFT_SOCKET = dai.CameraBoardSocket.CAM_B RIGHT_SOCKET = dai.CameraBoardSocket.CAM_C class FPSCounter: def __init__(self): self.frameTimes = [] def tick(self): now = time.time() self.frameTimes.append(now) self.frameTimes = self.frameTimes[-10:] def getFps(self): if len(self.frameTimes) <= 1: return 0 return (len(self.frameTimes) - 1) / (self.frameTimes[-1] - self.frameTimes[0]) pipeline = dai.Pipeline() platform = pipeline.getDefaultDevice().getPlatform() # Define sources and outputs camRgb = pipeline.create(dai.node.Camera).build(RGB_SOCKET) left = pipeline.create(dai.node.Camera).build(LEFT_SOCKET) right = pipeline.create(dai.node.Camera).build(RIGHT_SOCKET) stereo = pipeline.create(dai.node.StereoDepth) sync = pipeline.create(dai.node.Sync) if platform == dai.Platform.RVC4: align = pipeline.create(dai.node.ImageAlign) stereo.setExtendedDisparity(True) sync.setSyncThreshold(timedelta(seconds=1/(2*FPS))) rgbOut = camRgb.requestOutput(size = (1280, 960), fps = FPS, enableUndistortion=True) leftOut = left.requestOutput(size = (640, 400), fps = FPS) rightOut = right.requestOutput(size = (640, 400), fps = FPS) # Linking rgbOut.link(sync.inputs["rgb"]) leftOut.link(stereo.left) rightOut.link(stereo.right) if platform == dai.Platform.RVC4: stereo.depth.link(align.input) rgbOut.link(align.inputAlignTo) align.outputAligned.link(sync.inputs["depth_aligned"]) else: stereo.depth.link(sync.inputs["depth_aligned"]) rgbOut.link(stereo.inputAlignTo) queue = sync.out.createOutputQueue() def colorizeDepth(frameDepth): invalidMask = frameDepth == 0 # Log the depth, minDepth and maxDepth try: minDepth = np.percentile(frameDepth[frameDepth != 0], 3) maxDepth = np.percentile(frameDepth[frameDepth != 0], 95) logDepth = np.zeros_like(frameDepth, dtype=np.float32) np.log(frameDepth, where=frameDepth != 0, out=logDepth) logMinDepth = np.log(minDepth) logMaxDepth = np.log(maxDepth) np.nan_to_num(logDepth, copy=False, nan=logMinDepth) # Clip the values to be in the 0-255 range logDepth = np.clip(logDepth, logMinDepth, logMaxDepth) # Interpolate only valid logDepth values, setting the rest based on the mask depthFrameColor = np.interp(logDepth, (logMinDepth, logMaxDepth), (0, 255)) depthFrameColor = np.nan_to_num(depthFrameColor) depthFrameColor = depthFrameColor.astype(np.uint8) depthFrameColor = cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_JET) # Set invalid depth pixels to black depthFrameColor[invalidMask] = 0 except IndexError: # Frame is likely empty depthFrameColor = np.zeros((frameDepth.shape[0], frameDepth.shape[1], 3), dtype=np.uint8) except Exception as e: raise e return depthFrameColor rgbWeight = 0.4 depthWeight = 0.6 def updateBlendWeights(percentRgb): """ Update the rgb and depth weights used to blend depth/rgb image @param[in] percent_rgb The rgb weight expressed as a percentage (0..100) """ global depthWeight global rgbWeight rgbWeight = float(percentRgb) / 100.0 depthWeight = 1.0 - rgbWeight # Connect to device and start pipeline with pipeline: pipeline.start() # Configure windows; trackbar adjusts blending ratio of rgb/depth windowName = "rgb-depth" # Set the window to be resizable and the initial size cv2.namedWindow(windowName, cv2.WINDOW_NORMAL) cv2.resizeWindow(windowName, 1280, 720) cv2.createTrackbar( "RGB Weight %", windowName, int(rgbWeight * 100), 100, updateBlendWeights, ) fpsCounter = FPSCounter() while True: messageGroup = queue.get() fpsCounter.tick() assert isinstance(messageGroup, dai.MessageGroup) frameRgb = messageGroup["rgb"] assert isinstance(frameRgb, dai.ImgFrame) frameDepth = messageGroup["depth_aligned"] assert isinstance(frameDepth, dai.ImgFrame) # Blend when both received if frameDepth is not None: cvFrame = frameRgb.getCvFrame() # Colorize the aligned depth alignedDepthColorized = colorizeDepth(frameDepth.getFrame()) # Resize depth to match the rgb frame cv2.imshow("Depth aligned", alignedDepthColorized) if len(cvFrame.shape) == 2: cvFrameUndistorted = cv2.cvtColor(cvFrame, cv2.COLOR_GRAY2BGR) blended = cv2.addWeighted( cvFrame, rgbWeight, alignedDepthColorized, depthWeight, 0 ) cv2.putText( blended, f"FPS: {fpsCounter.getFps():.2f}", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, ) cv2.imshow(windowName, blended) key = cv2.waitKey(1) if key == ord("q"): break ``` #### C++ ```cpp #include #include #include #include #include #include #include #include #include #include #include "depthai/depthai.hpp" std::atomic quitEvent(false); void signalHandler(int) { quitEvent = true; } constexpr float FPS = 25.0f; const dai::CameraBoardSocket RGB_SOCKET = dai::CameraBoardSocket::CAM_A; const dai::CameraBoardSocket LEFT_SOCKET = dai::CameraBoardSocket::CAM_B; const dai::CameraBoardSocket RIGHT_SOCKET = dai::CameraBoardSocket::CAM_C; // FPS Counter class class FPSCounter { public: void tick() { auto now = std::chrono::steady_clock::now(); frameTimes.push_back(now); while(frameTimes.size() > 10) { frameTimes.pop_front(); } } float getFps() { if(frameTimes.size() <= 1) return 0.0f; auto duration = std::chrono::duration_cast>(frameTimes.back() - frameTimes.front()).count(); return (frameTimes.size() - 1) / duration; } private: std::deque frameTimes; }; // Depth colorization function from detection_network_remap.cpp cv::Mat colorizeDepth(cv::Mat frameDepth) { try { // Early exit if no valid pixels if(cv::countNonZero(frameDepth) == 0) { return cv::Mat::zeros(frameDepth.rows, frameDepth.cols, CV_8UC3); } // Convert to float once cv::Mat frameDepthFloat; frameDepth.convertTo(frameDepthFloat, CV_32F); double minVal, maxVal; cv::minMaxLoc(frameDepthFloat, &minVal, &maxVal, nullptr, nullptr, frameDepthFloat > 0); // Take log in-place cv::log(frameDepthFloat, frameDepthFloat); float logMinDepth = std::log(minVal); float logMaxDepth = std::log(maxVal); frameDepthFloat = (frameDepthFloat - logMinDepth) * (255.0f / (logMaxDepth - logMinDepth)); cv::Mat normalizedDepth; frameDepthFloat.convertTo(normalizedDepth, CV_8UC1); cv::Mat depthFrameColor; cv::applyColorMap(normalizedDepth, depthFrameColor, cv::COLORMAP_JET); // Mask invalid pixels depthFrameColor.setTo(0, frameDepth == 0); return depthFrameColor; } catch(const std::exception& e) { std::cerr << "Error in colorizeDepth: " << e.what() << std::endl; return cv::Mat::zeros(frameDepth.rows, frameDepth.cols, CV_8UC3); } } // Global blend weights float rgbWeight = 0.4f; float depthWeight = 0.6f; // Trackbar callback void updateBlendWeights(int percentRgb, void*) { rgbWeight = static_cast(percentRgb) / 100.0f; depthWeight = 1.0f - rgbWeight; } int main() { signal(SIGTERM, signalHandler); signal(SIGINT, signalHandler); dai::Pipeline pipeline; // Create and configure nodes auto camRgb = pipeline.create(); camRgb->build(RGB_SOCKET); auto left = pipeline.create(); left->build(LEFT_SOCKET); auto right = pipeline.create(); right->build(RIGHT_SOCKET); auto stereo = pipeline.create(); auto sync = pipeline.create(); // Check if platform is RVC4 and create ImageAlign node if needed auto platform = pipeline.getDefaultDevice()->getPlatform(); std::shared_ptr align; if(platform == dai::Platform::RVC4) { align = pipeline.create(); } stereo->setExtendedDisparity(true); sync->setSyncThreshold(std::chrono::duration(static_cast(1e9 / (2.0 * FPS)))); // Configure outputs auto rgbOut = camRgb->requestOutput(std::make_pair(1280, 960), dai::ImgFrame::Type::NV12, dai::ImgResizeMode::CROP, FPS, true); auto leftOut = left->requestOutput(std::make_pair(640, 400), std::nullopt, dai::ImgResizeMode::CROP, FPS); auto rightOut = right->requestOutput(std::make_pair(640, 400), std::nullopt, dai::ImgResizeMode::CROP, FPS); // Link nodes rgbOut->link(sync->inputs["rgb"]); leftOut->link(stereo->left); rightOut->link(stereo->right); if(platform == dai::Platform::RVC4) { stereo->depth.link(align->input); rgbOut->link(align->inputAlignTo); align->outputAligned.link(sync->inputs["depth_aligned"]); } else { stereo->depth.link(sync->inputs["depth_aligned"]); rgbOut->link(stereo->inputAlignTo); } // Create output queue auto queue = sync->out.createOutputQueue(); // Create and configure windows const std::string windowName = "rgb-depth"; cv::namedWindow(windowName, cv::WINDOW_NORMAL); cv::resizeWindow(windowName, 1280, 720); cv::createTrackbar("RGB Weight %", windowName, nullptr, 100, updateBlendWeights); cv::setTrackbarPos("RGB Weight %", windowName, static_cast(rgbWeight * 100)); FPSCounter fpsCounter; // Start pipeline pipeline.start(); while(pipeline.isRunning() && !quitEvent) { auto messageGroup = queue->get(); fpsCounter.tick(); auto frameRgb = messageGroup->get("rgb"); auto frameDepth = messageGroup->get("depth_aligned"); if(frameDepth != nullptr) { cv::Mat cvFrame = frameRgb->getCvFrame(); // Colorize depth cv::Mat alignedDepthColorized = colorizeDepth(frameDepth->getFrame()); cv::imshow("Depth aligned", alignedDepthColorized); // Convert grayscale to BGR if needed if(cvFrame.channels() == 1) { cv::cvtColor(cvFrame, cvFrame, cv::COLOR_GRAY2BGR); } // Blend frames cv::Mat blended; cv::addWeighted(cvFrame, rgbWeight, alignedDepthColorized, depthWeight, 0, blended); // Add FPS text cv::putText(blended, "FPS: " + std::to_string(fpsCounter.getFps()), cv::Point(10, 30), cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(255, 255, 255), 2); cv::imshow(windowName, blended); } if(cv::waitKey(1) == 'q') { break; } } pipeline.stop(); pipeline.wait(); return 0; } ``` ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.