# Stereo Depth Filters This example showcases how to use various filters available in the [StereoDepth node](https://docs.luxonis.com/software-v3/depthai/depthai-components/nodes/stereo_depth.md) to improve the quality of the depth map. ## Demo This example requires the DepthAI v3 API, see [installation instructions](https://docs.luxonis.com/software-v3/depthai.md). ## Pipeline ### examples/stereo_depth_filters.pipeline.json ```json { "pipeline": { "connections": [ { "node1Id": 0, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 2, "node2Input": "left", "node2InputGroup": "" }, { "node1Id": 1, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 2, "node2Input": "right", "node2InputGroup": "" } ], "globalProperties": { "calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "eepromId": 0, "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1 }, "nodes": [ [ 0, { "alias": "", "id": 0, "ioInfo": [ [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 0, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false } ], [ [ "dynamicOutputs", "0" ], { "blocking": false, "group": "dynamicOutputs", "id": 3, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "mockIsp" ], { "blocking": true, "group": "", "id": 1, "name": "mockIsp", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 2, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "logLevel": 3, "name": "Camera", "parentId": -1, "properties": { "boardSocket": 1, "cameraName": "", "fps": -1.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "enableHdr": false, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "miscControls": [], "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "isp3aFps": 0, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [ { "enableUndistortion": null, "fps": { "value": { "index": 0, "value": 20.0 } }, "resizeMode": 0, "size": { "value": { "index": 0, "value": [ 640, 400 ] } }, "type": null } ], "resolutionHeight": -1, "resolutionWidth": -1, "sensorType": -1 } } ], [ 1, { "alias": "", "id": 1, "ioInfo": [ [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 4, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false } ], [ [ "dynamicOutputs", "0" ], { "blocking": false, "group": "dynamicOutputs", "id": 7, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "mockIsp" ], { "blocking": true, "group": "", "id": 5, "name": "mockIsp", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 6, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "logLevel": 3, "name": "Camera", "parentId": -1, "properties": { "boardSocket": 2, "cameraName": "", "fps": -1.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "enableHdr": false, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "miscControls": [], "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "isp3aFps": 0, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [ { "enableUndistortion": null, "fps": { "value": { "index": 0, "value": 20.0 } }, "resizeMode": 0, "size": { "value": { "index": 0, "value": [ 640, 400 ] } }, "type": null } ], "resolutionHeight": -1, "resolutionWidth": -1, "sensorType": -1 } } ], [ 2, { "alias": "", "id": 2, "ioInfo": [ [ [ "", "inputAlignTo" ], { "blocking": false, "group": "", "id": 9, "name": "inputAlignTo", "queueSize": 1, "type": 3, "waitForMessage": true } ], [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 8, "name": "inputConfig", "queueSize": 3, "type": 3, "waitForMessage": false } ], [ [ "", "left" ], { "blocking": true, "group": "", "id": 10, "name": "left", "queueSize": 3, "type": 3, "waitForMessage": false } ], [ [ "", "debugExtDispLrCheckIt1" ], { "blocking": false, "group": "", "id": 21, "name": "debugExtDispLrCheckIt1", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "right" ], { "blocking": true, "group": "", "id": 11, "name": "right", "queueSize": 3, "type": 3, "waitForMessage": false } ], [ [ "", "depth" ], { "blocking": false, "group": "", "id": 12, "name": "depth", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "disparity" ], { "blocking": false, "group": "", "id": 13, "name": "disparity", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "syncedLeft" ], { "blocking": false, "group": "", "id": 14, "name": "syncedLeft", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "syncedRight" ], { "blocking": false, "group": "", "id": 15, "name": "syncedRight", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "rectifiedLeft" ], { "blocking": false, "group": "", "id": 16, "name": "rectifiedLeft", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "rectifiedRight" ], { "blocking": false, "group": "", "id": 17, "name": "rectifiedRight", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "confidenceMap" ], { "blocking": false, "group": "", "id": 24, "name": "confidenceMap", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "outConfig" ], { "blocking": false, "group": "", "id": 18, "name": "outConfig", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "debugDispLrCheckIt1" ], { "blocking": false, "group": "", "id": 19, "name": "debugDispLrCheckIt1", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "debugDispLrCheckIt2" ], { "blocking": false, "group": "", "id": 20, "name": "debugDispLrCheckIt2", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "debugDispCostDump" ], { "blocking": false, "group": "", "id": 23, "name": "debugDispCostDump", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "debugExtDispLrCheckIt2" ], { "blocking": false, "group": "", "id": 22, "name": "debugExtDispLrCheckIt2", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "logLevel": 3, "name": "StereoDepth", "parentId": -1, "properties": { "alphaScaling": null, "baseline": null, "depthAlignCamera": -1, "depthAlignmentUseSpecTranslation": null, "disparityToDepthUseSpecTranslation": null, "enableFrameSync": true, "enableRectification": true, "enableRuntimeStereoModeSwitch": false, "focalLength": null, "focalLengthFromCalibration": true, "height": null, "initialConfig": { "algorithmControl": { "centerAlignmentShiftFactor": null, "customDepthUnitMultiplier": 1000.0, "depthAlign": 1, "depthUnit": 2, "disparityShift": 0, "enableExtended": false, "enableLeftRightCheck": false, "enableSubpixel": false, "enableSwLeftRightCheck": false, "leftRightCheckThreshold": 10, "numInvalidateEdgePixels": 0, "subpixelFractionalBits": 5 }, "censusTransform": { "enableMeanMode": true, "kernelMask": 0, "kernelSize": -1, "noiseThresholdOffset": 1, "noiseThresholdScale": 1, "threshold": 0 }, "confidenceMetrics": { "flatnessConfidenceThreshold": 2, "flatnessConfidenceWeight": 8, "flatnessOverride": false, "motionVectorConfidenceThreshold": 1, "motionVectorConfidenceWeight": 4, "occlusionConfidenceWeight": 20 }, "costAggregation": { "divisionFactor": 1, "horizontalPenaltyCostP1": 250, "horizontalPenaltyCostP2": 500, "p1Config": { "defaultValue": 11, "edgeThreshold": 15, "edgeValue": 10, "enableAdaptive": true, "smoothThreshold": 5, "smoothValue": 22 }, "p2Config": { "defaultValue": 33, "edgeValue": 22, "enableAdaptive": true, "smoothValue": 63 }, "verticalPenaltyCostP1": 250, "verticalPenaltyCostP2": 500 }, "costMatching": { "confidenceThreshold": 55, "disparityWidth": 1, "enableCompanding": false, "enableSwConfidenceThresholding": false, "invalidDisparityValue": 0, "linearEquationParameters": { "alpha": 0, "beta": 2, "threshold": 127 } }, "filtersBackend": 2, "postProcessing": { "adaptiveMedianFilter": { "confidenceThreshold": 200, "enable": true }, "bilateralSigmaValue": 0, "brightnessFilter": { "maxBrightness": 256, "minBrightness": 0 }, "decimationFilter": { "decimationFactor": 1, "decimationMode": 0 }, "filteringOrder": [ 3, 1, 2, 4, 5 ], "holeFilling": { "enable": true, "fillConfidenceThreshold": 200, "highConfidenceThreshold": 210, "invalidateDisparities": true, "minValidDisparity": 1 }, "median": 0, "spatialFilter": { "alpha": 0.5, "delta": 3, "enable": false, "holeFillingRadius": 2, "numIterations": 1 }, "speckleFilter": { "differenceThreshold": 2, "enable": false, "speckleRange": 50 }, "temporalFilter": { "alpha": 0.4000000059604645, "delta": 3, "enable": false, "persistencyMode": 3 }, "thresholdFilter": { "maxRange": 65535, "minRange": 0 } } }, "mesh": { "meshLeftUri": "", "meshRightUri": "", "meshSize": null, "stepHeight": 16, "stepWidth": 16 }, "numFramesPool": 3, "numPostProcessingMemorySlices": -1, "numPostProcessingShaves": -1, "outHeight": null, "outKeepAspectRatio": true, "outWidth": null, "rectificationUseSpecTranslation": null, "rectifyEdgeFillColor": 0, "useHomographyRectification": null, "width": null } } ] ] } } ``` ## Source code #### Python ```python #!/usr/bin/env python3 import argparse import random import cv2 import depthai as dai import numpy as np FPS = 20 def getRandomMedianFilterParams(): return random.choice( [ dai.node.ImageFilters.MedianFilterParams.MEDIAN_OFF, dai.node.ImageFilters.MedianFilterParams.KERNEL_3x3, dai.node.ImageFilters.MedianFilterParams.KERNEL_5x5, ] ) def getRandomTemporalFilterParams(): params = dai.node.ImageFilters.TemporalFilterParams() params.enable = random.choice([True, False]) params.persistencyMode = random.choice( [ dai.filters.params.TemporalFilter.PersistencyMode.PERSISTENCY_OFF, dai.filters.params.TemporalFilter.PersistencyMode.VALID_2_IN_LAST_4, dai.filters.params.TemporalFilter.PersistencyMode.VALID_1_IN_LAST_2, dai.filters.params.TemporalFilter.PersistencyMode.VALID_1_IN_LAST_8, dai.filters.params.TemporalFilter.PersistencyMode.PERSISTENCY_INDEFINITELY, ] ) params.alpha = random.uniform(0.3, 0.9) return params def getRandomSpeckleFilterParams(): params = dai.node.ImageFilters.SpeckleFilterParams() params.enable = random.choice([True, False]) return params def getRandomSpatialFilterParams(): params = dai.node.ImageFilters.SpatialFilterParams() params.enable = random.choice([True, False]) return params def main(args: argparse.Namespace): # Create pipeline with dai.Pipeline() as pipeline: monoLeft = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_B) monoRight = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_C) outLeft = monoLeft.requestOutput((640, 400), fps=FPS) outRight = monoRight.requestOutput((640, 400), fps=FPS) depth = pipeline.create(dai.node.StereoDepth) filterPipeline = pipeline.create(dai.node.ImageFilters) filterFactories = [ getRandomSpeckleFilterParams, getRandomTemporalFilterParams, getRandomSpatialFilterParams, getRandomMedianFilterParams, ] filterPipeline.setRunOnHost(True) depth.setLeftRightCheck(args.lr_check) depth.setExtendedDisparity(args.extended_disparity) depth.setSubpixel(args.subpixel) depth.inputConfig.setBlocking(False) # Linking outLeft.link(depth.left) outRight.link(depth.right) depthQueue = depth.disparity.createOutputQueue() filterPipeline.build(depth.disparity) ## Create a new filter pipeline filterPipeline.initialConfig.filterIndices = [] filterPipeline.initialConfig.filterParams = [ filterFactory() for filterFactory in filterFactories ] configInputQueue = filterPipeline.inputConfig.createInputQueue() filterOutputQueue = filterPipeline.output.createOutputQueue() pipeline.start() import time tSwitch = time.time() while pipeline.isRunning(): inDisparity: dai.ImgFrame = ( depthQueue.get() ) # blocking call, will wait until a new data has arrived frame = inDisparity.getFrame() filterFrame = filterOutputQueue.get() filterFrame = ( filterFrame.getFrame() * (255 / depth.initialConfig.getMaxDisparity()) ).astype(np.uint8) frame = (frame * (255 / depth.initialConfig.getMaxDisparity())).astype( np.uint8 ) cv2.imshow("disparity", frame) frame = cv2.applyColorMap(frame, cv2.COLORMAP_JET) cv2.imshow("disparity_color", frame) cv2.imshow( "filtered_disparity_color", cv2.applyColorMap(filterFrame, cv2.COLORMAP_JET), ) ## Update filter pipeline if time.time() - tSwitch > 1.0: index = random.randint(0, len(filterFactories) - 1) new_params = filterFactories[index]() config = dai.ImageFiltersConfig().updateFilterAtIndex( index, new_params ) configInputQueue.send(config) tSwitch = time.time() print(f"Filter at index {index} changed to {new_params}") key = cv2.waitKey(1) if key == ord("q"): break if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--extended_disparity", action="store_true", help="Use extended disparity" ) parser.add_argument("--subpixel", action="store_true", help="Use subpixel") parser.add_argument("--lr_check", action="store_true", help="Use left-right check") args = parser.parse_args() main(args) ``` ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.