# PointCloud Control This example demonstrates how to use PointCloudConfig message to dynamically update the transformation matrix of a point cloud and visualize the transformed point cloud using Open3D. The transformation matrix is updated to make it look like the point cloud is rotating about the Z-axis. This is achieved by first moving the point cloud along the Y axis: ```python # Move the point cloud along Y axis translate_y_matrix = np.array([ [1, 0, 0, 0], [0, 1, 0, 500], [0, 0, 1, 0], [0, 0, 0, 1] ]) ``` Then, the point cloud is rotated about the Z axis: ```python # Rotate the point cloud about Z axis rotate_z_matrix = np.array([ [np.cos(angle), -np.sin(angle), 0, 0], [np.sin(angle), np.cos(angle), 0, 0], [0, 0, 1, 0], [0, 0, 0, 1] ]) ``` ## Demo ## Setup Ensure DepthAI and Open3D are installed in your Python environment: ```bash python3 -m pip install depthai open3d ``` ## Source code The example initializes the DepthAI pipeline with color and mono cameras and a stereo depth node to generate depth information. It then creates a point cloud node, dynamically updates its transformation matrix based on a rotation value, and visualizes this transformed point cloud using Open3D. Each point's color corresponds to the color image captured by the RGB camera. #### Python ```python #!/usr/bin/env python3 import depthai as dai import numpy as np import cv2 import sys try: import open3d as o3d except ImportError: sys.exit(f"Critical dependency missing: Open3D. Please install it using the command: '{sys.executable} -m pip install open3d' and then rerun the script.") # Continue with the DepthAI and Open3D setup as before pipeline = dai.Pipeline() camRgb = pipeline.create(dai.node.ColorCamera) monoLeft = pipeline.create(dai.node.MonoCamera) monoRight = pipeline.create(dai.node.MonoCamera) depth = pipeline.create(dai.node.StereoDepth) pointcloud: dai.node.PointCloud = pipeline.create(dai.node.PointCloud) sync = pipeline.create(dai.node.Sync) xOut = pipeline.create(dai.node.XLinkOut) xOut.input.setBlocking(False) camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P) camRgb.setBoardSocket(dai.CameraBoardSocket.CAM_A) camRgb.setIspScale(1,3) monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P) monoLeft.setCamera("left") monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P) monoRight.setCamera("right") depth.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY) depth.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_7x7) depth.setLeftRightCheck(True) depth.setExtendedDisparity(False) depth.setSubpixel(True) depth.setDepthAlign(dai.CameraBoardSocket.CAM_A) config = depth.initialConfig.get() config.postProcessing.thresholdFilter.minRange = 200 config.postProcessing.thresholdFilter.maxRange = 1000 depth.initialConfig.set(config) monoLeft.out.link(depth.left) monoRight.out.link(depth.right) depth.depth.link(pointcloud.inputDepth) camRgb.isp.link(sync.inputs["rgb"]) pointcloud.outputPointCloud.link(sync.inputs["pcl"]) pointcloud.initialConfig.setSparse(False) sync.out.link(xOut.input) xOut.setStreamName("out") inConfig = pipeline.create(dai.node.XLinkIn) inConfig.setStreamName("config") inConfig.out.link(pointcloud.inputConfig) with dai.Device(pipeline) as device: isRunning = True def key_callback(vis, action, mods): global isRunning if action == 0: isRunning = False q = device.getOutputQueue(name="out", maxSize=4, blocking=False) pclConfIn = device.getInputQueue(name="config", maxSize=4, blocking=False) vis = o3d.visualization.Visualizer() vis.create_window() pcd = o3d.geometry.PointCloud() coordinateFrame = o3d.geometry.TriangleMesh.create_coordinate_frame(size=1000, origin=[0,0,0]) vis.add_geometry(coordinateFrame) vis.add_geometry(pcd) first = True rot = 0 while device.isPipelineRunning(): inMessage = q.get() inColor = inMessage["rgb"] inPointCloud = inMessage["pcl"] cvColorFrame = inColor.getCvFrame() cvRGBFrame = cv2.cvtColor(cvColorFrame, cv2.COLOR_BGR2RGB) if inPointCloud: points = inPointCloud.getPoints().astype(np.float64) pcd.points = o3d.utility.Vector3dVector(points) colors = (cvRGBFrame.reshape(-1, 3) / 255.0).astype(np.float64) pcd.colors = o3d.utility.Vector3dVector(colors) vis.update_geometry(pcd) vis.poll_events() vis.update_renderer() # Create a transformation matrix that rotates around all 3 axes at the same time rotate_z_matrix = np.array([[np.cos(rot), -np.sin(rot), 0, 0], [np.sin(rot), np.cos(rot), 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) translate_y_matrix = np.array([[1, 0, 0, 0], [0, 1, 0, 500], [0, 0, 1, 0], [0, 0, 0, 1]]) # Combine the 3 transformation matrices into one transformation_matrix = rotate_z_matrix @ translate_y_matrix rot += 0.05 pclConfig = dai.PointCloudConfig() pclConfig.setTransformationMatrix(transformation_matrix) pclConfig.setSparse(False) pclConfIn.send(pclConfig) vis.destroy_window() cv2.destroyAllWindows() ``` ## Pipeline ### examples/pointcloud_control.pipeline.json ```json { "pipeline": { "connections": [ { "node1Id": 2, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "right", "node2InputGroup": "" }, { "node1Id": 1, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "left", "node2InputGroup": "" }, { "node1Id": 3, "node1Output": "depth", "node1OutputGroup": "", "node2Id": 4, "node2Input": "inputDepth", "node2InputGroup": "" }, { "node1Id": 7, "node1Output": "out", "node1OutputGroup": "", "node2Id": 4, "node2Input": "inputConfig", "node2InputGroup": "" }, { "node1Id": 4, "node1Output": "outputPointCloud", "node1OutputGroup": "", "node2Id": 5, "node2Input": "pcl", "node2InputGroup": "inputs" }, { "node1Id": 0, "node1Output": "isp", "node1OutputGroup": "", "node2Id": 5, "node2Input": "rgb", "node2InputGroup": "inputs" }, { "node1Id": 5, "node1Output": "out", "node1OutputGroup": "", "node2Id": 6, "node2Input": "in", "node2InputGroup": "" } ], "globalProperties": { "calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1 }, "nodes": [ [ 0, { "id": 0, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 1, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 6, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "still" ], { "blocking": false, "group": "", "id": 7, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 2, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "video" ], { "blocking": false, "group": "", "id": 3, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "isp" ], { "blocking": false, "group": "", "id": 4, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "preview" ], { "blocking": false, "group": "", "id": 5, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 8, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ColorCamera", "properties": { "boardSocket": 0, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 30.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": true, "isp3aFps": 0, "ispScale": { "horizDenominator": 3, "horizNumerator": 1, "vertDenominator": 3, "vertNumerator": 1 }, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 300, "previewKeepAspectRatio": true, "previewWidth": 300, "rawPacked": null, "resolution": 0, "sensorCropX": -1.0, "sensorCropY": -1.0, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1 } } ], [ 1, { "id": 1, "ioInfo": [ [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 9, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 10, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 11, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 12, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "MonoCamera", "properties": { "boardSocket": -1, "cameraName": "left", "fps": 30.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "isp3aFps": 0, "numFramesPool": 3, "numFramesPoolRaw": 3, "rawPacked": null, "resolution": 2 } } ], [ 2, { "id": 2, "ioInfo": [ [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 13, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 14, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 15, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 16, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "MonoCamera", "properties": { "boardSocket": -1, "cameraName": "right", "fps": 30.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "isp3aFps": 0, "numFramesPool": 3, "numFramesPoolRaw": 3, "rawPacked": null, "resolution": 2 } } ], [ 3, { "id": 3, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 17, "name": "inputConfig", "queueSize": 4, "type": 3, "waitForMessage": false } ], [ [ "", "left" ], { "blocking": false, "group": "", "id": 18, "name": "left", "queueSize": 8, "type": 3, "waitForMessage": true } ], [ [ "", "debugExtDispLrCheckIt1" ], { "blocking": false, "group": "", "id": 30, "name": "debugExtDispLrCheckIt1", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "right" ], { "blocking": false, "group": "", "id": 19, "name": "right", "queueSize": 8, "type": 3, "waitForMessage": true } ], [ [ "", "syncedLeft" ], { "blocking": false, "group": "", "id": 20, "name": "syncedLeft", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "depth" ], { "blocking": false, "group": "", "id": 21, "name": "depth", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "disparity" ], { "blocking": false, "group": "", "id": 22, "name": "disparity", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "syncedRight" ], { "blocking": false, "group": "", "id": 23, "name": "syncedRight", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "debugDispCostDump" ], { "blocking": false, "group": "", "id": 28, "name": "debugDispCostDump", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "debugDispLrCheckIt2" ], { "blocking": false, "group": "", "id": 29, "name": "debugDispLrCheckIt2", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "rectifiedLeft" ], { "blocking": false, "group": "", "id": 24, "name": "rectifiedLeft", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "debugExtDispLrCheckIt2" ], { "blocking": false, "group": "", "id": 31, "name": "debugExtDispLrCheckIt2", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "rectifiedRight" ], { "blocking": false, "group": "", "id": 25, "name": "rectifiedRight", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "confidenceMap" ], { "blocking": false, "group": "", "id": 32, "name": "confidenceMap", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "outConfig" ], { "blocking": false, "group": "", "id": 26, "name": "outConfig", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "debugDispLrCheckIt1" ], { "blocking": false, "group": "", "id": 27, "name": "debugDispLrCheckIt1", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "StereoDepth", "properties": { "alphaScaling": null, "baseline": null, "depthAlignCamera": 0, "depthAlignmentUseSpecTranslation": null, "disparityToDepthUseSpecTranslation": null, "enableRectification": true, "enableRuntimeStereoModeSwitch": false, "focalLength": null, "focalLengthFromCalibration": true, "height": null, "initialConfig": { "algorithmControl": { "centerAlignmentShiftFactor": null, "customDepthUnitMultiplier": 1000.0, "depthAlign": 0, "depthUnit": 2, "disparityShift": 0, "enableExtended": false, "enableLeftRightCheck": true, "enableSubpixel": true, "leftRightCheckThreshold": 10, "numInvalidateEdgePixels": 0, "subpixelFractionalBits": 3 }, "censusTransform": { "enableMeanMode": true, "kernelMask": 0, "kernelSize": -1, "threshold": 0 }, "costAggregation": { "divisionFactor": 1, "horizontalPenaltyCostP1": 250, "horizontalPenaltyCostP2": 500, "verticalPenaltyCostP1": 250, "verticalPenaltyCostP2": 500 }, "costMatching": { "confidenceThreshold": 245, "disparityWidth": 1, "enableCompanding": false, "invalidDisparityValue": 0, "linearEquationParameters": { "alpha": 0, "beta": 2, "threshold": 127 } }, "postProcessing": { "bilateralSigmaValue": 0, "brightnessFilter": { "maxBrightness": 256, "minBrightness": 0 }, "decimationFilter": { "decimationFactor": 1, "decimationMode": 0 }, "median": 7, "spatialFilter": { "alpha": 0.5, "delta": 0, "enable": false, "holeFillingRadius": 2, "numIterations": 1 }, "speckleFilter": { "enable": false, "speckleRange": 50 }, "temporalFilter": { "alpha": 0.4000000059604645, "delta": 0, "enable": false, "persistencyMode": 3 }, "thresholdFilter": { "maxRange": 1000, "minRange": 200 } } }, "mesh": { "meshLeftUri": "", "meshRightUri": "", "meshSize": null, "stepHeight": 16, "stepWidth": 16 }, "numFramesPool": 3, "numPostProcessingMemorySlices": -1, "numPostProcessingShaves": -1, "outHeight": null, "outKeepAspectRatio": true, "outWidth": null, "rectificationUseSpecTranslation": null, "rectifyEdgeFillColor": 0, "useHomographyRectification": null, "width": null } } ], [ 4, { "id": 4, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 33, "name": "inputConfig", "queueSize": 4, "type": 3, "waitForMessage": false } ], [ [ "", "inputDepth" ], { "blocking": false, "group": "", "id": 34, "name": "inputDepth", "queueSize": 4, "type": 3, "waitForMessage": true } ], [ [ "", "outputPointCloud" ], { "blocking": false, "group": "", "id": 35, "name": "outputPointCloud", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "passthroughDepth" ], { "blocking": false, "group": "", "id": 36, "name": "passthroughDepth", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "PointCloud", "properties": { "initialConfig": { "sparse": false, "transformationMatrix": [ [ 1.0, 0.0, 0.0, 0.0 ], [ 0.0, 1.0, 0.0, 0.0 ], [ 0.0, 0.0, 1.0, 0.0 ], [ 0.0, 0.0, 0.0, 1.0 ] ] }, "numFramesPool": 4 } } ], [ 5, { "id": 5, "ioInfo": [ [ [ "inputs", "rgb" ], { "blocking": true, "group": "inputs", "id": 37, "name": "rgb", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "inputs", "pcl" ], { "blocking": true, "group": "inputs", "id": 38, "name": "pcl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 39, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "Sync", "properties": { "syncAttempts": -1, "syncThresholdNs": 10000000 } } ], [ 6, { "id": 6, "ioInfo": [ [ [ "", "in" ], { "blocking": false, "group": "", "id": 40, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "out" } } ], [ 7, { "id": 7, "ioInfo": [ [ [ "", "out" ], { "blocking": false, "group": "", "id": 41, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "XLinkIn", "properties": { "maxDataSize": 5242880, "numFrames": 8, "streamName": "config" } } ] ] } } ``` ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.