# RGBD Point Cloud Visualization

This example demonstrates how to use the [RGBD](https://docs.luxonis.com/software-v3/depthai/depthai-components/nodes/rgbd.md)
node to generate RGB-D data and visualize it as a 3D point cloud. The core pipeline remains the same, while visualization can be
done in one of three interchangeable ways:

 * Rerun
 * Open3D
 * OAK Visualizer - Native viewer bundled with DepthAI

The only difference between these modes is in how the `RGBD` and point cloud output is displayed. You can switch the visualizer in
the code as needed.

## Demo

This example requires the DepthAI v3 API, see [installation instructions](https://docs.luxonis.com/software-v3/depthai.md).

## Pipeline

Pipeline nodes:
- Camera(0)
- StereoDepth (3)
- Sync (5)
- XLinkOut (7)
- Camera(1)
- Camera(2)

Pipeline connections:
- Sync (5) -> XLinkOut (7) (out -> in)
- StereoDepth (3) -> Sync (5) (depth -> inDepthSync)
- Camera(2) -> Sync (5) (0 -> inColorSync)
- Camera(2) -> StereoDepth (3) (0 -> inputAlignTo)
- Camera(1) -> StereoDepth (3) (0 -> right)
- Camera(0) -> StereoDepth (3) (0 -> left)

## Source code with OAK Visualizer in the loop

#### Python

```python
import depthai as dai

from argparse import ArgumentParser

fps = 30

parser = ArgumentParser()
parser.add_argument("--webSocketPort", type=int, default=8765)
parser.add_argument("--httpPort", type=int, default=8082)
args = parser.parse_args()

with dai.Pipeline() as p:
    remoteConnector = dai.RemoteConnection(
        webSocketPort=args.webSocketPort, httpPort=args.httpPort
    )

    size = (640, 400)
    colorSockets = p.getDefaultDevice().getConnectedCameras(dai.CameraSensorType.COLOR)
    colorSocket = colorSockets[0] if colorSockets else dai.CameraBoardSocket.CAM_A
    color = p.create(dai.node.Camera).build(colorSocket, sensorFps=fps)
    depth = p.create(dai.node.Depth).build(dai.node.Depth.Algorithm.AUTO, fps, size)

    rgbd = p.create(dai.node.RGBD).build(color, depth, size, fps)

    remoteConnector.addTopic("pcl", rgbd.pcl, "common")
    p.start()
    remoteConnector.registerPipeline(p)

    print("Pipeline started")

    while p.isRunning():
        key = remoteConnector.waitKey(1)
        if key == ord("q"):
            print("Got q key from the remote connection!")
            break
```

#### C++

```cpp
#include <algorithm>
#include <argparse/argparse.hpp>
#include <csignal>
#include <depthai/remote_connection/RemoteConnection.hpp>
#include <iostream>

#include "depthai/depthai.hpp"

constexpr float NEURAL_FPS = 8.0f;
constexpr float STEREO_DEFAULT_FPS = 30.0f;

// Signal handling for clean shutdown
static bool isRunning = true;
void signalHandler(int signum) {
    (void)signum;
    isRunning = false;
}

int main(int argc, char** argv) {
    // Initialize argument parser
    argparse::ArgumentParser program("visualizer_rgbd", "1.0.0");
    program.add_description("RGBD point cloud visualizer with configurable depth source");
    program.add_argument("--webSocketPort").default_value(8765).scan<'i', int>().help("WebSocket port for remote connection");
    program.add_argument("--httpPort").default_value(8082).scan<'i', int>().help("HTTP port for remote connection");
    program.add_argument("--depthSource").default_value(std::string("stereo")).help("Depth source: stereo, neural, tof");

    // Parse arguments
    try {
        program.parse_args(argc, argv);
    } catch(const std::runtime_error& err) {
        std::cerr << err.what() << std::endl;
        std::cerr << program;
        return EXIT_FAILURE;
    }

    // Get arguments
    int webSocketPort = program.get<int>("--webSocketPort");
    int httpPort = program.get<int>("--httpPort");
    std::string depthSourceArg = program.get<std::string>("--depthSource");

    // Validate depth source argument
    if(depthSourceArg != "stereo" && depthSourceArg != "neural" && depthSourceArg != "tof") {
        std::cerr << "Invalid depth source: " << depthSourceArg << std::endl;
        std::cerr << "Valid options are: stereo, neural, tof" << std::endl;
        return EXIT_FAILURE;
    }

    // Register signal handler
    std::signal(SIGINT, signalHandler);

    try {
        // Create RemoteConnection
        dai::RemoteConnection remoteConnector(dai::RemoteConnection::DEFAULT_ADDRESS, webSocketPort, true, httpPort);

        // Create pipeline
        dai::Pipeline pipeline;

        float fps = STEREO_DEFAULT_FPS;
        if(depthSourceArg == "neural") {
            fps = NEURAL_FPS;
        }

        const std::pair<int, int> size = std::make_pair(640, 400);

        // Create color camera
        auto colorSockets = pipeline.getDefaultDevice()->getConnectedCameras(dai::CameraSensorType::COLOR);
        auto colorSocket = colorSockets.empty() ? dai::CameraBoardSocket::CAM_A : colorSockets.front();
        auto color = pipeline.create<dai::node::Camera>();
        color->build(colorSocket, std::nullopt, fps);

        // Create depth source based on argument
        dai::node::DepthSource depthSource;

        if(depthSourceArg == "stereo") {
            auto depth = pipeline.create<dai::node::Depth>();
            depth->build(dai::node::Depth::Algorithm::AUTO, fps, std::make_pair(640u, 400u));

            depthSource = depth;
        } else if(depthSourceArg == "neural") {
            auto left = pipeline.create<dai::node::Camera>();
            auto right = pipeline.create<dai::node::Camera>();

            left->build(dai::CameraBoardSocket::CAM_B, std::nullopt, fps);
            right->build(dai::CameraBoardSocket::CAM_C, std::nullopt, fps);

            auto neuralDepth = pipeline.create<dai::node::NeuralDepth>();
            neuralDepth->build(*left->requestFullResolutionOutput(), *right->requestFullResolutionOutput(), dai::DeviceModelZoo::NEURAL_DEPTH_LARGE);

            depthSource = neuralDepth;
        } else if(depthSourceArg == "tof") {
            auto tof = pipeline.create<dai::node::ToF>();
            tof->build(dai::CameraBoardSocket::AUTO, dai::ToFConfig::Profile::MID_RANGE, fps);
            depthSource = tof;
        }

        // Create RGBD node using the unified build method with DepthSource variant
        auto rgbd = pipeline.create<dai::node::RGBD>();
        rgbd->build(color, depthSource, size, fps);

        remoteConnector.addTopic("pcl", rgbd->pcl);
        pipeline.start();
        remoteConnector.registerPipeline(pipeline);

        auto device = pipeline.getDefaultDevice();
        device->setIrLaserDotProjectorIntensity(0.7);

        std::cout << "Pipeline started with depth source: " << depthSourceArg << std::endl;

        // Main loop
        while(isRunning && pipeline.isRunning()) {
            int key = remoteConnector.waitKey(1);
            if(key == 'q') {
                std::cout << "Got 'q' key from the remote connection!" << std::endl;
                break;
            }
        }

        std::cout << "Pipeline stopped." << std::endl;

    } catch(const std::exception& e) {
        std::cerr << "Error: " << e.what() << std::endl;
        return EXIT_FAILURE;
    }

    return EXIT_SUCCESS;
}
```

## Source code with Open3D in the loop

#### Python

```python
import time
import depthai as dai
import sys
import numpy as np

try:
    import open3d as o3d
except ImportError:
    sys.exit(
        "Critical dependency missing: Open3D. Please install it using the command: '{} -m pip install open3d' and then rerun the script.".format(
            sys.executable
        )
    )

class O3DNode(dai.node.ThreadedHostNode):
    def __init__(self):
        dai.node.ThreadedHostNode.__init__(self)
        self.inputPCL = self.createInput()

    def run(self):
        def key_callback(vis, action, mods):
            global isRunning
            if action == 0:
                isRunning = False

        vis = o3d.visualization.VisualizerWithKeyCallback()
        vis.create_window()
        vis.register_key_action_callback(81, key_callback)
        pcd = o3d.geometry.PointCloud()
        coordinateFrame = o3d.geometry.TriangleMesh.create_coordinate_frame(
            size=1000, origin=[0, 0, 0]
        )
        vis.add_geometry(coordinateFrame)
        first = True
        while self.mainLoop():
            try:
                inPointCloud = self.inputPCL.tryGet()
            except dai.MessageQueue.QueueException:
                return # Pipeline closed
            if inPointCloud is not None:
                points, colors = inPointCloud.getPointsRGB()
                pcd.points = o3d.utility.Vector3dVector(points.astype(np.float64))
                colors = (colors / 255.0).astype(np.float64)
                pcd.colors = o3d.utility.Vector3dVector(np.delete(colors, 3, 1))
                if first:
                    vis.add_geometry(pcd)
                    first = False
                else:
                    vis.update_geometry(pcd)
            vis.poll_events()
            vis.update_renderer()

        vis.destroy_window()

# Create pipeline

with dai.Pipeline() as p:
    fps = 30
    # Define sources and outputs
    colorSockets = p.getDefaultDevice().getConnectedCameras(dai.CameraSensorType.COLOR)
    colorSocket = colorSockets[0] if colorSockets else dai.CameraBoardSocket.CAM_A
    color = p.create(dai.node.Camera).build(colorSocket)
    depth = p.create(dai.node.Depth).build(dai.node.Depth.Algorithm.AUTO, fps, (640, 400))
    o3dViewer = p.create(O3DNode)

    rgbd = p.create(dai.node.RGBD).build(color, depth, (640, 400), fps)

    rgbd.pcl.link(o3dViewer.inputPCL)

    p.start()
    while p.isRunning():
        time.sleep(1)
```

## Source code with Rerun in the loop

#### Python

```python
import time
import depthai as dai
import sys

from pathlib import Path
installExamplesStr = Path(__file__).absolute().parents[1] / 'install_requirements.py --install_rerun'
try:
    import rerun as rr
except ImportError:
    sys.exit("Critical dependency missing: Rerun. Please install it using the command: '{} {}' and then rerun the script.".format(sys.executable, installExamplesStr))

import cv2

class RerunNode(dai.node.ThreadedHostNode):
    def __init__(self):
        dai.node.ThreadedHostNode.__init__(self)
        self.inputPCL = self.createInput()

    def run(self):
        rr.init("depthai-rgbd", spawn=True)
        rr.log("world", rr.ViewCoordinates.RDF)
        rr.log("world/ground", rr.Boxes3D(half_sizes=[3.0, 3.0, 0.00001]))
        while self.mainLoop():
            try:
                inPointCloud = self.inputPCL.get()
            except dai.MessageQueue.QueueException:
                return # Pipeline closed
            if inPointCloud is not None:
                points, colors = inPointCloud.getPointsRGB()
                rr.log("world/pcl", rr.Points3D(points, colors=colors, radii=[0.01]))

# Create pipeline

with dai.Pipeline() as p:
    fps = 30
    # Define sources and outputs
    colorSockets = p.getDefaultDevice().getConnectedCameras(dai.CameraSensorType.COLOR)
    colorSocket = colorSockets[0] if colorSockets else dai.CameraBoardSocket.CAM_A
    color = p.create(dai.node.Camera).build(colorSocket)
    depth = p.create(dai.node.Depth).build(dai.node.Depth.Algorithm.AUTO, fps, (640, 400))
    rerunViewer = p.create(RerunNode)

    rgbd = p.create(dai.node.RGBD).build(color, depth, (640, 400), fps)
    rgbd.setDepthUnits(dai.StereoDepthConfig.AlgorithmControl.DepthUnit.METER)

    rgbd.pcl.link(rerunViewer.inputPCL)

    p.start()
    while p.isRunning():
        time.sleep(1)
```

#### C++

```cpp
#include <algorithm>

#include "depthai/capabilities/ImgFrameCapability.hpp"
#include "depthai/depthai.hpp"
#include "rerun.hpp"
#include "rerun/archetypes/depth_image.hpp"

class RerunNode : public dai::NodeCRTP<dai::node::ThreadedHostNode, RerunNode> {
   public:
    constexpr static const char* NAME = "RerunNode";

   public:
    Input inputPCL{*this, {.name = "inPCL", .types = {{dai::DatatypeEnum::PointCloudData, true}}}};
    Input inputRGBD{*this, {.name = "inRGBD", .types = {{dai::DatatypeEnum::RGBDData, true}}}};

    void run() override {
        const auto rec = rerun::RecordingStream("rerun");
        rec.spawn().exit_on_failure();
        rec.log_static("world", rerun::ViewCoordinates::RDF);
        while(mainLoop()) {
            auto pclIn = inputPCL.get<dai::PointCloudData>();
            auto rgbdIn = inputRGBD.get<dai::RGBDData>();
            if(pclIn != nullptr) {
                std::vector<rerun::Position3D> points;
                std::vector<rerun::Color> colors;
                const auto& size = pclIn->getWidth() * pclIn->getHeight();
                points.reserve(size);
                colors.reserve(size);
                const auto& pclData = pclIn->getPointsRGB();
                for(size_t i = 0; i < size; ++i) {
                    points.emplace_back(pclData[i].x, pclData[i].y, pclData[i].z);
                    colors.emplace_back(pclData[i].r, pclData[i].g, pclData[i].b);
                }
                rec.log("world/obstacle_pcl", rerun::Points3D(points).with_colors(colors).with_radii({0.01f}));
                auto rgbFrame = rgbdIn->getRGBFrame();
                if(!rgbFrame.has_value()) continue;
                auto colorFrame = std::get<std::shared_ptr<dai::ImgFrame>>(rgbFrame.value())->getCvFrame();
                cv::cvtColor(colorFrame, colorFrame, cv::COLOR_BGR2RGB);
                rec.log("rgb",
                        rerun::Image(reinterpret_cast<const uint8_t*>(colorFrame.data),
                                     {static_cast<uint32_t>(colorFrame.cols), static_cast<uint32_t>(colorFrame.rows)},
                                     rerun::datatypes::ColorModel::RGB));
            }
        }
    }
};
int main() {
    using namespace std;
    // Create pipeline
    dai::Pipeline pipeline;
    // Define sources and outputs
    auto colorSockets = pipeline.getDefaultDevice()->getConnectedCameras(dai::CameraSensorType::COLOR);
    auto colorSocket = colorSockets.empty() ? dai::CameraBoardSocket::CAM_A : colorSockets.front();
    auto color = pipeline.create<dai::node::Camera>();
    color->build(colorSocket);
    auto depth = pipeline.create<dai::node::Depth>();
    depth->build(dai::node::Depth::Algorithm::AUTO, 30.0f, std::make_pair(640u, 400u));
    auto rerun = pipeline.create<RerunNode>();

    auto rgbd = pipeline.create<dai::node::RGBD>()->build(color, depth, std::make_pair(640, 400), 30.0f);
    rgbd->setDepthUnit(dai::StereoDepthConfig::AlgorithmControl::DepthUnit::METER);

    // Linking
    rgbd->pcl.link(rerun->inputPCL);
    rgbd->rgbd.link(rerun->inputRGBD);
    pipeline.start();
    auto device = pipeline.getDefaultDevice();
    device->setIrLaserDotProjectorIntensity(0.7);
    pipeline.wait();
}
```

### Need assistance?

Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.