Neural Depth

1#!/usr/bin/env python3
2
3import cv2
4import depthai as dai
5import numpy as np
6
7FPS = 10
8# Create pipeline
9with dai.Pipeline() as pipeline:
10    cameraLeft = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_B, sensorFps=FPS)
11    cameraRight = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_C, sensorFps=FPS)
12    leftOutput = cameraLeft.requestFullResolutionOutput()
13    rightOutput = cameraRight.requestFullResolutionOutput()
14
15    neuralDepth = pipeline.create(dai.node.NeuralDepth).build(leftOutput, rightOutput, dai.DeviceModelZoo.NEURAL_DEPTH_LARGE)
16
17    confidenceQueue = neuralDepth.confidence.createOutputQueue()
18    edgeQueue = neuralDepth.edge.createOutputQueue()
19    disparityQueue = neuralDepth.disparity.createOutputQueue()
20
21    inputConfigQueue = neuralDepth.inputConfig.createInputQueue()
22    # Connect to device and start pipeline
23    pipeline.start()
24    maxDisparity = 1
25    colorMap = cv2.applyColorMap(np.arange(256, dtype=np.uint8), cv2.COLORMAP_JET)
26    colorMap[0] = [0, 0, 0]  # to make zero-disparity pixels black
27    currentConfig = neuralDepth.initialConfig
28    print("For adjusting thresholds, use keys:")
29    print(" - 'w': Increase confidence threshold")
30    print(" - 's': Decrease confidence threshold")
31    print(" - 'd': Increase edge threshold")
32    print(" - 'a': Decrease edge threshold")
33    print(" - 't': Toggle temporal filtering")
34    while pipeline.isRunning():
35        confidenceData = confidenceQueue.get()
36        assert isinstance(confidenceData, dai.ImgFrame)
37        npConfidence = confidenceData.getFrame()
38        colorizedConfidence = cv2.applyColorMap(((npConfidence)).astype(np.uint8), colorMap)
39        cv2.imshow("confidence", colorizedConfidence)
40
41        edgeData = edgeQueue.get()
42        assert isinstance(edgeData, dai.ImgFrame)
43        npEdge = edgeData.getFrame()
44        colorizedEdge = cv2.applyColorMap(((npEdge)).astype(np.uint8), colorMap)
45        cv2.imshow("edge", colorizedEdge)
46
47
48        disparityData = disparityQueue.get()
49        assert isinstance(disparityData, dai.ImgFrame)
50        npDisparity = disparityData.getFrame()
51        maxDisparity = max(maxDisparity, np.max(npDisparity))
52        colorizedDisparity = cv2.applyColorMap(((npDisparity / maxDisparity) * 255).astype(np.uint8), colorMap)
53        cv2.imshow("disparity", colorizedDisparity)
54
55        key = cv2.waitKey(1)
56        if key == ord('q'):
57            pipeline.stop()
58            break
59        if key == ord('w'):
60            currentThreshold = currentConfig.getConfidenceThreshold()
61            currentConfig.setConfidenceThreshold((currentThreshold + 5) % 255)
62            print("Setting confidence threshold to:", currentConfig.getConfidenceThreshold())
63            inputConfigQueue.send(currentConfig)
64        if key == ord('s'):
65            currentThreshold = currentConfig.getConfidenceThreshold()
66            currentConfig.setConfidenceThreshold((currentThreshold - 5) % 255)
67            print("Setting confidence threshold to:", currentConfig.getConfidenceThreshold())
68            inputConfigQueue.send(currentConfig)
69        if key == ord('d'):
70            currentThreshold = currentConfig.getEdgeThreshold()
71            currentConfig.setEdgeThreshold((currentThreshold + 1) % 255)
72            print("Setting edge threshold to:", currentConfig.getEdgeThreshold())
73            inputConfigQueue.send(currentConfig)
74        if key == ord('a'):
75            currentThreshold = currentConfig.getEdgeThreshold()
76            currentConfig.setEdgeThreshold((currentThreshold - 1) % 255)
77            print("Setting edge threshold to:", currentConfig.getEdgeThreshold())
78            inputConfigQueue.send(currentConfig)
79        if key == ord('t'):
80            currentConfig.postProcessing.temporalFilter.enable = not currentConfig.postProcessing.temporalFilter.enable
81            print("Temporal filtering:", "on" if currentConfig.postProcessing.temporalFilter.enable else "off")
82            inputConfigQueue.send(currentConfig)
83
84        if cv2.waitKey(1) == ord('q'):
85            break

1#include <atomic>
2#include <csignal>
3#include <iostream>
4#include <opencv2/opencv.hpp>
5
6#include "depthai/depthai.hpp"
7
8// Global flag for graceful shutdown
9std::atomic<bool> quitEvent(false);
10
11void signalHandler(int signum) {
12    quitEvent = true;
13}
14
15int main() {
16    // Set up signal handlers for clean exit
17    signal(SIGTERM, signalHandler);
18    signal(SIGINT, signalHandler);
19
20    constexpr float FPS = 10.0f;
21
22    // Create pipeline
23    dai::Pipeline pipeline;
24
25    // Define sources
26    auto cameraLeft = pipeline.create<dai::node::Camera>();
27    cameraLeft->build(dai::CameraBoardSocket::CAM_B, std::nullopt, FPS);
28
29    auto cameraRight = pipeline.create<dai::node::Camera>();
30    cameraRight->build(dai::CameraBoardSocket::CAM_C, std::nullopt, FPS);
31
32    // Get full resolution outputs
33    auto leftOutput = cameraLeft->requestFullResolutionOutput();
34    auto rightOutput = cameraRight->requestFullResolutionOutput();
35
36    // Create and build NeuralDepth node
37    auto neuralDepth = pipeline.create<dai::node::NeuralDepth>();
38    neuralDepth->build(*leftOutput, *rightOutput, dai::DeviceModelZoo::NEURAL_DEPTH_LARGE);
39
40    // Create output queues
41    auto confidenceQueue = neuralDepth->confidence.createOutputQueue();
42    auto edgeQueue = neuralDepth->edge.createOutputQueue();
43    auto disparityQueue = neuralDepth->disparity.createOutputQueue();
44
45    // Create input queue for runtime configuration
46    auto inputConfigQueue = neuralDepth->inputConfig.createInputQueue();
47
48    // Start the pipeline
49    pipeline.start();
50
51    // Variables for visualization
52    double maxDisparity = 1.0;
53    cv::Mat colorMap;
54    cv::Mat gray(256, 1, CV_8UC1);
55    for(int i = 0; i < 256; i++) {
56        gray.at<uchar>(i) = i;
57    }
58    cv::applyColorMap(gray, colorMap, cv::COLORMAP_JET);
59    colorMap.at<cv::Vec3b>(0) = cv::Vec3b(0, 0, 0);  // Set zero-disparity pixels to black
60
61    // Get the initial configuration
62    auto currentConfig = neuralDepth->initialConfig;
63
64    std::cout << "For adjusting thresholds, use keys:" << std::endl;
65    std::cout << " - 'w': Increase confidence threshold" << std::endl;
66    std::cout << " - 's': Decrease confidence threshold" << std::endl;
67    std::cout << " - 'd': Increase edge threshold" << std::endl;
68    std::cout << " - 'a': Decrease edge threshold" << std::endl;
69    std::cout << " - 't': Toggle temporal filtering" << std::endl;
70
71    while(!quitEvent && pipeline.isRunning()) {
72        // Get confidence data and display it
73        auto confidenceData = confidenceQueue->get<dai::ImgFrame>();
74        cv::Mat npConfidence = confidenceData->getFrame();
75        cv::Mat colorizedConfidence;
76        cv::applyColorMap(npConfidence, colorizedConfidence, colorMap);
77        cv::imshow("confidence", colorizedConfidence);
78
79        // Get edge data and display it
80        auto edgeData = edgeQueue->get<dai::ImgFrame>();
81        cv::Mat npEdge = edgeData->getFrame();
82        cv::Mat colorizedEdge;
83        cv::applyColorMap(npEdge, colorizedEdge, colorMap);
84        cv::imshow("edge", colorizedEdge);
85
86        // Get disparity data, normalize, and display it
87        auto disparityData = disparityQueue->get<dai::ImgFrame>();
88        cv::Mat npDisparity = disparityData->getFrame();
89
90        double minVal, curMax;
91        cv::minMaxLoc(npDisparity, &minVal, &curMax);
92        maxDisparity = std::max(maxDisparity, curMax);
93
94        cv::Mat normalized;
95        npDisparity.convertTo(normalized, CV_8UC1, 255.0 / maxDisparity);
96
97        cv::Mat colorizedDisparity;
98        cv::applyColorMap(normalized, colorizedDisparity, colorMap);
99        cv::imshow("disparity", colorizedDisparity);
100
101        // Check for keyboard input
102        int key = cv::waitKey(1);
103        bool configChanged = false;
104        if(key == 'q') {
105            break;
106        } else if(key == 'w') {  // Increase confidence threshold
107            uint8_t currentThreshold = currentConfig->getConfidenceThreshold();
108            currentConfig->setConfidenceThreshold((currentThreshold + 5) % 255);
109            std::cout << "Setting confidence threshold to: " << (int)currentConfig->getConfidenceThreshold() << std::endl;
110            configChanged = true;
111        } else if(key == 's') {  // Decrease confidence threshold
112            uint8_t currentThreshold = currentConfig->getConfidenceThreshold();
113            currentConfig->setConfidenceThreshold(currentThreshold - 5);  // uint8_t will wrap around on underflow
114            std::cout << "Setting confidence threshold to: " << (int)currentConfig->getConfidenceThreshold() << std::endl;
115            configChanged = true;
116        } else if(key == 'd') {  // Increase edge threshold
117            uint8_t currentThreshold = currentConfig->getEdgeThreshold();
118            currentConfig->setEdgeThreshold((currentThreshold + 1) % 255);
119            std::cout << "Setting edge threshold to: " << (int)currentConfig->getEdgeThreshold() << std::endl;
120            configChanged = true;
121        } else if(key == 'a') {  // Decrease edge threshold
122            uint8_t currentThreshold = currentConfig->getEdgeThreshold();
123            currentConfig->setEdgeThreshold(currentThreshold - 1);  // uint8_t will wrap around on underflow
124            std::cout << "Setting edge threshold to: " << (int)currentConfig->getEdgeThreshold() << std::endl;
125            configChanged = true;
126        } else if(key == 't') {
127            bool enable = !currentConfig->postProcessing.temporalFilter.enable;
128            currentConfig->postProcessing.temporalFilter.enable = enable;
129            std::cout << "Temporal filtering: " << (enable ? "on" : "off") << std::endl;
130            configChanged = true;
131        }
132
133        // Send the updated configuration to the device
134        if(configChanged) {
135            inputConfigQueue->send(currentConfig);
136        }
137    }
138
139    // Pipeline is stopped automatically when the 'pipeline' object goes out of scope.
140    return 0;
141}