Stereo Depth Remap

1import depthai as dai
2import cv2
3import numpy as np
4
5def draw_rotated_rectangle(frame, center, size, angle, color, thickness=2):
6    """
7    Draws a rotated rectangle on the given frame.
8
9    Args:
10        frame (numpy.ndarray): The image/frame to draw on.
11        center (tuple): The (x, y) coordinates of the rectangle's center.
12        size (tuple): The (width, height) of the rectangle.
13        angle (float): The rotation angle of the rectangle in degrees (counter-clockwise).
14        color (tuple): The color of the rectangle in BGR format (e.g., (0, 255, 0) for green).
15        thickness (int): The thickness of the rectangle edges. Default is 2.
16    """
17    # Create a rotated rectangle
18    rect = ((center[0], center[1]), (size[0], size[1]), angle)
19
20    # Get the four vertices of the rotated rectangle
21    box = cv2.boxPoints(rect)
22    box = np.intp(box)  # Convert to integer coordinates
23
24    # Draw the rectangle on the frame
25    cv2.polylines(frame, [box], isClosed=True, color=color, thickness=thickness)
26
27def processDepthFrame(depthFrame):
28    depth_downscaled = depthFrame[::4]
29    if np.all(depth_downscaled == 0):
30        min_depth = 0
31    else:
32        min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1)
33    max_depth = np.percentile(depth_downscaled, 99)
34    depthFrameColor = np.interp(depthFrame, (min_depth, max_depth), (0, 255)).astype(np.uint8)
35    return cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_HOT)
36
37with dai.Pipeline() as pipeline:
38    color = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_A)
39    monoLeft = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_B)
40    monoRight = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_C)
41    stereo = pipeline.create(dai.node.StereoDepth)
42
43    stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.DEFAULT)
44    # stereo.setDepthAlign(dai.CameraBoardSocket.CAM_A)
45    # stereo.setOutputSize(640, 400)
46
47    colorCamOut = color.requestOutput((640, 480))
48
49    monoLeftOut = monoLeft.requestOutput((640, 480))
50    monoRightOut = monoRight.requestOutput((640, 480))
51
52    monoLeftOut.link(stereo.left)
53    monoRightOut.link(stereo.right)
54
55    colorOut = colorCamOut.createOutputQueue()
56    rightOut = monoRightOut.createOutputQueue()
57    stereoOut = stereo.depth.createOutputQueue()
58
59    pipeline.start()
60    while pipeline.isRunning():
61        colorFrame = colorOut.get()
62        stereoFrame = stereoOut.get()
63
64        assert colorFrame.validateTransformations()
65        assert stereoFrame.validateTransformations()
66
67        clr = colorFrame.getCvFrame()
68        depth = processDepthFrame(stereoFrame.getCvFrame())
69
70        rect = dai.RotatedRect(dai.Point2f(300, 200), dai.Size2f(200, 100), 10)
71        remappedRect = colorFrame.getTransformation().remapRectTo(stereoFrame.getTransformation(), rect)
72
73        print(f"Original rect x: {rect.center.x} y: {rect.center.y} width: {rect.size.width} height: {rect.size.height} angle: {rect.angle}")
74        print(f"Remapped rect x: {remappedRect.center.x} y: {remappedRect.center.y} width: {remappedRect.size.width} height: {remappedRect.size.height} angle: {remappedRect.angle}")
75
76        draw_rotated_rectangle(clr, (rect.center.x, rect.center.y), (rect.size.width, rect.size.height), rect.angle, (255, 0, 0))
77        draw_rotated_rectangle(depth, (remappedRect.center.x, remappedRect.center.y), (remappedRect.size.width, remappedRect.size.height), remappedRect.angle, (255, 0, 0))
78
79        cv2.imshow("color", clr)
80        cv2.imshow("depth", depth)
81
82        if cv2.waitKey(1) == ord('q'):
83            break
84    pipeline.stop()

1#include <iostream>
2#include <opencv2/opencv.hpp>
3
4#include "depthai/depthai.hpp"
5
6// Helper function to draw rotated rectangle
7void drawRotatedRectangle(cv::Mat& frame, const cv::Point2f& center, const cv::Size2f& size, float angle, const cv::Scalar& color, int thickness = 2) {
8    // Create a rotated rectangle
9    cv::RotatedRect rect(center, size, angle);
10
11    // Get the four vertices of the rotated rectangle
12    cv::Point2f vertices[4];
13    rect.points(vertices);
14
15    // Convert vertices to integer points
16    std::vector<cv::Point> points;
17    for(int i = 0; i < 4; i++) {
18        points.push_back(cv::Point(static_cast<int>(vertices[i].x), static_cast<int>(vertices[i].y)));
19    }
20
21    // Draw the rectangle
22    cv::polylines(frame, points, true, color, thickness);
23}
24
25// Helper function to process depth frame
26cv::Mat processDepthFrame(const cv::Mat& depthFrame) {
27    cv::Mat depth_downscaled;
28    cv::resize(depthFrame, depth_downscaled, cv::Size(), 0.25, 0.25);
29
30    double min_depth = 0;
31    if(!cv::countNonZero(depth_downscaled == 0)) {
32        std::vector<uint16_t> nonZeroDepth;
33        nonZeroDepth.reserve(depth_downscaled.rows * depth_downscaled.cols);
34
35        for(int i = 0; i < depth_downscaled.rows; i++) {
36            for(int j = 0; j < depth_downscaled.cols; j++) {
37                uint16_t depth = depth_downscaled.at<uint16_t>(i, j);
38                if(depth > 0) nonZeroDepth.push_back(depth);
39            }
40        }
41
42        if(!nonZeroDepth.empty()) {
43            std::sort(nonZeroDepth.begin(), nonZeroDepth.end());
44            min_depth = nonZeroDepth[static_cast<int>(nonZeroDepth.size() * 0.01)];  // 1st percentile
45        }
46    }
47
48    std::vector<uint16_t> allDepth;
49    allDepth.reserve(depth_downscaled.rows * depth_downscaled.cols);
50    for(int i = 0; i < depth_downscaled.rows; i++) {
51        for(int j = 0; j < depth_downscaled.cols; j++) {
52            allDepth.push_back(depth_downscaled.at<uint16_t>(i, j));
53        }
54    }
55    std::sort(allDepth.begin(), allDepth.end());
56    double max_depth = allDepth[static_cast<int>(allDepth.size() * 0.99)];  // 99th percentile
57
58    // Normalize and colorize
59    cv::Mat normalized;
60    cv::normalize(depthFrame, normalized, 0, 255, cv::NORM_MINMAX, CV_8UC1, depthFrame > min_depth);
61    cv::Mat colorized;
62    cv::applyColorMap(normalized, colorized, cv::COLORMAP_HOT);
63    return colorized;
64}
65
66int main() {
67    // Create pipeline
68    dai::Pipeline pipeline;
69
70    // Create and configure nodes
71    auto color = pipeline.create<dai::node::Camera>();
72    color->build(dai::CameraBoardSocket::CAM_A);
73
74    auto monoLeft = pipeline.create<dai::node::Camera>();
75    monoLeft->build(dai::CameraBoardSocket::CAM_B);
76
77    auto monoRight = pipeline.create<dai::node::Camera>();
78    monoRight->build(dai::CameraBoardSocket::CAM_C);
79
80    auto stereo = pipeline.create<dai::node::StereoDepth>();
81
82    // Configure stereo node
83    stereo->setDefaultProfilePreset(dai::node::StereoDepth::PresetMode::DEFAULT);
84    // Uncomment to align depth to RGB
85    // stereo->setDepthAlign(dai::CameraBoardSocket::CAM_A);
86    // stereo->setOutputSize(640, 400);
87
88    // Configure outputs
89    auto colorCamOut = color->requestOutput(std::make_pair(640, 480));
90    auto monoLeftOut = monoLeft->requestOutput(std::make_pair(640, 480));
91    auto monoRightOut = monoRight->requestOutput(std::make_pair(640, 480));
92
93    // Link mono cameras to stereo
94    monoLeftOut->link(stereo->left);
95    monoRightOut->link(stereo->right);
96
97    // Create output queues
98    auto colorOut = colorCamOut->createOutputQueue();
99    auto rightOut = monoRightOut->createOutputQueue();
100    auto stereoOut = stereo->depth.createOutputQueue();
101
102    pipeline.start();
103
104    while(pipeline.isRunning()) {
105        auto colorFrame = colorOut->get<dai::ImgFrame>();
106        auto stereoFrame = stereoOut->get<dai::ImgFrame>();
107
108        if(colorFrame == nullptr || stereoFrame == nullptr) continue;
109
110        // Validate transformations
111        if(!colorFrame->validateTransformations() || !stereoFrame->validateTransformations()) {
112            std::cerr << "Invalid transformations!" << std::endl;
113            continue;
114        }
115
116        // Get frames
117        cv::Mat clr = colorFrame->getCvFrame();
118        cv::Mat depth = processDepthFrame(stereoFrame->getCvFrame());
119
120        // Create and remap rectangle
121        dai::RotatedRect rect(dai::Point2f(300, 200), dai::Size2f(200, 100), 10);
122        auto remappedRect = colorFrame->transformation.remapRectTo(stereoFrame->transformation, rect);
123
124        // Print rectangle information
125        std::cout << "Original rect x: " << rect.center.x << " y: " << rect.center.y << " width: " << rect.size.width << " height: " << rect.size.height
126                  << " angle: " << rect.angle << std::endl;
127        std::cout << "Remapped rect x: " << remappedRect.center.x << " y: " << remappedRect.center.y << " width: " << remappedRect.size.width
128                  << " height: " << remappedRect.size.height << " angle: " << remappedRect.angle << std::endl;
129
130        // Draw rectangles
131        drawRotatedRectangle(clr, cv::Point2f(rect.center.x, rect.center.y), cv::Size2f(rect.size.width, rect.size.height), rect.angle, cv::Scalar(255, 0, 0));
132
133        drawRotatedRectangle(depth,
134                             cv::Point2f(remappedRect.center.x, remappedRect.center.y),
135                             cv::Size2f(remappedRect.size.width, remappedRect.size.height),
136                             remappedRect.angle,
137                             cv::Scalar(255, 0, 0));
138
139        // Show frames
140        cv::imshow("color", clr);
141        cv::imshow("depth", depth);
142
143        if(cv::waitKey(1) == 'q') {
144            break;
145        }
146    }
147
148    return 0;
149}