RGB Rotate Warp

This example shows usage of ImageManip to crop a rotated rectangle area on a frame, or perform various image transforms: rotate, mirror, flip, perspective transform.

Setup

Please run the install script to download all required dependencies. Please note that this script must be ran from git context, so you have to download the depthai-python repository first and then run the script

git clone https://github.com/luxonis/depthai-python.git
cd depthai-python/examples
python3 install_requirements.py

For additional information, please follow installation guide

Demo

https://user-images.githubusercontent.com/18037362/152208899-461fa163-42ec-4922-84b5-5cd09332ea32.png 
=== Controls:
z -rotated rectangle crop, decrease rate
x -rotated rectangle crop, increase rate
c -warp 4-point transform, cycle through modes
v -resize cropped region, or disable resize
h -print controls (help)

Source code

Python

C++

Also available on GitHub

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
#!/usr/bin/env python3

"""
This example shows usage of ImageManip to crop a rotated rectangle area on a frame,
or perform various image transforms: rotate, mirror, flip, perspective transform.
"""

import depthai as dai
import cv2
import numpy as np

keyRotateDecr = 'z'
keyRotateIncr = 'x'
keyResizeInc = 'v'
keyWarpTestCycle = 'c'

def printControls():
    print("=== Controls:")
    print(keyRotateDecr, "-rotated rectangle crop, decrease rate")
    print(keyRotateIncr, "-rotated rectangle crop, increase rate")
    print(keyWarpTestCycle, "-warp 4-point transform, cycle through modes")
    print(keyResizeInc, "-resize cropped region, or disable resize")
    print("h -print controls (help)")

rotateRateMax = 5.0
rotateRateInc = 0.1

resizeMaxW = 800
resizeMaxH = 600
resizeFactorMax = 5

'''
The crop points are specified in clockwise order,
with first point mapped to output top-left, as:
    P0  ->  P1
     ^       v
    P3  <-  P2
'''
P0 = [0, 0]  # top-left
P1 = [1, 0]  # top-right
P2 = [1, 1]  # bottom-right
P3 = [0, 1]  # bottom-left

warpList = [
    # points order, normalized cordinates, description
    # [[[0, 0], [1, 0], [1, 1], [0, 1]], True, "passthrough"],
    # [[[0, 0], [639, 0], [639, 479], [0, 479]], False, "passthrough (pixels)"],
    [[P0, P1, P2, P3], True, "1. passthrough"],
    [[P3, P0, P1, P2], True, "2. rotate 90"],
    [[P2, P3, P0, P1], True, "3. rotate 180"],
    [[P1, P2, P3, P0], True, "4. rotate 270"],
    [[P1, P0, P3, P2], True, "5. horizontal mirror"],
    [[P3, P2, P1, P0], True, "6. vertical flip"],
    [[[-0.1, -0.1], [1.1, -0.1], [1.1, 1.1], [-0.1, 1.1]], True, "7. add black borders"],
    [[[-0.3, 0], [1, 0], [1.3, 1], [0, 1]], True, "8. parallelogram transform"],
    [[[-0.2, 0], [1.8, 0], [1, 1], [0, 1]], True, "9. trapezoid transform"],
]

# Create pipeline
pipeline = dai.Pipeline()

# Define sources and outputs
camRgb = pipeline.create(dai.node.ColorCamera)
manip = pipeline.create(dai.node.ImageManip)

camOut = pipeline.create(dai.node.XLinkOut)
manipOut = pipeline.create(dai.node.XLinkOut)
manipCfg = pipeline.create(dai.node.XLinkIn)

camOut.setStreamName("preview")
manipOut.setStreamName("manip")
manipCfg.setStreamName("manipCfg")

# Properties
camRgb.setPreviewSize(640, 480)
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
camRgb.setInterleaved(False)
camRgb.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
manip.setMaxOutputFrameSize(2000 * 1500 * 3)

# Linking
camRgb.preview.link(camOut.input)
camRgb.preview.link(manip.inputImage)
manip.out.link(manipOut.input)
manipCfg.out.link(manip.inputConfig)

# Connect to device and start pipeline
with dai.Device(pipeline) as device:

    # Create input & output queues
    qPreview = device.getOutputQueue(name="preview", maxSize=4)
    qManip = device.getOutputQueue(name="manip", maxSize=4)
    qManipCfg = device.getInputQueue(name="manipCfg")

    key = -1
    angleDeg = 0
    rotateRate = 1.0
    resizeFactor = 0
    resizeX = 0
    resizeY = 0
    testFourPt = False
    warpIdx = -1

    printControls()

    while key != ord('q'):
        if key > 0:
            print("Pressed: ", key)
            if key == ord(keyRotateDecr) or key == ord(keyRotateIncr):
                if key == ord(keyRotateDecr):
                    if rotateRate > -rotateRateMax:
                        rotateRate -= rotateRateInc
                if key == ord(keyRotateIncr):
                    if rotateRate < rotateRateMax:
                        rotateRate += rotateRateInc
                testFourPt = False
                print("Crop rotated rectangle, rate per frame: {:.1f} degrees".format(rotateRate))
            elif key == ord(keyResizeInc):
                resizeFactor += 1
                if resizeFactor > resizeFactorMax:
                    resizeFactor = 0
                    print("Crop region not resized")
                else:
                    resizeX = resizeMaxW // resizeFactor
                    resizeY = resizeMaxH // resizeFactor
                    print("Crop region resized to: ", resizeX, 'x', resizeY)
            elif key == ord(keyWarpTestCycle):
                # Disable resizing initially
                resizeFactor = 0
                warpIdx = (warpIdx + 1) % len(warpList)
                testFourPt = True
                testDescription = warpList[warpIdx][2]
                print("Warp 4-point transform: ", testDescription)
            elif key == ord('h'):
                printControls()

        # Send an updated config with continuous rotate, or after a key press
        if key >= 0 or (not testFourPt and abs(rotateRate) > 0.0001):
            cfg = dai.ImageManipConfig()
            if testFourPt:
                test = warpList[warpIdx]
                points, normalized = test[0], test[1]
                point2fList = []
                for p in points:
                    pt = dai.Point2f()
                    pt.x, pt.y = p[0], p[1]
                    point2fList.append(pt)
                cfg.setWarpTransformFourPoints(point2fList, normalized)
            else:
                angleDeg += rotateRate
                rotatedRect = ((320, 240), (400, 400), angleDeg)
                rr = dai.RotatedRect()
                rr.center.x, rr.center.y = rotatedRect[0]
                rr.size.width, rr.size.height = rotatedRect[1]
                rr.angle = rotatedRect[2]
                cfg.setCropRotatedRect(rr, False)
            if resizeFactor > 0:
                cfg.setResize(resizeX, resizeY)
            # cfg.setWarpBorderFillColor(255, 0, 0)
            # cfg.setWarpBorderReplicatePixels()
            qManipCfg.send(cfg)

        for q in [qPreview, qManip]:
            pkt = q.get()
            name = q.getName()
            shape = (3, pkt.getHeight(), pkt.getWidth())
            frame = pkt.getCvFrame()
            if name == "preview" and not testFourPt:
                # Draw RotatedRect cropped area on input frame
                points = np.int0(cv2.boxPoints(rotatedRect))
                cv2.drawContours(frame, [points], 0, (255, 0, 0), 1)
                # Mark top-left corner
                cv2.circle(frame, tuple(points[1]), 10, (255, 0, 0), 2)
            cv2.imshow(name, frame)
        key = cv2.waitKey(1)

Also available on GitHub

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
#include <iostream>

#include "depthai/depthai.hpp"
#include "utility.hpp"

static constexpr auto keyRotateDecr = 'z';
static constexpr auto keyRotateIncr = 'x';
static constexpr auto keyResizeInc = 'v';
static constexpr auto keyWarpTestCycle = 'c';

void printControls() {
    printf("\n=== Controls:\n");
    printf(" %c -rotated rectangle crop, decrease rate\n", keyRotateDecr);
    printf(" %c -rotated rectangle crop, increase rate\n", keyRotateIncr);
    printf(" %c -warp 4-point transform, cycle through modes\n", keyWarpTestCycle);
    printf(" %c -resize cropped region, or disable resize\n", keyResizeInc);
    printf(" h -print controls (help)\n");
}

static constexpr auto ROTATE_RATE_MAX = 5.0f;
static constexpr auto ROTATE_RATE_INC = 0.1f;

static constexpr auto RESIZE_MAX_W = 800;
static constexpr auto RESIZE_MAX_H = 600;
static constexpr auto RESIZE_FACTOR_MAX = 5;

/* The crop points are specified in clockwise order,
 * with first point mapped to output top-left, as:
 *   P0  ->  P1
 *    ^       v
 *   P3  <-  P2
 */
static const dai::Point2f P0 = {0, 0};  // top-left
static const dai::Point2f P1 = {1, 0};  // top-right
static const dai::Point2f P2 = {1, 1};  // bottom-right
static const dai::Point2f P3 = {0, 1};  // bottom-left
struct warpFourPointTest {
    std::vector<dai::Point2f> points;
    bool normalizedCoords;
    const char* description;
};

std::vector<warpFourPointTest> warpList = {
    //{{{  0,  0},{  1,  0},{  1,  1},{  0,  1}}, true, "passthrough"},
    //{{{  0,  0},{639,  0},{639,479},{  0,479}}, false,"passthrough (pixels)"},
    {{P0, P1, P2, P3}, true, "1. passthrough"},
    {{P3, P0, P1, P2}, true, "2. rotate 90"},
    {{P2, P3, P0, P1}, true, "3. rotate 180"},
    {{P1, P2, P3, P0}, true, "4. rotate 270"},
    {{P1, P0, P3, P2}, true, "5. horizontal mirror"},
    {{P3, P2, P1, P0}, true, "6. vertical flip"},
    {{{-0.1f, -0.1f}, {1.1f, -0.1f}, {1.1f, 1.1f}, {-0.1f, 1.1f}}, true, "7. add black borders"},
    {{{-0.3f, 0}, {1, 0}, {1.3f, 1}, {0, 1}}, true, "8. parallelogram transform"},
    {{{-0.2f, 0}, {1.8f, 0}, {1, 1}, {0, 1}}, true, "9. trapezoid transform"},
};

int main() {
    // Create pipeline
    dai::Pipeline pipeline;

    // Define sources and outputs
    auto camRgb = pipeline.create<dai::node::ColorCamera>();
    auto manip = pipeline.create<dai::node::ImageManip>();

    auto camOut = pipeline.create<dai::node::XLinkOut>();
    auto manipOut = pipeline.create<dai::node::XLinkOut>();
    auto manipCfg = pipeline.create<dai::node::XLinkIn>();

    camOut->setStreamName("preview");
    manipOut->setStreamName("manip");
    manipCfg->setStreamName("manipCfg");

    // Properties
    camRgb->setPreviewSize(640, 480);
    camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P);
    camRgb->setInterleaved(false);
    camRgb->setColorOrder(dai::ColorCameraProperties::ColorOrder::BGR);
    manip->setMaxOutputFrameSize(2000 * 1500 * 3);

    // Linking
    camRgb->preview.link(camOut->input);
    camRgb->preview.link(manip->inputImage);
    manip->out.link(manipOut->input);
    manipCfg->out.link(manip->inputConfig);

    // Connect to device and start pipeline
    dai::Device device(pipeline);

    // Create input & output queues
    auto qPreview = device.getOutputQueue("preview", 8, false);
    auto qManip = device.getOutputQueue("manip", 8, false);
    auto qManipCfg = device.getInputQueue("manipCfg");

    std::vector<decltype(qPreview)> frameQueues{qPreview, qManip};

    // keep processing data
    int key = -1;
    float angleDeg = 0;
    float rotateRate = 1.0;
    int resizeFactor = 0;
    int resizeX = 0;
    int resizeY = 0;
    bool testFourPt = false;
    int warpIdx = -1;

    printControls();

    while(key != 'q') {
        if(key >= 0) {
            printf("Pressed: %c | ", key);
            if(key == keyRotateDecr || key == keyRotateIncr) {
                if(key == keyRotateDecr) {
                    if(rotateRate > -ROTATE_RATE_MAX) rotateRate -= ROTATE_RATE_INC;
                } else if(key == keyRotateIncr) {
                    if(rotateRate < ROTATE_RATE_MAX) rotateRate += ROTATE_RATE_INC;
                }
                testFourPt = false;
                printf("Crop rotated rectangle, rate: %g degrees", rotateRate);
            } else if(key == keyResizeInc) {
                resizeFactor++;
                if(resizeFactor > RESIZE_FACTOR_MAX) {
                    resizeFactor = 0;
                    printf("Crop region not resized");
                } else {
                    resizeX = RESIZE_MAX_W / resizeFactor;
                    resizeY = RESIZE_MAX_H / resizeFactor;
                    printf("Crop region resized to: %d x %d", resizeX, resizeY);
                }
            } else if(key == keyWarpTestCycle) {
                resizeFactor = 0;  // Disable resizing initially
                warpIdx = (warpIdx + 1) % warpList.size();
                printf("Warp 4-point transform: %s", warpList[warpIdx].description);
                testFourPt = true;
            } else if(key == 'h') {
                printControls();
            }
            printf("\n");
        }

        // Send an updated config with continuous rotate, or after a key press
        if(key >= 0 || (!testFourPt && std::abs(rotateRate) > 0.0001)) {
            dai::ImageManipConfig cfg;
            if(testFourPt) {
                cfg.setWarpTransformFourPoints(warpList[warpIdx].points, warpList[warpIdx].normalizedCoords);
            } else {
                angleDeg += rotateRate;
                dai::RotatedRect rr = {{320, 240},  // center
                                       {640, 480},  //{400, 400}, // size
                                       angleDeg};
                cfg.setCropRotatedRect(rr, false);
            }
            if(resizeFactor > 0) {
                cfg.setResize(resizeX, resizeY);
            }
            // cfg.setWarpBorderFillColor(255, 0, 0);
            // cfg.setWarpBorderReplicatePixels();
            qManipCfg->send(cfg);
        }

        for(const auto& q : frameQueues) {
            auto img = q->get<dai::ImgFrame>();
            auto mat = toMat(img->getData(), img->getWidth(), img->getHeight(), 3, 1);
            cv::imshow(q->getName(), mat);
        }
        key = cv::waitKey(1);
    }
    return 0;
}

Got questions?

We’re always happy to help with code or other questions you might have.

Discord
Community Discord
forum
Discussion Forum
forum
Email Support