# Interactive Warp Mesh

This example shows usage of [Warp](https://docs.luxonis.com/software/depthai-components/nodes/warp.md) node to warp the input
image frame. It let's you interactively change the mesh points to warp the image. After changing the points, user has to press r
to restart the pipeline and apply the changes.

User-defined arguments:

 * --mesh_dims - Mesh dimensions (default: 4x4).
 * --resolution - Resolution of the input image (default: 512x512). Width must be divisible by 16.
 * --random - To generate random mesh points (disabled by default).

Originally developed by [geaxgx](https://github.com/geaxgx).

## Setup

Please run the [install script](https://github.com/luxonis/depthai-python/blob/main/examples/install_requirements.py) to download
all required dependencies. Please note that this script must be ran from git context, so you have to download the
[depthai-python](https://github.com/luxonis/depthai-python) repository first and then run the script

```bash
git clone https://github.com/luxonis/depthai-python.git
cd depthai-python/examples
python3 install_requirements.py
```

For additional information, please follow the [installation guide](https://docs.luxonis.com/software/depthai/manual-install.md).

## Demo

## Source code

#### Python

```python
#!/usr/bin/env python3
import cv2
import depthai as dai
import numpy as np
import argparse
import re
import sys
from random import randint

parser = argparse.ArgumentParser()
parser.add_argument("-m", "--mesh_dims", type=str, default="4x4", help="mesh dimensions widthxheight (default=%(default)s)")
parser.add_argument("-r", "--resolution", type=str, default="512x512", help="preview resolution (default=%(default)s)")
parser.add_argument("-rnd", "--random", action="store_true", help="Generate random initial mesh")
args = parser.parse_args()

# mesh dimensions
match = re.search(r'.*?(\d+)x(\d+).*', args.mesh_dims)
if not match:
    raise Exception(f"Mesh dimensions format incorrect '{args.resolution}'!")
mesh_w = int(match.group(1))
mesh_h = int(match.group(2))

# Preview resolution
match = re.search(r'.*?(\d+)x(\d+).*', args.resolution)
if not match:
    raise Exception(f"Resolution format incorrect '{args.resolution}'!")
preview_w = int(match.group(1))
preview_h = int(match.group(2))
if preview_w % 16 != 0:
    raise Exception(f"Preview width must be a multiple of 16!")

# Create an initial mesh (optionally random) of dimension mesh_w x mesh_h
first_point_x = int(preview_w / 10)
between_points_x = int(4 * preview_w / (5 * (mesh_w - 1)))
first_point_y = int(preview_h / 10)
between_points_y = int(4 * preview_h / (5 * (mesh_h - 1)))
if args.random:
    max_rnd_x = int(between_points_x / 4)
    max_rnd_y = int(between_points_y / 4)
mesh = []
for i in range(mesh_h):
    for j in range(mesh_w):
        x = first_point_x + j * between_points_x
        y = first_point_y + i * between_points_y
        if args.random:
            rnd_x = randint(-max_rnd_x, max_rnd_x)
            if x + rnd_x > 0 and x + rnd_x < preview_w:
                x += rnd_x
            rnd_y = randint(-max_rnd_y, max_rnd_y)
            if y + rnd_y > 0 and y + rnd_y < preview_h:
                y += rnd_y
        mesh.append((x, y))

def create_pipeline(mesh):
    print(mesh)
    # Create pipeline
    pipeline = dai.Pipeline()

    camRgb = pipeline.create(dai.node.ColorCamera)
    camRgb.setPreviewSize(preview_w, preview_h)
    camRgb.setInterleaved(False)
    width = camRgb.getPreviewWidth()
    height = camRgb.getPreviewHeight()

    # Output source
    xout_source = pipeline.create(dai.node.XLinkOut)
    xout_source.setStreamName('source')
    camRgb.preview.link(xout_source.input)
    # Warp source frame
    warp = pipeline.create(dai.node.Warp)
    warp.setWarpMesh(mesh, mesh_w, mesh_h)
    warp.setOutputSize(width, height)
    warp.setMaxOutputFrameSize(width * height * 3)
    camRgb.preview.link(warp.inputImage)

    warp.setHwIds([1])
    warp.setInterpolation(dai.Interpolation.NEAREST_NEIGHBOR)
    # Output warped
    xout_warped = pipeline.create(dai.node.XLinkOut)
    xout_warped.setStreamName('warped')
    warp.out.link(xout_warped.input)
    return pipeline

point_selected = None

def mouse_callback(event, x, y, flags, param):
    global mesh, point_selected, mesh_changed
    if event == cv2.EVENT_LBUTTONDOWN:
        if point_selected is None:
            # Which point is selected ?
            min_dist = 100

            for i in range(len(mesh)):
                dist = np.linalg.norm((x - mesh[i][0], y - mesh[i][1]))
                if dist < 20 and dist < min_dist:
                    min_dist = dist
                    point_selected = i
            if point_selected is not None:
                mesh[point_selected] = (x, y)
                mesh_changed = True

    elif event == cv2.EVENT_LBUTTONUP:
        point_selected = None
    elif event == cv2.EVENT_MOUSEMOVE:
        if point_selected is not None:
            mesh[point_selected] = (x, y)
            mesh_changed = True

cv2.namedWindow("Source")
cv2.setMouseCallback("Source", mouse_callback)

running = True

print("Use your mouse to modify the mesh by clicking/moving points of the mesh in the Source window")
print("Then press 'r' key to restart the device/pipeline")
while running:
    pipeline = create_pipeline(mesh)
    # Connect to device and start pipeline
    with dai.Device(pipeline) as device:
        print("Starting device")
        # Output queue will be used to get the rgb frames from the output defined above
        q_source = device.getOutputQueue(name="source", maxSize=4, blocking=False)
        q_warped = device.getOutputQueue(name="warped", maxSize=4, blocking=False)

        restart_device = False
        mesh_changed = False
        while not restart_device:
            in0 = q_source.get()
            if in0 is not None:
                source = in0.getCvFrame()
                color = (0, 0,255) if mesh_changed else (0,255,0)
                for i in range(len(mesh)):
                    cv2.circle(source, (mesh[i][0], mesh[i][1]), 4, color, -1)
                    if i % mesh_w != mesh_w -1:
                        cv2.line(source, (mesh[i][0], mesh[i][1]), (mesh[i+1][0], mesh[i+1][1]), color, 2)
                    if i + mesh_w < len(mesh):
                        cv2.line(source, (mesh[i][0], mesh[i][1]), (mesh[i+mesh_w][0], mesh[i+mesh_w][1]), color, 2)
                cv2.imshow("Source", source)

            in1 = q_warped.get()
            if in1 is not None:
                cv2.imshow("Warped", in1.getCvFrame())

            key = cv2.waitKey(1)
            if key == ord('r'): # Restart the device if mesh has changed
                if mesh_changed:
                    print("Restart requested...")
                    mesh_changed = False
                    restart_device = True
            elif key == 27 or key == ord('q'): # Exit
                running = False
                break
```

#### C++

WIP

## Pipeline

### examples/warp_mesh_interactive.pipeline.json

```json
{
  "pipeline": {
    "connections": [
      {
        "node1Id": 0,
        "node1Output": "preview",
        "node1OutputGroup": "",
        "node2Id": 1,
        "node2Input": "in",
        "node2InputGroup": ""
      },
      {
        "node1Id": 0,
        "node1Output": "preview",
        "node1OutputGroup": "",
        "node2Id": 2,
        "node2Input": "inputImage",
        "node2InputGroup": ""
      },
      {
        "node1Id": 2,
        "node1Output": "out",
        "node1OutputGroup": "",
        "node2Id": 3,
        "node2Input": "in",
        "node2InputGroup": ""
      }
    ],
    "globalProperties": {
      "calibData": null,
      "cameraTuningBlobSize": null,
      "cameraTuningBlobUri": "",
      "leonCssFrequencyHz": 700000000.0,
      "leonMssFrequencyHz": 700000000.0,
      "pipelineName": null,
      "pipelineVersion": null,
      "sippBufferSize": 18432,
      "sippDmaBufferSize": 16384,
      "xlinkChunkSize": -1
    },
    "nodes": [
      [
        0,
        {
          "id": 0,
          "ioInfo": [
            [
              [
                "",
                "inputConfig"
              ],
              {
                "blocking": false,
                "group": "",
                "id": 1,
                "name": "inputConfig",
                "queueSize": 8,
                "type": 3,
                "waitForMessage": false
              }
            ],
            [
              [
                "",
                "raw"
              ],
              {
                "blocking": false,
                "group": "",
                "id": 6,
                "name": "raw",
                "queueSize": 8,
                "type": 0,
                "waitForMessage": false
              }
            ],
            [
              [
                "",
                "still"
              ],
              {
                "blocking": false,
                "group": "",
                "id": 7,
                "name": "still",
                "queueSize": 8,
                "type": 0,
                "waitForMessage": false
              }
            ],
            [
              [
                "",
                "inputControl"
              ],
              {
                "blocking": true,
                "group": "",
                "id": 2,
                "name": "inputControl",
                "queueSize": 8,
                "type": 3,
                "waitForMessage": false
              }
            ],
            [
              [
                "",
                "video"
              ],
              {
                "blocking": false,
                "group": "",
                "id": 3,
                "name": "video",
                "queueSize": 8,
                "type": 0,
                "waitForMessage": false
              }
            ],
            [
              [
                "",
                "isp"
              ],
              {
                "blocking": false,
                "group": "",
                "id": 4,
                "name": "isp",
                "queueSize": 8,
                "type": 0,
                "waitForMessage": false
              }
            ],
            [
              [
                "",
                "preview"
              ],
              {
                "blocking": false,
                "group": "",
                "id": 5,
                "name": "preview",
                "queueSize": 8,
                "type": 0,
                "waitForMessage": false
              }
            ],
            [
              [
                "",
                "frameEvent"
              ],
              {
                "blocking": false,
                "group": "",
                "id": 8,
                "name": "frameEvent",
                "queueSize": 8,
                "type": 0,
                "waitForMessage": false
              }
            ]
          ],
          "name": "ColorCamera",
          "properties": {
            "boardSocket": -1,
            "cameraName": "",
            "colorOrder": 0,
            "fp16": false,
            "fps": 30.0,
            "imageOrientation": -1,
            "initialControl": {
              "aeLockMode": false,
              "aeMaxExposureTimeUs": 0,
              "aeRegion": {
                "height": 0,
                "priority": 0,
                "width": 0,
                "x": 0,
                "y": 0
              },
              "afRegion": {
                "height": 0,
                "priority": 0,
                "width": 0,
                "x": 0,
                "y": 0
              },
              "antiBandingMode": 0,
              "autoFocusMode": 3,
              "awbLockMode": false,
              "awbMode": 0,
              "brightness": 0,
              "captureIntent": 0,
              "chromaDenoise": 0,
              "cmdMask": 0,
              "contrast": 0,
              "controlMode": 0,
              "effectMode": 0,
              "expCompensation": 0,
              "expManual": {
                "exposureTimeUs": 0,
                "frameDurationUs": 0,
                "sensitivityIso": 0
              },
              "frameSyncMode": 0,
              "lensPosAutoInfinity": 0,
              "lensPosAutoMacro": 0,
              "lensPosition": 0,
              "lensPositionRaw": 0.0,
              "lowPowerNumFramesBurst": 0,
              "lowPowerNumFramesDiscard": 0,
              "lumaDenoise": 0,
              "saturation": 0,
              "sceneMode": 0,
              "sharpness": 0,
              "strobeConfig": {
                "activeLevel": 0,
                "enable": 0,
                "gpioNumber": 0
              },
              "strobeTimings": {
                "durationUs": 0,
                "exposureBeginOffsetUs": 0,
                "exposureEndOffsetUs": 0
              },
              "wbColorTemp": 0
            },
            "interleaved": false,
            "isp3aFps": 0,
            "ispScale": {
              "horizDenominator": 0,
              "horizNumerator": 0,
              "vertDenominator": 0,
              "vertNumerator": 0
            },
            "numFramesPoolIsp": 3,
            "numFramesPoolPreview": 4,
            "numFramesPoolRaw": 3,
            "numFramesPoolStill": 4,
            "numFramesPoolVideo": 4,
            "previewHeight": 512,
            "previewKeepAspectRatio": true,
            "previewWidth": 512,
            "rawPacked": null,
            "resolution": 0,
            "sensorCropX": -1.0,
            "sensorCropY": -1.0,
            "stillHeight": -1,
            "stillWidth": -1,
            "videoHeight": -1,
            "videoWidth": -1
          }
        }
      ],
      [
        1,
        {
          "id": 1,
          "ioInfo": [
            [
              [
                "",
                "in"
              ],
              {
                "blocking": true,
                "group": "",
                "id": 9,
                "name": "in",
                "queueSize": 8,
                "type": 3,
                "waitForMessage": true
              }
            ]
          ],
          "name": "XLinkOut",
          "properties": {
            "maxFpsLimit": -1.0,
            "metadataOnly": false,
            "streamName": "source"
          }
        }
      ],
      [
        2,
        {
          "id": 2,
          "ioInfo": [
            [
              [
                "",
                "inputImage"
              ],
              {
                "blocking": true,
                "group": "",
                "id": 10,
                "name": "inputImage",
                "queueSize": 8,
                "type": 3,
                "waitForMessage": true
              }
            ],
            [
              [
                "",
                "out"
              ],
              {
                "blocking": false,
                "group": "",
                "id": 11,
                "name": "out",
                "queueSize": 8,
                "type": 0,
                "waitForMessage": false
              }
            ]
          ],
          "name": "Warp",
          "properties": {
            "interpolation": 2,
            "meshHeight": 4,
            "meshUri": "asset:mesh",
            "meshWidth": 4,
            "numFramesPool": 4,
            "outputFrameSize": 786432,
            "outputHeight": 512,
            "outputWidth": 512,
            "warpHwIds": [
              1
            ]
          }
        }
      ],
      [
        3,
        {
          "id": 3,
          "ioInfo": [
            [
              [
                "",
                "in"
              ],
              {
                "blocking": true,
                "group": "",
                "id": 12,
                "name": "in",
                "queueSize": 8,
                "type": 3,
                "waitForMessage": true
              }
            ]
          ],
          "name": "XLinkOut",
          "properties": {
            "maxFpsLimit": -1.0,
            "metadataOnly": false,
            "streamName": "warped"
          }
        }
      ]
    ]
  }
}
```

### Need assistance?

Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.