# Multi-Input Frame Concatenation Example concatenates all 3 inputs with a simple custom model created with PyTorch ([link here](https://github.com/luxonis/oak-examples/blob/master/gen2-custom-models/generate_model/pytorch_concat.py), [tutorial here](https://docs.luxonis.com/software/depthai/on-device-programming.md)). It uses [NeuralNetwork](https://docs.luxonis.com/software/depthai-components/nodes/neural_network.md)'s multiple input feature and links all 3 camera streams directly to the NeuralNetwork node. ## Demo ## Setup Please run the [install script](https://github.com/luxonis/depthai-python/blob/main/examples/install_requirements.py) to download all required dependencies. Please note that this script must be ran from git context, so you have to download the [depthai-python](https://github.com/luxonis/depthai-python) repository first and then run the script ```bash git clone https://github.com/luxonis/depthai-python.git cd depthai-python/examples python3 install_requirements.py ``` For additional information, please follow the [installation guide](https://docs.luxonis.com/software/depthai/manual-install.md). ## Source code #### Python ```python #!/usr/bin/env python3 from pathlib import Path import sys import numpy as np import cv2 import depthai as dai SHAPE = 300 # Get argument first nnPath = str((Path(__file__).parent / Path('../models/concat_openvino_2021.4_6shave.blob')).resolve().absolute()) if len(sys.argv) > 1: nnPath = sys.argv[1] if not Path(nnPath).exists(): import sys raise FileNotFoundError(f'Required file/s not found, please run "{sys.executable} install_requirements.py"') p = dai.Pipeline() p.setOpenVINOVersion(dai.OpenVINO.VERSION_2021_4) camRgb = p.createColorCamera() camRgb.setPreviewSize(SHAPE, SHAPE) camRgb.setInterleaved(False) camRgb.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR) def create_mono(p, socket): mono = p.create(dai.node.MonoCamera) mono.setBoardSocket(socket) mono.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P) # ImageManip for cropping (face detection NN requires input image of 300x300) and to change frame type manip = p.create(dai.node.ImageManip) manip.initialConfig.setResize(300, 300) manip.initialConfig.setFrameType(dai.RawImgFrame.Type.BGR888p) mono.out.link(manip.inputImage) return manip.out # NN that detects faces in the image nn = p.createNeuralNetwork() nn.setBlobPath(nnPath) nn.setNumInferenceThreads(2) camRgb.preview.link(nn.inputs['img2']) create_mono(p, dai.CameraBoardSocket.CAM_B).link(nn.inputs['img1']) create_mono(p, dai.CameraBoardSocket.CAM_C).link(nn.inputs['img3']) # Send bouding box from the NN to the host via XLink nn_xout = p.createXLinkOut() nn_xout.setStreamName("nn") nn.out.link(nn_xout.input) # Pipeline is defined, now we can connect to the device with dai.Device(p) as device: qNn = device.getOutputQueue(name="nn", maxSize=4, blocking=False) shape = (3, SHAPE, SHAPE * 3) while True: inNn = np.array(qNn.get().getFirstLayerFp16()) # Planar INT8 frame frame = inNn.reshape(shape).astype(np.uint8).transpose(1, 2, 0) cv2.imshow("Concat", frame) if cv2.waitKey(1) == ord('q'): break ``` #### C++ ```cpp #include #include #include // Inludes common necessary includes for development using depthai library #include "depthai/depthai.hpp" #include "utility.hpp" int main(int argc, char** argv) { using namespace std; // Default blob path provided by Hunter private data download // Applicable for easier example usage only std::string nnPath(BLOB_PATH); // If path to blob specified, use that if(argc > 1) { nnPath = std::string(argv[1]); } // Print which blob we are using printf("Using blob at path: %s\n", nnPath.c_str()); // Create pipeline dai::Pipeline pipeline; pipeline.setOpenVINOVersion(dai::OpenVINO::Version::VERSION_2021_4); // Define sources and outputs auto camRgb = pipeline.create(); camRgb->setPreviewSize(300, 300); // NN input camRgb->setInterleaved(false); camRgb->setColorOrder(dai::ColorCameraProperties::ColorOrder::BGR); auto right = pipeline.create(); right->setCamera("right"); right->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P); auto manipRight = pipeline.create(); manipRight->initialConfig.setResize(300, 300); manipRight->initialConfig.setFrameType(dai::ImgFrame::Type::BGR888p); right->out.link(manipRight->inputImage); auto left = pipeline.create(); left->setCamera("left"); left->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P); auto manipLeft = pipeline.create(); manipLeft->initialConfig.setResize(300, 300); manipLeft->initialConfig.setFrameType(dai::ImgFrame::Type::BGR888p); left->out.link(manipLeft->inputImage); auto nn = pipeline.create(); nn->setBlobPath(nnPath); nn->setNumInferenceThreads(2); manipLeft->out.link(nn->inputs["img1"]); camRgb->preview.link(nn->inputs["img2"]); manipRight->out.link(nn->inputs["img3"]); auto xout = pipeline.create(); xout->setStreamName("nn"); nn->out.link(xout->input); // Connect to device and start pipeline dai::Device device(pipeline); // Output queues will be used to get the rgb frames and nn data from the outputs defined above auto qNn = device.getOutputQueue("nn", 4, false); while(true) { auto inNn = qNn->get(); cv::imshow("Concat", fromPlanarFp16(inNn->getFirstLayerFp16(), 900, 300)); int key = cv::waitKey(1); if(key == 'q' || key == 'Q') { return 0; } } return 0; } ``` ## Pipeline ### examples/concat_multi_input.pipeline.json ```json { "pipeline": { "connections": [ { "node1Id": 5, "node1Output": "out", "node1OutputGroup": "", "node2Id": 1, "node2Input": "img3", "node2InputGroup": "inputs" }, { "node1Id": 3, "node1Output": "out", "node1OutputGroup": "", "node2Id": 1, "node2Input": "img1", "node2InputGroup": "inputs" }, { "node1Id": 0, "node1Output": "preview", "node1OutputGroup": "", "node2Id": 1, "node2Input": "img2", "node2InputGroup": "inputs" }, { "node1Id": 2, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "inputImage", "node2InputGroup": "" }, { "node1Id": 4, "node1Output": "out", "node1OutputGroup": "", "node2Id": 5, "node2Input": "inputImage", "node2InputGroup": "" }, { "node1Id": 1, "node1Output": "out", "node1OutputGroup": "", "node2Id": 6, "node2Input": "in", "node2InputGroup": "" } ], "globalProperties": { "calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1 }, "nodes": [ [ 0, { "id": 0, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 1, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 6, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "still" ], { "blocking": false, "group": "", "id": 7, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 2, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "video" ], { "blocking": false, "group": "", "id": 3, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "isp" ], { "blocking": false, "group": "", "id": 4, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "preview" ], { "blocking": false, "group": "", "id": 5, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 8, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ColorCamera", "properties": { "boardSocket": -1, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 30.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": false, "isp3aFps": 0, "ispScale": { "horizDenominator": 0, "horizNumerator": 0, "vertDenominator": 0, "vertNumerator": 0 }, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 300, "previewKeepAspectRatio": true, "previewWidth": 300, "rawPacked": null, "resolution": 0, "sensorCropX": -1.0, "sensorCropY": -1.0, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1 } } ], [ 1, { "id": 1, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 9, "name": "in", "queueSize": 5, "type": 3, "waitForMessage": true } ], [ [ "inputs", "img2" ], { "blocking": false, "group": "inputs", "id": 10, "name": "img2", "queueSize": 1, "type": 3, "waitForMessage": true } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 13, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "inputs", "img1" ], { "blocking": false, "group": "inputs", "id": 11, "name": "img1", "queueSize": 1, "type": 3, "waitForMessage": true } ], [ [ "inputs", "img3" ], { "blocking": false, "group": "inputs", "id": 12, "name": "img3", "queueSize": 1, "type": 3, "waitForMessage": true } ], [ [ "", "passthrough" ], { "blocking": false, "group": "", "id": 14, "name": "passthrough", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "NeuralNetwork", "properties": { "blobSize": 1024, "blobUri": "asset:__blob", "numFrames": 8, "numNCEPerThread": 0, "numThreads": 2 } } ], [ 2, { "id": 2, "ioInfo": [ [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 15, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 16, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 17, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 18, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "MonoCamera", "properties": { "boardSocket": 1, "cameraName": "", "fps": 30.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "isp3aFps": 0, "numFramesPool": 3, "numFramesPoolRaw": 3, "rawPacked": null, "resolution": 2 } } ], [ 3, { "id": 3, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": true, "group": "", "id": 19, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "inputImage" ], { "blocking": true, "group": "", "id": 20, "name": "inputImage", "queueSize": 8, "type": 3, "waitForMessage": true } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 21, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ImageManip", "properties": { "initialConfig": { "cropConfig": { "cropRatio": 1.0, "cropRect": { "xmax": 0.0, "xmin": 0.0, "ymax": 0.0, "ymin": 0.0 }, "cropRotatedRect": { "angle": 0.0, "center": { "x": 0.0, "y": 0.0 }, "size": { "height": 0.0, "width": 0.0 } }, "enableCenterCropRectangle": false, "enableRotatedRect": false, "normalizedCoords": true, "widthHeightAspectRatio": 1.0 }, "enableCrop": false, "enableFormat": true, "enableResize": true, "formatConfig": { "colormap": 0, "colormapMax": 255, "colormapMin": 0, "flipHorizontal": false, "flipVertical": false, "type": 8 }, "interpolation": -1, "resizeConfig": { "bgBlue": 0, "bgGreen": 0, "bgRed": 0, "enableRotation": false, "enableWarp4pt": false, "enableWarpMatrix": false, "height": 300, "keepAspectRatio": true, "lockAspectRatioFill": false, "normalizedCoords": true, "rotationAngleDeg": 0.0, "warpBorderReplicate": false, "warpFourPoints": [], "warpMatrix3x3": [], "width": 300 }, "reusePreviousImage": false, "skipCurrentImage": false }, "meshHeight": 0, "meshUri": "", "meshWidth": 0, "numFramesPool": 4, "outputFrameSize": 1048576 } } ], [ 4, { "id": 4, "ioInfo": [ [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 22, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 23, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 24, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 25, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "MonoCamera", "properties": { "boardSocket": 2, "cameraName": "", "fps": 30.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "isp3aFps": 0, "numFramesPool": 3, "numFramesPoolRaw": 3, "rawPacked": null, "resolution": 2 } } ], [ 5, { "id": 5, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": true, "group": "", "id": 26, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "inputImage" ], { "blocking": true, "group": "", "id": 27, "name": "inputImage", "queueSize": 8, "type": 3, "waitForMessage": true } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 28, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ImageManip", "properties": { "initialConfig": { "cropConfig": { "cropRatio": 1.0, "cropRect": { "xmax": 0.0, "xmin": 0.0, "ymax": 0.0, "ymin": 0.0 }, "cropRotatedRect": { "angle": 0.0, "center": { "x": 0.0, "y": 0.0 }, "size": { "height": 0.0, "width": 0.0 } }, "enableCenterCropRectangle": false, "enableRotatedRect": false, "normalizedCoords": true, "widthHeightAspectRatio": 1.0 }, "enableCrop": false, "enableFormat": true, "enableResize": true, "formatConfig": { "colormap": 0, "colormapMax": 255, "colormapMin": 0, "flipHorizontal": false, "flipVertical": false, "type": 8 }, "interpolation": -1, "resizeConfig": { "bgBlue": 0, "bgGreen": 0, "bgRed": 0, "enableRotation": false, "enableWarp4pt": false, "enableWarpMatrix": false, "height": 300, "keepAspectRatio": true, "lockAspectRatioFill": false, "normalizedCoords": true, "rotationAngleDeg": 0.0, "warpBorderReplicate": false, "warpFourPoints": [], "warpMatrix3x3": [], "width": 300 }, "reusePreviousImage": false, "skipCurrentImage": false }, "meshHeight": 0, "meshUri": "", "meshWidth": 0, "numFramesPool": 4, "outputFrameSize": 1048576 } } ], [ 6, { "id": 6, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 29, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "nn" } } ] ] } } ``` ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.