# Neural Network Multi-input Utilizes [NeuralNetwork](https://docs.luxonis.com/software-v3/depthai/depthai-components/nodes/neural_network.md) node to run a NN model which concatenates two input images and runs "inference" on the combined image. One of the input images is a static image sent from the host at startup (and it re-used for every frame), the other one is a live frame from the camera. ## Demo This example requires the DepthAI v3 API, see [installation instructions](https://docs.luxonis.com/software-v3/depthai.md). ## Pipeline ### examples/neural_network_multi_input.pipeline.json ```json {"pipeline": {"connections": [{"node1Id": 4, "node1Output": "out", "node1OutputGroup": "", "node2Id": 1, "node2Input": "image2", "node2InputGroup": "inputs"}, {"node1Id": 1, "node1Output": "out", "node1OutputGroup": "", "node2Id": 5, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 1, "node2Input": "image1", "node2InputGroup": "inputs"}], "globalProperties": {"calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1}, "nodes": [[5, {"alias": "", "id": 5, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 10, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_1_out"}}], [4, {"alias": "", "id": 4, "ioInfo": [[["", "out"], {"blocking": false, "group": "", "id": 9, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkIn", "parentId": -1, "properties": {"maxDataSize": 5242880, "numFrames": 8, "streamName": "__x_1_inputs_image2"}}], [1, {"alias": "", "id": 1, "ioInfo": [[["", "passthrough"], {"blocking": false, "group": "", "id": 8, "name": "passthrough", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "out"], {"blocking": false, "group": "", "id": 7, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["inputs", "image1"], {"blocking": false, "group": "inputs", "id": 6, "name": "image1", "queueSize": 1, "type": 3, "waitForMessage": true}], [["inputs", "image2"], {"blocking": false, "group": "inputs", "id": 5, "name": "image2", "queueSize": 1, "type": 3, "waitForMessage": false}], [["", "in"], {"blocking": true, "group": "", "id": 4, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": true}]], "logLevel": 3, "name": "NeuralNetwork", "parentId": -1, "properties": {"backend": "", "backendProperties": {}, "blobSize": 1876, "blobUri": "asset:__blob", "modelSource": 0, "modelUri": "", "numFrames": 8, "numNCEPerThread": 0, "numShavesPerThread": 0, "numThreads": 0}}], [0, {"alias": "", "id": 0, "ioInfo": [[["dynamicOutputs", "0"], {"blocking": false, "group": "dynamicOutputs", "id": 3, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "raw"], {"blocking": false, "group": "", "id": 2, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "mockIsp"], {"blocking": true, "group": "", "id": 1, "name": "mockIsp", "queueSize": 8, "type": 3, "waitForMessage": false}], [["", "inputControl"], {"blocking": true, "group": "", "id": 0, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "Camera", "parentId": -1, "properties": {"boardSocket": 0, "cameraName": "", "fps": -1.0, "imageOrientation": -1, "initialControl": {"aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": {"height": 0, "priority": 0, "width": 0, "x": 0, "y": 0}, "afRegion": {"height": 65535, "priority": 4294967295, "width": 65535, "x": 0, "y": 0}, "antiBandingMode": 255, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 255, "brightness": 0, "captureIntent": 255, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 255, "effectMode": 255, "enableHdr": false, "expCompensation": 0, "expManual": {"exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0}, "frameSyncMode": 255, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "miscControls": [], "saturation": 0, "sceneMode": 255, "sharpness": 0, "strobeConfig": {"activeLevel": 0, "enable": 255, "gpioNumber": 0}, "strobeTimings": {"durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0}, "wbColorTemp": 0}, "isp3aFps": 0, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [{"enableUndistortion": null, "fps": {"value": null}, "resizeMode": 0, "size": {"value": {"index": 0, "value": [256, 256]}}, "type": 7}], "resolutionHeight": -1, "resolutionWidth": -1}}]]}} ``` ## Source code #### Python ```python #!/usr/bin/env python3 import cv2 import depthai as dai import numpy as np from pathlib import Path # Get the absolute path of the current script's directory script_dir = Path(__file__).resolve().parent examplesRoot = (script_dir / Path('../')).resolve() # This resolves the parent directory correctly models = examplesRoot / 'models' tagImage = models / 'lenna.png' # Decode the image using OpenCV lenaImage = cv2.imread(str(tagImage.resolve())) lenaImage = cv2.resize(lenaImage, (256, 256)) lenaImage = np.array(lenaImage) device = dai.Device() platform = device.getPlatform() if(platform == dai.Platform.RVC2): daiType = dai.ImgFrame.Type.RGB888p elif(platform == dai.Platform.RVC4): daiType = dai.ImgFrame.Type.RGB888i else: raise RuntimeError("Platform not supported") daiLenaImage = dai.ImgFrame() daiLenaImage.setCvFrame(lenaImage, daiType) with dai.Pipeline(device) as pipeline: model = dai.NNModelDescription("depthai-test-models/simple-concatenate-model") model.platform = platform.name nnArchive = dai.NNArchive(dai.getModelFromZoo(model)) cam = pipeline.create(dai.node.Camera).build() camOut = cam.requestOutput((256,256), daiType) neuralNetwork = pipeline.create(dai.node.NeuralNetwork) neuralNetwork.setNNArchive(nnArchive) camOut.link(neuralNetwork.inputs["image1"]) lennaInputQueue = neuralNetwork.inputs["image2"].createInputQueue() # No need to send the second image everytime neuralNetwork.inputs["image2"].setReusePreviousMessage(True) qNNData = neuralNetwork.out.createOutputQueue() pipeline.start() lennaInputQueue.send(daiLenaImage) while pipeline.isRunning(): inNNData: dai.NNData = qNNData.get() tensor : np.ndarray = inNNData.getFirstTensor() # Drop the first dimension tensor = tensor.squeeze().astype(np.uint8) # Check the shape - in case 3 is not the last dimension, permute it to the last if tensor.shape[0] == 3: tensor = np.transpose(tensor, (1, 2, 0)) print(tensor.shape) cv2.imshow("Combined", tensor) key = cv2.waitKey(1) if key == ord('q'): break ``` #### C++ ```cpp #include #include #include #include "depthai/depthai.hpp" #include "depthai/modelzoo/Zoo.hpp" int main() { // Decode the image using OpenCV cv::Mat lenaImage = cv::imread(LENNA_PATH); cv::resize(lenaImage, lenaImage, cv::Size(256, 256)); // Create pipeline dai::Pipeline pipeline; // Create model description dai::NNModelDescription model; model.model = "depthai-test-models/simple-concatenate-model"; model.platform = pipeline.getDefaultDevice()->getPlatformAsString(); dai::NNArchive archive(dai::getModelFromZoo(model)); dai::ImgFrame::Type daiType; if(pipeline.getDefaultDevice()->getPlatform() == dai::Platform::RVC2) { daiType = dai::ImgFrame::Type::RGB888p; } else { daiType = dai::ImgFrame::Type::RGB888i; } // Create and set up nodes auto cam = pipeline.create()->build(); auto camOut = cam->requestOutput(std::make_pair(256, 256), daiType); auto neuralNetwork = pipeline.create(); neuralNetwork->setNNArchive(archive); camOut->link(neuralNetwork->inputs["image1"]); auto lennaInputQueue = neuralNetwork->inputs["image2"].createInputQueue(); // No need to send the second image everytime neuralNetwork->inputs["image2"].setReusePreviousMessage(true); auto qNNData = neuralNetwork->out.createOutputQueue(); // Stt pipeline pipeline.start(); // Create and set the image frame auto daiLenaImage = std::make_shared(); daiLenaImage->setCvFrame(lenaImage, daiType); lennaInputQueue->send(daiLenaImage); // Main loop while(pipeline.isRunning()) { auto inNNData = qNNData->get(); auto tensor = inNNData->getFirstTensor(); auto tensor_uint8 = xt::eval(xt::squeeze(xt::cast(tensor), 0)); cv::Mat output; if(tensor_uint8.shape()[0] == 3) { tensor_uint8 = xt::transpose(tensor_uint8, {1, 2, 0}); } output = cv::Mat(tensor_uint8.shape()[0], tensor_uint8.shape()[1], CV_8UC3); std::memcpy(output.data, tensor_uint8.data(), tensor_uint8.size()); cv::imshow("Combined", output); char key = cv::waitKey(1); if(key == 'q') { break; } } return 0; } ``` ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.