# Neural Network Utilizes [NeuralNetwork](https://docs.luxonis.com/software-v3/depthai/depthai-components/nodes/neural_network.md) node to run a NN model on the device. The example uses a pre-trained yolov6-nano that it downloads from [HubAI model zoo](https://models.luxonis.com/), but you could easily switch it with your own custom-trained model. To deploy a custom-trained model, you should first need to convert the model, either: * (Recommended) [Online via HubAI](https://docs.luxonis.com/cloud/hubai/model-registry/detailed-conversion.md) * [Offline via ModelConverter](https://docs.luxonis.com/software-v3/ai-inference/conversion/rvc-conversion/offline/modelconverter.md) - HubAI uses the same tool, but it's more user-friendly ## Demo When you run this example code, it will just print the output layer shape, and it won't do any model decoding / visualization: ```bash Received NN data: (1, 9, 16, 85) Received NN data: (1, 9, 16, 85) Received NN data: (1, 9, 16, 85) Received NN data: (1, 9, 16, 85) Received NN data: (1, 9, 16, 85) Received NN data: (1, 9, 16, 85) ``` Understanding the output of the Yolov6-nano: * Batch size 1, as it runs inference on every (single) frame * 9x16 grid, as YOLO divides the input image into a grid of size 9x16 cells * 85 (prediction) values per grid, 4 for bounding box, 1 for confidence score, and 80 for class probabilities (as it was trained on 80-class dataset - COCO) For this NN architecture we could just use the [DetectionNetwork](https://docs.luxonis.com/software-v3/depthai/depthai-components/nodes/detection_network.md) node, as it does both inference and decoding on the device (see [Detection network example](https://docs.luxonis.com/software-v3/depthai/examples/detection_network/detection_network.md) for more info). ### Deploying custom model If you're using .dlc, you can deploy it to OAK4 by editing [NeuralNetwork example](https://docs.luxonis.com/software-v3/depthai/examples/neural_network/neural_network.md) with the following snippet: ```python nn = pipeline.create(dai.node.NeuralNetwork) nn.setModelPath('my_model.dlc') nn.setBackend("snpe") # Specify SNPE NN backend. This usually gets set under the hood # Specify SNPE (RVC4) specific settings, like DSP runtime and NN performance profile nn.setBackendProperties({"runtime": "dsp", "performance_profile": "default"}) ``` Or, if you're using archive.tar.xz, you can edit that example with this snippet: ```python cam = pipeline.create(dai.node.Camera).build(socket) # If your nn model requires 640x640 input size (BGR): cam_out = cam.requestOutput((640, 640), dai.ImgFrame.Type.BGR888p) nn_archive = dai.NNArchive('./my_nn_archive.tar.xz') nn = pipeline.create(dai.node.NeuralNetwork).build(cam_out, nn_archive) ``` This example requires the DepthAI v3 API, see [installation instructions](https://docs.luxonis.com/software-v3/depthai.md). ## Pipeline ### examples/neural_network.pipeline.json ```json {"pipeline": {"connections": [{"node1Id": 1, "node1Output": "out", "node1OutputGroup": "", "node2Id": 2, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 1, "node2Input": "in", "node2InputGroup": ""}], "globalProperties": {"calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1}, "nodes": [[2, {"alias": "", "id": 2, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 7, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_1_out"}}], [1, {"alias": "", "id": 1, "ioInfo": [[["", "passthrough"], {"blocking": false, "group": "", "id": 6, "name": "passthrough", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "out"], {"blocking": false, "group": "", "id": 5, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "in"], {"blocking": true, "group": "", "id": 4, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": true}]], "logLevel": 3, "name": "NeuralNetwork", "parentId": -1, "properties": {"backend": "", "backendProperties": {}, "blobSize": 8689834, "blobUri": "asset:__blob", "modelSource": 0, "modelUri": "", "numFrames": 8, "numNCEPerThread": 0, "numShavesPerThread": 0, "numThreads": 0}}], [0, {"alias": "", "id": 0, "ioInfo": [[["dynamicOutputs", "0"], {"blocking": false, "group": "dynamicOutputs", "id": 3, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "raw"], {"blocking": false, "group": "", "id": 2, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "mockIsp"], {"blocking": true, "group": "", "id": 1, "name": "mockIsp", "queueSize": 8, "type": 3, "waitForMessage": false}], [["", "inputControl"], {"blocking": true, "group": "", "id": 0, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "Camera", "parentId": -1, "properties": {"boardSocket": 0, "cameraName": "", "fps": -1.0, "imageOrientation": -1, "initialControl": {"aeLockMode": false, "aeMaxExposureTimeUs": 100663297, "aeRegion": {"height": 0, "priority": 339768512, "width": 0, "x": 4163, "y": 1}, "afRegion": {"height": 0, "priority": 0, "width": 0, "x": 24915, "y": 0}, "antiBandingMode": 64, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 8, "brightness": 18, "captureIntent": 20, "chromaDenoise": 0, "cmdMask": 0, "contrast": 37, "controlMode": 83, "effectMode": 97, "enableHdr": false, "expCompensation": 118, "expManual": {"exposureTimeUs": 24915, "frameDurationUs": 24915, "sensitivityIso": 339768620}, "frameSyncMode": 0, "lensPosAutoInfinity": 64, "lensPosAutoMacro": 117, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "miscControls": [], "saturation": 27, "sceneMode": 117, "sharpness": 0, "strobeConfig": {"activeLevel": 104, "enable": 0, "gpioNumber": 123}, "strobeTimings": {"durationUs": 8, "exposureBeginOffsetUs": 24915, "exposureEndOffsetUs": 17}, "wbColorTemp": 0}, "isp3aFps": 0, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [{"enableUndistortion": null, "fps": {"value": null}, "resizeMode": 0, "size": {"value": {"index": 0, "value": [416, 416]}}, "type": 8}], "resolutionHeight": -1, "resolutionWidth": -1}}]]}} ``` ## Source code #### Python ```python #!/usr/bin/env python3 import cv2 import depthai as dai import numpy as np import time # Create pipeline with dai.Pipeline() as pipeline: cameraNode = pipeline.create(dai.node.Camera).build() # Longer form - useful in case of a local NNArchive # modelDescription = dai.NNModelDescription("yolov6-nano", platform=pipeline.getDefaultDevice().getPlatformAsString()) # archive = dai.NNArchive(dai.getModelFromZoo(modelDescription)) # neuralNetwork = pipeline.create(dai.node.NeuralNetwork).build(cameraNode, archive) neuralNetwork = pipeline.create(dai.node.NeuralNetwork).build(cameraNode, dai.NNModelDescription("yolov6-nano")) qNNData = neuralNetwork.out.createOutputQueue() pipeline.start() while pipeline.isRunning(): inNNData: dai.NNData = qNNData.get() tensor = inNNData.getFirstTensor() assert(isinstance(tensor, np.ndarray)) print(f"Received NN data: {tensor.shape}") ``` #### C++ ```cpp #include #include #include "depthai/depthai.hpp" int main() { // Create pipeline dai::Pipeline pipeline; // Create nodes auto cameraNode = pipeline.create(); auto neuralNetwork = pipeline.create(); // Build nodes cameraNode->build(); // Longer form - useful in case of a local NNArchive // auto modelDescription = dai::NNModelDescription("yolov6-nano", pipeline.getDefaultDevice()->getPlatformAsString()); // auto archive = dai::NNArchive(dai::getModelFromZoo(modelDescription)); // neuralNetwork->build(cameraNode, archive); dai::NNModelDescription modelDescription; modelDescription.model = "yolov6-nano"; modelDescription.platform = pipeline.getDefaultDevice()->getPlatformAsString(); neuralNetwork->build(cameraNode, modelDescription); // Create output queue auto qNNData = neuralNetwork->out.createOutputQueue(); // Start pipeline pipeline.start(); // Main loop while(pipeline.isRunning()) { auto inNNData = qNNData->get(); auto tensor = inNNData->getFirstTensor(); std::cout << "Received NN data: " << tensor.shape()[0] << "x" << tensor.shape()[1] << std::endl; } return 0; } ``` ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.