# Frame Normalization This example shows how you can normalize a frame before sending it to another neural network. Many neural network models require frames with RGB values (pixels) in range between -0.5 to 0.5. [ColorCamera](https://docs.luxonis.com/software/depthai-components/nodes/color_camera.md)'s preview outputs values between 0 and 255. Simple custom model, created with PyTorch ([link here](https://github.com/luxonis/oak-examples/blob/master/gen2-custom-models/generate_model/pytorch_normalize.py), [tutorial here](https://docs.luxonis.com/software/ai-inference/conversion.md)), allows users to specify mean and scale factors that will be applied to all frame values (pixels). ```python output = (input - mean) / scale ``` On the host, values are converted back to 0-255, so they can be displayed by OpenCV. > This is just a demo, for normalization you should use OpenVINO's > [model optimizer](https://docs.openvinotoolkit.org/latest/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html) > arguments > `--mean_values` > and > `--scale_values` > . ## Setup Please run the [install script](https://github.com/luxonis/depthai-python/blob/main/examples/install_requirements.py) to download all required dependencies. Please note that this script must be ran from git context, so you have to download the [depthai-python](https://github.com/luxonis/depthai-python) repository first and then run the script ```bash git clone https://github.com/luxonis/depthai-python.git cd depthai-python/examples python3 install_requirements.py ``` For additional information, please follow the [installation guide](https://docs.luxonis.com/software/depthai/manual-install.md). ## Source code #### Python ```python #!/usr/bin/env python3 from pathlib import Path import sys import numpy as np import cv2 import depthai as dai SHAPE = 300 # Get argument first nnPath = str((Path(__file__).parent / Path('../models/normalize_openvino_2021.4_4shave.blob')).resolve().absolute()) if len(sys.argv) > 1: nnPath = sys.argv[1] if not Path(nnPath).exists(): import sys raise FileNotFoundError(f'Required file/s not found, please run "{sys.executable} install_requirements.py"') p = dai.Pipeline() p.setOpenVINOVersion(dai.OpenVINO.VERSION_2021_4) camRgb = p.createColorCamera() # Model expects values in FP16, as we have compiled it with `-ip FP16` camRgb.setFp16(True) camRgb.setInterleaved(False) camRgb.setPreviewSize(SHAPE, SHAPE) nn = p.createNeuralNetwork() nn.setBlobPath(nnPath) nn.setNumInferenceThreads(2) script = p.create(dai.node.Script) script.setScript(""" # Run script only once. We could also send these values from host. # Model formula: # output = (input - mean) / scale # This configuration will subtract all frame values (pixels) by 127.5 # 0.0 .. 255.0 -> -127.5 .. 127.5 data = NNData(2) data.setLayer("mean", [127.5]) node.io['mean'].send(data) # This configuration will divide all frame values (pixels) by 255.0 # -127.5 .. 127.5 -> -0.5 .. 0.5 data = NNData(2) data.setLayer("scale", [255.0]) node.io['scale'].send(data) """) # Re-use the initial values for multiplier/addend script.outputs['mean'].link(nn.inputs['mean']) nn.inputs['mean'].setWaitForMessage(False) script.outputs['scale'].link(nn.inputs['scale']) nn.inputs['scale'].setWaitForMessage(False) # Always wait for the new frame before starting inference camRgb.preview.link(nn.inputs['frame']) # Send normalized frame values to host nn_xout = p.createXLinkOut() nn_xout.setStreamName("nn") nn.out.link(nn_xout.input) # Pipeline is defined, now we can connect to the device with dai.Device(p) as device: qNn = device.getOutputQueue(name="nn", maxSize=4, blocking=False) shape = (3, SHAPE, SHAPE) while True: inNn = np.array(qNn.get().getData()) # Get back the frame. It's currently normalized to -0.5 - 0.5 frame = inNn.view(np.float16).reshape(shape).transpose(1, 2, 0) # To get original frame back (0-255), we add multiply all frame values (pixels) by 255 and then add 127.5 to them frame = (frame * 255.0 + 127.5).astype(np.uint8) # Show the initial frame cv2.imshow("Original frame", frame) if cv2.waitKey(1) == ord('q'): break ``` #### C++ ```cpp #include #include #include // Inludes common necessary includes for development using depthai library #include "depthai/depthai.hpp" #include "utility.hpp" int main(int argc, char** argv) { using namespace std; // Default blob path provided by Hunter private data download // Applicable for easier example usage only std::string nnPath(BLOB_PATH); // If path to blob specified, use that if(argc > 1) { nnPath = std::string(argv[1]); } // Print which blob we are using printf("Using blob at path: %s\n", nnPath.c_str()); // Create pipeline dai::Pipeline pipeline; pipeline.setOpenVINOVersion(dai::OpenVINO::Version::VERSION_2021_4); // Define sources and outputs auto camRgb = pipeline.create(); // Model expects values in FP16, as we have compiled it with `-ip FP16` camRgb->setFp16(true); camRgb->setInterleaved(false); camRgb->setPreviewSize(300, 300); // NN input auto nn = pipeline.create(); nn->setBlobPath(nnPath); nn->setNumInferenceThreads(2); auto script = pipeline.create(); script->setScript(R"( # Run script only once # Model formula: # output = (input - mean) / scale # This configuration will subtract all frame values (pixels) by 127.5 # 0.0 .. 255.0 -> -127.5 .. 127.5 data = NNData(2) data.setLayer("mean", [127.5]) node.io['mean'].send(data) # This configuration will divide all frame values (pixels) by 255.0 # -127.5 .. 127.5 -> -0.5 .. 0.5 data = NNData(2) data.setLayer("scale", [255.0]) node.io['scale'].send(data) )"); // Re-use the initial values for mean/scale script->outputs["mean"].link(nn->inputs["mean"]); nn->inputs["mean"].setWaitForMessage(false); script->outputs["scale"].link(nn->inputs["scale"]); nn->inputs["scale"].setWaitForMessage(false); // Always wait for the new frame before starting inference camRgb->preview.link(nn->inputs["frame"]); auto xout = pipeline.create(); xout->setStreamName("nn"); nn->out.link(xout->input); // Connect to device and start pipeline dai::Device device(pipeline); // Output queues will be used to get the rgb frames and nn data from the outputs defined above auto qNn = device.getOutputQueue("nn", 4, false); while(true) { auto inNn = qNn->get(); // To get original frame back (0-255), we add multiply all frame values (pixels) by 255 and then add 127.5 to them. cv::imshow("Original Frame", fromPlanarFp16(inNn->getFirstLayerFp16(), 300, 300, 127.5, 255.0)); int key = cv::waitKey(1); if(key == 'q' || key == 'Q') { return 0; } } return 0; } ``` ## Pipeline ### examples/normalization_multi_input.pipeline.json ```json { "pipeline": { "connections": [ { "node1Id": 2, "node1Output": "mean", "node1OutputGroup": "io", "node2Id": 1, "node2Input": "mean", "node2InputGroup": "inputs" }, { "node1Id": 2, "node1Output": "scale", "node1OutputGroup": "io", "node2Id": 1, "node2Input": "scale", "node2InputGroup": "inputs" }, { "node1Id": 0, "node1Output": "preview", "node1OutputGroup": "", "node2Id": 1, "node2Input": "frame", "node2InputGroup": "inputs" }, { "node1Id": 1, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "in", "node2InputGroup": "" } ], "globalProperties": { "calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1 }, "nodes": [ [ 0, { "id": 0, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 1, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 6, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "still" ], { "blocking": false, "group": "", "id": 7, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 2, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "video" ], { "blocking": false, "group": "", "id": 3, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "isp" ], { "blocking": false, "group": "", "id": 4, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "preview" ], { "blocking": false, "group": "", "id": 5, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 8, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ColorCamera", "properties": { "boardSocket": -1, "cameraName": "", "colorOrder": 0, "fp16": true, "fps": 30.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": false, "isp3aFps": 0, "ispScale": { "horizDenominator": 0, "horizNumerator": 0, "vertDenominator": 0, "vertNumerator": 0 }, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 300, "previewKeepAspectRatio": true, "previewWidth": 300, "rawPacked": null, "resolution": 0, "sensorCropX": -1.0, "sensorCropY": -1.0, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1 } } ], [ 1, { "id": 1, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 9, "name": "in", "queueSize": 5, "type": 3, "waitForMessage": true } ], [ [ "inputs", "mean" ], { "blocking": false, "group": "inputs", "id": 10, "name": "mean", "queueSize": 1, "type": 3, "waitForMessage": false } ], [ [ "", "passthrough" ], { "blocking": false, "group": "", "id": 14, "name": "passthrough", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "inputs", "scale" ], { "blocking": false, "group": "inputs", "id": 11, "name": "scale", "queueSize": 1, "type": 3, "waitForMessage": false } ], [ [ "inputs", "frame" ], { "blocking": false, "group": "inputs", "id": 12, "name": "frame", "queueSize": 1, "type": 3, "waitForMessage": true } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 13, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "NeuralNetwork", "properties": { "blobSize": 1216, "blobUri": "asset:__blob", "numFrames": 8, "numNCEPerThread": 0, "numThreads": 2 } } ], [ 2, { "id": 2, "ioInfo": [ [ [ "io", "mean" ], { "blocking": false, "group": "io", "id": 15, "name": "mean", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "io", "scale" ], { "blocking": false, "group": "io", "id": 16, "name": "scale", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "Script", "properties": { "processor": 1, "scriptName": "