# Visualizer Encoded The example utilizes DepthAI to create a pipeline that performs YOLOv6-Nano object detection, generates custom image annotations with bounding boxes and labels, encodes high-resolution (1920x1440) camera frames as MJPEG, and streams the results over to a remote connection for visualization. This example requires the DepthAI v3 API, see [installation instructions](https://docs.luxonis.com/software-v3/depthai.md). ## Pipeline ### examples/visualizer_encoded.pipeline.json ```json {"pipeline": {"connections": [{"node1Id": 5, "node1Output": "out", "node1OutputGroup": "", "node2Id": 8, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 2, "node1Output": "passthrough", "node1OutputGroup": "", "node2Id": 3, "node2Input": "imageIn", "node2InputGroup": ""}, {"node1Id": 2, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 3, "node1Output": "out", "node1OutputGroup": "", "node2Id": 6, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "1", "node1OutputGroup": "dynamicOutputs", "node2Id": 5, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 2, "node2Input": "in", "node2InputGroup": ""}], "globalProperties": {"calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1}, "nodes": [[8, {"alias": "", "id": 8, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 15, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_5_out"}}], [6, {"alias": "", "id": 6, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 14, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_3__out"}}], [5, {"alias": "", "id": 5, "ioInfo": [[["", "out"], {"blocking": false, "group": "", "id": 13, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "bitstream"], {"blocking": false, "group": "", "id": 12, "name": "bitstream", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "in"], {"blocking": true, "group": "", "id": 11, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "VideoEncoder", "parentId": -1, "properties": {"bitrate": 0, "frameRate": 30.0, "keyframeFrequency": 30, "lossless": false, "maxBitrate": 0, "numBFrames": 0, "numFramesPool": 0, "outputFrameSize": 0, "profile": 4, "quality": 95, "rateCtrlMode": 0}}], [3, {"alias": "detectionParser", "id": 3, "ioInfo": [[["", "out"], {"blocking": false, "group": "", "id": 10, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "imageIn"], {"blocking": false, "group": "", "id": 9, "name": "imageIn", "queueSize": 1, "type": 3, "waitForMessage": true}], [["", "in"], {"blocking": true, "group": "", "id": 8, "name": "in", "queueSize": 1, "type": 3, "waitForMessage": true}]], "logLevel": 3, "name": "DetectionParser", "parentId": 1, "properties": {"networkInputs": {"images": {"dataType": 1, "dims": [416, 416, 3, 1], "name": "images", "numDimensions": 4, "offset": 0, "order": 17185, "qpScale": 1.0, "qpZp": 0.0, "quantization": false, "strides": []}}, "numFramesPool": 8, "parser": {"anchorMasks": {}, "anchors": [], "anchorsV2": [], "classNames": ["person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"], "classes": 80, "confidenceThreshold": 0.5, "coordinates": 4, "iouThreshold": 0.5, "nnFamily": 0, "subtype": "yolov6"}}}], [2, {"alias": "neuralNetwork", "id": 2, "ioInfo": [[["", "passthrough"], {"blocking": false, "group": "", "id": 7, "name": "passthrough", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "out"], {"blocking": false, "group": "", "id": 6, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "in"], {"blocking": true, "group": "", "id": 5, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": true}]], "logLevel": 3, "name": "NeuralNetwork", "parentId": 1, "properties": {"backend": "", "backendProperties": {}, "blobSize": 8689834, "blobUri": "asset:__blob", "modelSource": 0, "modelUri": "", "numFrames": 8, "numNCEPerThread": 0, "numShavesPerThread": 0, "numThreads": 0}}], [0, {"alias": "", "id": 0, "ioInfo": [[["dynamicOutputs", "0"], {"blocking": false, "group": "dynamicOutputs", "id": 4, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false}], [["dynamicOutputs", "1"], {"blocking": false, "group": "dynamicOutputs", "id": 3, "name": "1", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "raw"], {"blocking": false, "group": "", "id": 2, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "mockIsp"], {"blocking": true, "group": "", "id": 1, "name": "mockIsp", "queueSize": 8, "type": 3, "waitForMessage": false}], [["", "inputControl"], {"blocking": true, "group": "", "id": 0, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "Camera", "parentId": -1, "properties": {"boardSocket": 0, "cameraName": "", "fps": -1.0, "imageOrientation": -1, "initialControl": {"aeLockMode": false, "aeMaxExposureTimeUs": 100663297, "aeRegion": {"height": 0, "priority": 2434620032, "width": 0, "x": 4163, "y": 1}, "afRegion": {"height": 0, "priority": 0, "width": 0, "x": 21857, "y": 0}, "antiBandingMode": 29, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 200, "brightness": -105, "captureIntent": 145, "chromaDenoise": 0, "cmdMask": 0, "contrast": -33, "controlMode": 97, "effectMode": 85, "enableHdr": false, "expCompensation": 39, "expManual": {"exposureTimeUs": 21857, "frameDurationUs": 21857, "sensitivityIso": 2434620140}, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 91, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "miscControls": [], "saturation": -34, "sceneMode": 90, "sharpness": 0, "strobeConfig": {"activeLevel": 40, "enable": 0, "gpioNumber": 97}, "strobeTimings": {"durationUs": 16, "exposureBeginOffsetUs": 21857, "exposureEndOffsetUs": 17}, "wbColorTemp": 0}, "isp3aFps": 0, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [{"enableUndistortion": null, "fps": {"value": null}, "resizeMode": 0, "size": {"value": {"index": 0, "value": [416, 416]}}, "type": 8}, {"enableUndistortion": null, "fps": {"value": null}, "resizeMode": 0, "size": {"value": {"index": 0, "value": [1920, 1440]}}, "type": 22}], "resolutionHeight": -1, "resolutionWidth": -1}}]]}} ``` ## Source code #### Python ```python #!/usr/bin/env python3 import depthai as dai from argparse import ArgumentParser parser = ArgumentParser() parser.add_argument("--webSocketPort", type=int, default=8765) parser.add_argument("--httpPort", type=int, default=8082) args = parser.parse_args() remoteConnector = dai.RemoteConnection(webSocketPort=args.webSocketPort, httpPort=args.httpPort) ENCODER_PROFILE = dai.VideoEncoderProperties.Profile.MJPEG class ImgAnnotationsGenerator(dai.node.ThreadedHostNode): def __init__(self): super().__init__() self.inputDet = self.createInput() self.output = self.createOutput() def setLabelMap(self, labelMap): self.labelMap = labelMap def run(self): while self.mainLoop(): nnData = self.inputDet.get() detections = nnData.detections imgAnnt = dai.ImgAnnotations() imgAnnt.setTimestamp(nnData.getTimestamp()) imgAnnt.setTransformation(nnData.getTransformation()) annotation = dai.ImgAnnotation() for detection in detections: pointsAnnotation = dai.PointsAnnotation() pointsAnnotation.type = dai.PointsAnnotationType.LINE_STRIP pointsAnnotation.points = dai.VectorPoint2f([ dai.Point2f(detection.xmin, detection.ymin), dai.Point2f(detection.xmax, detection.ymin), dai.Point2f(detection.xmax, detection.ymax), dai.Point2f(detection.xmin, detection.ymax), ]) outlineColor = dai.Color(1.0, 0.5, 0.5, 1.0) pointsAnnotation.outlineColor = outlineColor fillColor = dai.Color(0.5, 1.0, 0.5, 0.5) pointsAnnotation.fillColor = fillColor pointsAnnotation.thickness = 2.0 text = dai.TextAnnotation() text.position = dai.Point2f(detection.xmin, detection.ymin) text.text = f"{self.labelMap[detection.label]} {int(detection.confidence * 100)}%" text.fontSize = 50.5 textColor = dai.Color(0.5, 0.5, 1.0, 1.0) text.textColor = textColor backgroundColor = dai.Color(1.0, 1.0, 0.5, 1.0) text.backgroundColor = backgroundColor annotation.points.append(pointsAnnotation) annotation.texts.append(text) imgAnnt.annotations.append(annotation) self.output.send(imgAnnt) # Create pipeline with dai.Pipeline() as pipeline: cameraNode = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_A) detectionNetwork = pipeline.create(dai.node.DetectionNetwork).build( cameraNode, dai.NNModelDescription("yolov6-nano") ) imageAnnotationsGenerator = pipeline.create(ImgAnnotationsGenerator) outputToEncode = cameraNode.requestOutput((1920, 1440), type=dai.ImgFrame.Type.NV12) encoder = pipeline.create(dai.node.VideoEncoder) encoder.setDefaultProfilePreset(30, ENCODER_PROFILE) outputToEncode.link(encoder.input) detectionNetwork.out.link(imageAnnotationsGenerator.inputDet) labelMap = detectionNetwork.getClasses() imageAnnotationsGenerator.setLabelMap(labelMap) # Add the remote connector topics remoteConnector.addTopic("encoded", encoder.out, "images") remoteConnector.addTopic("detections", detectionNetwork.out, "images") remoteConnector.addTopic("annotations", imageAnnotationsGenerator.output, "images") pipeline.start() # Register the pipeline with the remote connector remoteConnector.registerPipeline(pipeline) while pipeline.isRunning(): if remoteConnector.waitKey(1) == ord("q"): pipeline.stop() break ``` #### C++ ```cpp #include #include #include #include #include #include #include "depthai/depthai.hpp" #include "depthai/pipeline/datatype/ImgAnnotations.hpp" #include "depthai/remote_connection/RemoteConnection.hpp" // Global flag for graceful shutdown std::atomic quitEvent(false); // Signal handler void signalHandler(int signum) { quitEvent = true; } // Custom host node for image annotations class ImgAnnotationsGenerator : public dai::NodeCRTP { public: Input& inputDet = inputs["detections"]; Output& output = out; std::vector labelMap; std::shared_ptr build(Output& detections) { detections.link(inputDet); return std::static_pointer_cast(this->shared_from_this()); } std::shared_ptr processGroup(std::shared_ptr in) override { auto nnData = in->get("detections"); auto detections = nnData->detections; auto imgAnnt = std::make_shared(); imgAnnt->setTimestamp(nnData->getTimestamp()); imgAnnt->transformation = nnData->transformation; dai::ImgAnnotation annotation; for(const auto& detection : detections) { // Create points annotation for bounding box auto pointsAnnotation = std::make_shared(); pointsAnnotation->type = dai::PointsAnnotationType::LINE_STRIP; pointsAnnotation->points = {dai::Point2f(detection.xmin, detection.ymin), dai::Point2f(detection.xmax, detection.ymin), dai::Point2f(detection.xmax, detection.ymax), dai::Point2f(detection.xmin, detection.ymax)}; // Set colors and thickness pointsAnnotation->outlineColor = dai::Color(1.0f, 0.5f, 0.5f, 1.0f); pointsAnnotation->fillColor = dai::Color(0.5f, 1.0f, 0.5f, 0.5f); pointsAnnotation->thickness = 2.0f; // Create text annotation auto text = std::make_shared(); text->position = dai::Point2f(detection.xmin, detection.ymin); // Get label text std::string labelText; try { labelText = labelMap[detection.label]; } catch(...) { labelText = std::to_string(detection.label); } text->text = labelText + " " + std::to_string(static_cast(detection.confidence * 100)) + "%"; text->fontSize = 50.5f; text->textColor = dai::Color(0.5f, 0.5f, 1.0f, 1.0f); text->backgroundColor = dai::Color(1.0f, 1.0f, 0.5f, 1.0f); annotation.points.push_back(*pointsAnnotation); annotation.texts.push_back(*text); } imgAnnt->annotations.push_back(annotation); return imgAnnt; } }; int main(int argc, char** argv) { // Set up signal handlers signal(SIGTERM, signalHandler); signal(SIGINT, signalHandler); try { // Create remote connection dai::RemoteConnection remoteConnector; // Create pipeline dai::Pipeline pipeline; // Create nodes auto cameraNode = pipeline.create(); cameraNode->build(dai::CameraBoardSocket::CAM_A); auto detectionNetwork = pipeline.create(); dai::NNModelDescription modelDesc; modelDesc.model = "yolov6-nano"; detectionNetwork->build(cameraNode, modelDesc); auto imageAnnotationsGenerator = pipeline.create(); auto outputToEncode = cameraNode->requestOutput(std::make_pair(1920, 1440), dai::ImgFrame::Type::NV12); auto encoder = pipeline.create(); encoder->setDefaultProfilePreset(30, dai::VideoEncoderProperties::Profile::MJPEG); // Linking outputToEncode->link(encoder->input); detectionNetwork->out.link(imageAnnotationsGenerator->inputDet); // Set label map imageAnnotationsGenerator->labelMap = detectionNetwork->getClasses().value(); // Add remote connector topics remoteConnector.addTopic("encoded", encoder->out, "images"); remoteConnector.addTopic("detections", detectionNetwork->out, "images"); remoteConnector.addTopic("annotations", imageAnnotationsGenerator->output, "images"); // Start pipeline pipeline.start(); // Register pipeline with remote connector remoteConnector.registerPipeline(pipeline); // Main loop while(pipeline.isRunning() && !quitEvent) { if(remoteConnector.waitKey(1) == 'q') { pipeline.stop(); break; } } // Cleanup pipeline.stop(); pipeline.wait(); } catch(const std::exception& e) { std::cerr << "Error: " << e.what() << std::endl; return 1; } return 0; } ``` ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.