## Template app

> **Deprecation Notice**
> This page describes features that have been removed or replaced in Luxonis Hub. Functionality may be limited and only available to Hub Original customers. Please refer to the up-to-date guides to provision devices, manage fleets, and deploy applications.

Let's dive into the basics using an example. We'll create a simple application that runs an object detection neural network and
streams color video with visualized neural network detections. Additionally, it uploads images and videos to Luxonis Hub when an
interesting situation occurs.

## Code organization

### Business logic

In smaller applications, the standard approach is to route all pipeline outputs to a unified location for processing business
logic. In this example, our actions include:

 * Transmitting h264 encoded frames with overlay data to the Luxonis Hub, enabling us to view the live stream via the Live View
   tab.
 * Sending an image event to the Luxonis Hub when a person is detected and the interval set by image_event_upload_interval_minutes
   has passed.
 * Sending a video event to the Luxonis Hub upon person detection and after the video_event_upload_interval_minutes interval.

These three actions constitute the core reporting functionalities provided by the robothub framework.

```python
class BusinessLogic:
    def __init__(self, frame_buffer: rh.FrameBuffer, live_view: rh.DepthaiLiveView):
        self.live_view: rh.DepthaiLiveView = live_view
        self.frame_buffer: rh.FrameBuffer = frame_buffer

        self.last_image_event_upload_seconds = time.time()
        self.last_video_event_upload_seconds = time.time()

    def process_pipeline_outputs(self, h264_frame: dai.ImgFrame, mjpeg_frame: dai.ImgFrame, object_detections: dai.ImgDetections):
        self.frame_buffer.add_frame(h264_frame) # make sure to store every h264 frame
        for detection in object_detections.detections:
            # visualize bounding box in the live view
            bbox = (detection.xmin, detection.ymin, detection.xmax, detection.ymax)
            self.live_view.add_rectangle(bbox, label=nn_config.labels[detection.label])

            current_time_seconds = time.time()
            # arbitrary condition for sending image events to RobotHub
            if current_time_seconds - self.last_image_event_upload_seconds > rh.CONFIGURATION["image_event_upload_interval_minutes"] * 60:
                if nn_config.labels[detection.label] == 'person':
                    self.last_image_event_upload_seconds = current_time_seconds
                    rh.send_image_event(image=mjpeg_frame.getCvFrame(), title='Person detected')
            # arbitrary condition for sending video events to RobotHub
            if current_time_seconds - self.last_video_event_upload_seconds > rh.CONFIGURATION["video_event_upload_interval_minutes"] * 60:
                if nn_config.labels[detection.label] == 'person':
                    self.last_video_event_upload_seconds = current_time_seconds
                    self.frame_buffer.save_video_event(before_seconds=60, after_seconds=60, title="Interesting video",
                                                       fps=rh.CONFIGURATION["fps"], frame_width=self.live_view.frame_width,
                                                       frame_height=self.live_view.frame_height)
        self.live_view.publish(h264_frame=h264_frame.getCvFrame())
```

### Application

A class derived from robothub.BaseDepthAIApplication acts as the entry point for the application. This class requires the user to
define two essential methods: setup_pipeline and manage_device.

```python
class Application(rh.BaseDepthAIApplication):

    def __init__(self):
        super().__init__()
        self.live_view = rh.DepthaiLiveView(name="live_view", unique_key="rgb",
                                            width=1920, height=1080)
        frame_buffer = rh.FrameBuffer(maxlen=rh.CONFIGURATION["fps"] * 60 * 2)  # buffer last 2 minutes
        self.business_logic = BusinessLogic(frame_buffer=frame_buffer, live_view=self.live_view)

    def setup_pipeline(self) -> dai.Pipeline:
        """Define the pipeline using DepthAI."""

        log.info(f"App config: {rh.CONFIGURATION}")
        pipeline = dai.Pipeline()
        rgb_sensor = create_rgb_sensor(pipeline=pipeline, preview_resolution=(640, 352))
        rgb_h264_encoder = create_h264_encoder(node_input=rgb_sensor.video, pipeline=pipeline)
        rgb_mjpeg_encoder = create_mjpeg_encoder(node_input=rgb_sensor.video, pipeline=pipeline)
        object_detection_nn = create_yolov7tiny_coco_nn(node_input=rgb_sensor.preview, pipeline=pipeline)

        create_output(pipeline=pipeline, node_input=rgb_h264_encoder.bitstream, stream_name="h264_frames")
        create_output(pipeline=pipeline, node_input=rgb_mjpeg_encoder.bitstream, stream_name="mjpeg_frames")
        create_output(pipeline=pipeline, node_input=object_detection_nn.out, stream_name="object_detections")
        return pipeline

    def manage_device(self, device: dai.Device):
        log.info(f"{device.getMxId()} creating output queues...")
        h264_frames_queue = device.getOutputQueue(name="h264_frames", maxSize=10, blocking=True)
        mjpeg_frames_queue = device.getOutputQueue(name="mjpeg_frames", maxSize=10, blocking=True)
        object_detections_queue = device.getOutputQueue(name="object_detections", maxSize=10, blocking=True)

        log.info(f"{device.getMxId()} Application started")
        while rh.app_is_running() and self.device_is_running:
            h264_frame = h264_frames_queue.get()
            mjpeg_frame = mjpeg_frames_queue.get()
            object_detections = object_detections_queue.get()
            self.business_logic.process_pipeline_outputs(h264_frame=h264_frame, mjpeg_frame=mjpeg_frame, object_detections=object_detections)
            time.sleep(0.001)

    def on_configuration_changed(self, configuration_changes: dict) -> None:
        log.info(f"CONFIGURATION CHANGES: {configuration_changes}")
        if "fps" in configuration_changes:
            log.info(f"FPS change needs a new pipeline. Restarting OAK device...")
            self.restart_device()
```

### Pipeline

This location is where the pipeline is created. The pipeline acts as a blueprint for the OAK device, defining its on-device data
processing. Use the [DepthAI](https://docs.luxonis.com/software-v3/depthai.md) library to define the pipeline that fits the needs
of your application.

```python
    def setup_pipeline(self) -> dai.Pipeline:
        """Define the pipeline using DepthAI."""

        log.info(f"App config: {rh.CONFIGURATION}")
        pipeline = dai.Pipeline()
        rgb_sensor = create_rgb_sensor(pipeline=pipeline, preview_resolution=(640, 352))
        rgb_h264_encoder = create_h264_encoder(node_input=rgb_sensor.video, pipeline=pipeline)
        rgb_mjpeg_encoder = create_mjpeg_encoder(node_input=rgb_sensor.video, pipeline=pipeline)
        object_detection_nn = create_yolov7tiny_coco_nn(node_input=rgb_sensor.preview, pipeline=pipeline)

        create_output(pipeline=pipeline, node_input=rgb_h264_encoder.bitstream, stream_name="h264_frames")
        create_output(pipeline=pipeline, node_input=rgb_mjpeg_encoder.bitstream, stream_name="mjpeg_frames")
        create_output(pipeline=pipeline, node_input=object_detection_nn.out, stream_name="object_detections")
        return pipeline
```

### Main loop

By the time execution reaches the manage_device method, the device is already executing the pipeline. To get the data into our
application, we must create output queues and retrieve data from them. Each XLinkOut defined in the pipeline corresponds to one
output queue.

```python
    def manage_device(self, device: dai.Device):
        log.info(f"{device.getMxId()} creating output queues...")
        h264_frames_queue = device.getOutputQueue(name="h264_frames", maxSize=10, blocking=True)
        mjpeg_frames_queue = device.getOutputQueue(name="mjpeg_frames", maxSize=10, blocking=True)
        object_detections_queue = device.getOutputQueue(name="object_detections", maxSize=10, blocking=True)

        log.info(f"{device.getMxId()} Application started")
        while rh.app_is_running() and self.device_is_running:
            h264_frame = h264_frames_queue.get()
            mjpeg_frame = mjpeg_frames_queue.get()
            object_detections = object_detections_queue.get()
            self.business_logic.process_pipeline_outputs(h264_frame=h264_frame, mjpeg_frame=mjpeg_frame, object_detections=object_detections)
            time.sleep(0.001)
```

### Config change

This is an optional method to implement and it is called when the configuration of the application is changed in Luxonis Hub. The
default behavior is the application restart.

```python
    def on_configuration_changed(self, configuration_changes: dict) -> None:
        log.info(f"CONFIGURATION CHANGES: {configuration_changes}")
        if "fps" in configuration_changes:
            log.info(f"FPS change needs a new pipeline. Restarting OAK device...")
            self.restart_device()
```

## Pipeline definition

### RGB sensor

We're making an object for the RGB sensor. This sensor will provide 1080p resolution and operate at configured frames per second
(FPS) rate. Typically, our devices have three sensors: one RGB sensor in the middle and two monochrome sensors on the sides.

```python
def create_rgb_sensor(pipeline: dai.Pipeline,
                      fps: int = 30,
                      resolution: dai.ColorCameraProperties.SensorResolution = dai.ColorCameraProperties.SensorResolution.THE_1080_P,
                      preview_resolution: tuple = (1280, 720),
                      ) -> dai.node.ColorCamera:
    node = pipeline.createColorCamera()
    node.setBoardSocket(dai.CameraBoardSocket.CAM_A)
    node.setInterleaved(False)
    node.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
    node.setPreviewNumFramesPool(4)
    node.setPreviewSize(*preview_resolution)
    node.setVideoSize(1920, 1080)
    node.setResolution(resolution)
    node.setFps(fps)
    return node
```

### Encoders

For nearly all Perception apps, you'll start with two encoders. The MJPEG encoder gives you .jpeg frames for image reports.
Meanwhile, the H264 encoder provides frames necessary for streaming live video to Luxonis Hub.

```python
def create_h264_encoder(node_input: dai.Node.Output, pipeline: dai.Pipeline, fps: int = 30):
    rh_encoder = pipeline.createVideoEncoder()
    rh_encoder_profile = dai.VideoEncoderProperties.Profile.H264_MAIN
    rh_encoder.setDefaultProfilePreset(fps, rh_encoder_profile)
    rh_encoder.input.setQueueSize(2)
    rh_encoder.input.setBlocking(False)
    rh_encoder.setKeyframeFrequency(fps)
    rh_encoder.setRateControlMode(dai.VideoEncoderProperties.RateControlMode.CBR)
    rh_encoder.setNumFramesPool(3)
    node_input.link(rh_encoder.input)
    return rh_encoder

def create_mjpeg_encoder(node_input: dai.Node.Output, pipeline: dai.Pipeline, fps: int = 30, quality: int = 100):
    encoder = pipeline.createVideoEncoder()
    encoder_profile = dai.VideoEncoderProperties.Profile.MJPEG
    encoder.setDefaultProfilePreset(fps, encoder_profile)
    encoder.setQuality(quality)
    node_input.link(encoder.input)
    return encoder
```

### Neural network

Next, we're setting up a neural network that processes images. We're using a model from our collection of pre-trained models,
compatible with the Myriad X accelerator. The expected input is an image with shape 640x352x3 [w,h,ch]. This is the reason why we
set the preview size to this exact shape.

```python
preview_resolution = (640, 352)
node.setPreviewSize(*preview_resolution)
```

### RGB sensor

Connect any pipeline node to a XLinkOut node and get the node data into your application.

```python
def create_output(pipeline, node_input: dai.Node.Output, stream_name: str):
    xout = pipeline.createXLinkOut()
    xout.setStreamName(stream_name)
    node_input.link(xout.input)
```

## Full code

### app.py

```python
import logging as log
import time

import blobconverter
import depthai as dai
import object_detector_config as nn_config
import robothub as rh

class BusinessLogic:
    def __init__(self, frame_buffer: rh.FrameBuffer, live_view: rh.DepthaiLiveView):
        self.live_view: rh.DepthaiLiveView = live_view
        self.frame_buffer: rh.FrameBuffer = frame_buffer

        self.last_image_event_upload_seconds = time.time()
        self.last_video_event_upload_seconds = time.time()

    def process_pipeline_outputs(self, h264_frame: dai.ImgFrame, mjpeg_frame: dai.ImgFrame, object_detections: dai.ImgDetections):
        self.frame_buffer.add_frame(h264_frame) # make sure to store every h264 frame
        for detection in object_detections.detections:
            # visualize bounding box in the live view
            bbox = (detection.xmin, detection.ymin, detection.xmax, detection.ymax)
            self.live_view.add_rectangle(bbox, label=nn_config.labels[detection.label])

            current_time_seconds = time.time()
            # arbitrary condition for sending image events to RobotHub
            if current_time_seconds - self.last_image_event_upload_seconds > rh.CONFIGURATION["image_event_upload_interval_minutes"] * 60:
                if nn_config.labels[detection.label] == 'person':
                    self.last_image_event_upload_seconds = current_time_seconds
                    rh.send_image_event(image=mjpeg_frame.getCvFrame(), title='Person detected')
            # arbitrary condition for sending video events to RobotHub
            if current_time_seconds - self.last_video_event_upload_seconds > rh.CONFIGURATION["video_event_upload_interval_minutes"] * 60:
                if nn_config.labels[detection.label] == 'person':
                    self.last_video_event_upload_seconds = current_time_seconds
                    self.frame_buffer.save_video_event(before_seconds=60, after_seconds=60, title="Interesting video",
                                                       fps=rh.CONFIGURATION["fps"], frame_width=self.live_view.frame_width,
                                                       frame_height=self.live_view.frame_height)
        self.live_view.publish(h264_frame=h264_frame.getCvFrame())

class Application(rh.BaseDepthAIApplication):

    def __init__(self):
        super().__init__()
        self.live_view = rh.DepthaiLiveView(name="live_view", unique_key="rgb",
                                            width=1920, height=1080)
        frame_buffer = rh.FrameBuffer(maxlen=rh.CONFIGURATION["fps"] * 60 * 2)  # buffer last 2 minutes
        self.business_logic = BusinessLogic(frame_buffer=frame_buffer, live_view=self.live_view)

    def setup_pipeline(self) -> dai.Pipeline:
        """Define the pipeline using DepthAI."""

        log.info(f"App config: {rh.CONFIGURATION}")
        pipeline = dai.Pipeline()
        rgb_sensor = create_rgb_sensor(pipeline=pipeline, preview_resolution=(640, 352))
        rgb_h264_encoder = create_h264_encoder(node_input=rgb_sensor.video, pipeline=pipeline)
        rgb_mjpeg_encoder = create_mjpeg_encoder(node_input=rgb_sensor.video, pipeline=pipeline)
        object_detection_nn = create_yolov7tiny_coco_nn(node_input=rgb_sensor.preview, pipeline=pipeline)

        create_output(pipeline=pipeline, node_input=rgb_h264_encoder.bitstream, stream_name="h264_frames")
        create_output(pipeline=pipeline, node_input=rgb_mjpeg_encoder.bitstream, stream_name="mjpeg_frames")
        create_output(pipeline=pipeline, node_input=object_detection_nn.out, stream_name="object_detections")
        return pipeline

    def manage_device(self, device: dai.Device):
        log.info(f"{device.getMxId()} creating output queues...")
        h264_frames_queue = device.getOutputQueue(name="h264_frames", maxSize=10, blocking=True)
        mjpeg_frames_queue = device.getOutputQueue(name="mjpeg_frames", maxSize=10, blocking=True)
        object_detections_queue = device.getOutputQueue(name="object_detections", maxSize=10, blocking=True)

        log.info(f"{device.getMxId()} Application started")
        while rh.app_is_running() and self.device_is_running:
            h264_frame = h264_frames_queue.get()
            mjpeg_frame = mjpeg_frames_queue.get()
            object_detections = object_detections_queue.get()
            self.business_logic.process_pipeline_outputs(h264_frame=h264_frame, mjpeg_frame=mjpeg_frame, object_detections=object_detections)
            time.sleep(0.001)

    def on_configuration_changed(self, configuration_changes: dict) -> None:
        log.info(f"CONFIGURATION CHANGES: {configuration_changes}")
        if "fps" in configuration_changes:
            log.info(f"FPS change needs a new pipeline. Restarting OAK device...")
            self.restart_device()

def create_rgb_sensor(pipeline: dai.Pipeline,
                      fps: int = 30,
                      resolution: dai.ColorCameraProperties.SensorResolution = dai.ColorCameraProperties.SensorResolution.THE_1080_P,
                      preview_resolution: tuple = (1280, 720),
                      ) -> dai.node.ColorCamera:
    node = pipeline.createColorCamera()
    node.setBoardSocket(dai.CameraBoardSocket.CAM_A)
    node.setInterleaved(False)
    node.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
    node.setPreviewNumFramesPool(4)
    node.setPreviewSize(*preview_resolution)
    node.setVideoSize(1920, 1080)
    node.setResolution(resolution)
    node.setFps(fps)
    return node

def create_h264_encoder(node_input: dai.Node.Output, pipeline: dai.Pipeline, fps: int = 30):
    rh_encoder = pipeline.createVideoEncoder()
    rh_encoder_profile = dai.VideoEncoderProperties.Profile.H264_MAIN
    rh_encoder.setDefaultProfilePreset(fps, rh_encoder_profile)
    rh_encoder.input.setQueueSize(2)
    rh_encoder.input.setBlocking(False)
    rh_encoder.setKeyframeFrequency(fps)
    rh_encoder.setRateControlMode(dai.VideoEncoderProperties.RateControlMode.CBR)
    rh_encoder.setNumFramesPool(3)
    node_input.link(rh_encoder.input)
    return rh_encoder

def create_mjpeg_encoder(node_input: dai.Node.Output, pipeline: dai.Pipeline, fps: int = 30, quality: int = 100):
    encoder = pipeline.createVideoEncoder()
    encoder_profile = dai.VideoEncoderProperties.Profile.MJPEG
    encoder.setDefaultProfilePreset(fps, encoder_profile)
    encoder.setQuality(quality)
    node_input.link(encoder.input)
    return encoder

def create_yolov7tiny_coco_nn(node_input: dai.Node.Output, pipeline: dai.Pipeline) -> dai.node.YoloDetectionNetwork:
    model = "yolov7tiny_coco_640x352"
    node = pipeline.createYoloDetectionNetwork()
    blob = dai.OpenVINO.Blob(blobconverter.from_zoo(name=model, zoo_type="depthai", shaves=6))
    node.setBlob(blob)
    node_input.link(node.input)
    node.input.setBlocking(False)
    # Yolo specific parameters
    node.setConfidenceThreshold(0.5)
    node.setNumClasses(80)
    node.setCoordinateSize(4)
    node.setAnchors([12.0, 16.0, 19.0, 36.0, 40.0, 28.0, 36.0, 75.0, 76.0, 55.0, 72.0, 146.0, 142.0, 110.0, 192.0, 243.0, 459.0, 401.0])
    node.setAnchorMasks({
        "side80": [0, 1, 2],
        "side40": [3, 4, 5],
        "side20": [6, 7, 8]
    })
    node.setIouThreshold(0.5)
    return node

def create_output(pipeline, node_input: dai.Node.Output, stream_name: str):
    xout = pipeline.createXLinkOut()
    xout.setStreamName(stream_name)
    node_input.link(xout.input)

if __name__ == "__main__":
    app = Application()
    app.run()
```

### robotapp.toml

```toml
config_version = "2.0"

[info]
name = "{{{ APP_NAME }}}"
description = "This is a generated description. Replace it with your own."

[runtime]
application = "app.py#Application"
workdir = "/app"

[runtime.runs_on]
type = "image"
name = "ghcr.io/luxonis/robothub-app-v2:2024.087.1457-rvc2-regular"

[[configuration]]
visual = "section"
title = "Camera config"

[[configuration]]
key = "fps"
label = "FPS"
field = "num_range"
step = 1
min = 1
max = 30
initial_value = 20

[[configuration]]
visual = "section"
title = "Upload intervals"

[[configuration]]
key = "image_event_upload_interval_minutes"
label = "Image event upload interval (minutes)"
field = "num_range"
step = 1
min = 1
max = 60
initial_value = 1

[[configuration]]
key = "video_event_upload_interval_minutes"
label = "Video event upload interval (minutes)"
field = "num_range"
step = 1
min = 1
max = 60
initial_value = 5
```
