# RGB-Thermal Align This example demonstrates how to align thermal information from a thermal camera to an RGB camera. This setup is useful for applications requiring the overlay or comparison of thermal and color data. An OpenCV window is created to display the blended image of the RGB and aligned thermal data. Trackbars are provided to adjust the blending ratio. ## Demo This example requires the DepthAI v3 API, see [installation instructions](https://docs.luxonis.com/software-v3/depthai.md). ## Source code #### Python ```python #!/usr/bin/env python3 """ Due to an issue with our calibration, you might receive the following error when running this script on early release OAK Thermal devices: ```bash [ImageAlign(4)] [error] Failed to get calibration data: Extrinsic connection between the requested cameraId's doesn't exist. Please recalibrate or modify your calibration data ``` If this happens, please download the calibration data + script from https://drive.google.com/drive/folders/1Q_MZMqWMKDC1eOqVHGPeDO-NJgFmnY5U, place them into the same folder, connect the camera to the computer and run the script. This will update the calibration and add required extrinsics between the camera sensors. """ import cv2 import depthai as dai import numpy as np import time from datetime import timedelta FPS = 25.0 RGB_SOCKET = dai.CameraBoardSocket.CAM_A COLOR_RESOLUTION = dai.ColorCameraProperties.SensorResolution.THE_1080_P class FPSCounter: def __init__(self): self.frameTimes = [] def tick(self): now = time.time() self.frameTimes.append(now) self.frameTimes = self.frameTimes[-100:] def getFps(self): if len(self.frameTimes) <= 1: return 0 # Calculate the FPS return (len(self.frameTimes) - 1) / (self.frameTimes[-1] - self.frameTimes[0]) device = dai.Device() thermalWidth, thermalHeight = -1, -1 thermalFound = False for features in device.getConnectedCameraFeatures(): if dai.CameraSensorType.THERMAL in features.supportedTypes: thermalFound = True thermalSocket = features.socket thermalWidth, thermalHeight = features.width, features.height break if not thermalFound: raise RuntimeError("No thermal camera found!") ISP_SCALE = 3 calibrationHandler = device.readCalibration() rgbDistortion = calibrationHandler.getDistortionCoefficients(RGB_SOCKET) distortionModel = calibrationHandler.getDistortionModel(RGB_SOCKET) if distortionModel != dai.CameraModel.Perspective: raise RuntimeError("Unsupported distortion model for RGB camera. This example supports only Perspective model.") pipeline = dai.Pipeline() # Define sources and outputs camRgb = pipeline.create(dai.node.ColorCamera) thermalCam = pipeline.create(dai.node.Camera) thermalCam.setBoardSocket(thermalSocket) thermalCam.setFps(FPS) sync = pipeline.create(dai.node.Sync) out = pipeline.create(dai.node.XLinkOut) align = pipeline.create(dai.node.ImageAlign) cfgIn = pipeline.create(dai.node.XLinkIn) camRgb.setBoardSocket(RGB_SOCKET) camRgb.setResolution(COLOR_RESOLUTION) camRgb.setFps(FPS) camRgb.setIspScale(1,ISP_SCALE) out.setStreamName("out") sync.setSyncThreshold(timedelta(seconds=0.5 / FPS)) cfgIn.setStreamName("config") cfg = align.initialConfig.get() staticDepthPlane = cfg.staticDepthPlane # Linking align.outputAligned.link(sync.inputs["aligned"]) camRgb.isp.link(sync.inputs["rgb"]) camRgb.isp.link(align.inputAlignTo) thermalCam.raw.link(align.input) sync.out.link(out.input) cfgIn.out.link(align.inputConfig) rgbWeight = 0.4 thermalWeight = 0.6 def updateBlendWeights(percentRgb): """ Update the rgb and depth weights used to blend depth/rgb image @param[in] percent_rgb The rgb weight expressed as a percentage (0..100) """ global thermalWeight global rgbWeight rgbWeight = float(percentRgb) / 100.0 thermalWeight = 1.0 - rgbWeight def updateDepthPlane(depth): global staticDepthPlane staticDepthPlane = depth # Connect to device and start pipeline with device: device.startPipeline(pipeline) queue = device.getOutputQueue("out", 8, False) cfgQ = device.getInputQueue("config") # Configure windows; trackbar adjusts blending ratio of rgb/depth windowName = "rgb-thermal" # Set the window to be resizable and the initial size cv2.namedWindow(windowName, cv2.WINDOW_NORMAL) cv2.resizeWindow(windowName, 1280, 720) cv2.createTrackbar( "RGB Weight %", windowName, int(rgbWeight * 100), 100, updateBlendWeights, ) cv2.createTrackbar( "Static Depth Plane [mm]", windowName, 0, 2000, updateDepthPlane, ) fpsCounter = FPSCounter() while True: messageGroup = queue.get() assert isinstance(messageGroup, dai.MessageGroup) frameRgb = messageGroup["rgb"] assert isinstance(frameRgb, dai.ImgFrame) thermalAligned = messageGroup["aligned"] assert isinstance(thermalAligned, dai.ImgFrame) frameRgbCv = frameRgb.getCvFrame() fpsCounter.tick() rgbIntrinsics = calibrationHandler.getCameraIntrinsics(RGB_SOCKET, int(frameRgbCv.shape[1]), int(frameRgbCv.shape[0])) cvFrameUndistorted = cv2.undistort( frameRgbCv, np.array(rgbIntrinsics), np.array(rgbDistortion), ) # Colorize the aligned depth thermalFrame = thermalAligned.getCvFrame().astype(np.float32) # Create a mask for nan values mask = np.isnan(thermalFrame) # Replace nan values with a mean for visualization thermalFrame[mask] = np.nanmean(thermalFrame) thermalFrame = cv2.normalize(thermalFrame, None, 0, 255, cv2.NORM_MINMAX, cv2.CV_8U) colormappedFrame = cv2.applyColorMap(thermalFrame, cv2.COLORMAP_MAGMA) # Apply the mask back with black pixels (0) colormappedFrame[mask] = 0 blended = cv2.addWeighted(cvFrameUndistorted, rgbWeight, colormappedFrame, thermalWeight, 0) cv2.putText( blended, f"FPS: {fpsCounter.getFps():.2f}", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, ) cv2.imshow(windowName, blended) key = cv2.waitKey(1) if key == ord("q"): break cfg.staticDepthPlane = staticDepthPlane cfgQ.send(cfg) ``` #### C++ ```cpp #include #include #include #include "depthai/depthai.hpp" #include "depthai/pipeline/datatype/ImageAlignConfig.hpp" constexpr auto FPS = 25.0; constexpr auto RGB_SOCKET = dai::CameraBoardSocket::CAM_A; constexpr auto COLOR_RESOLUTION = dai::ColorCameraProperties::SensorResolution::THE_1080_P; class FPSCounter { public: void tick() { auto now = std::chrono::steady_clock::now(); frameTimes.push(now); if(frameTimes.size() > 100) { frameTimes.pop(); } } double getFps() { if(frameTimes.size() <= 1) return 0; auto duration = std::chrono::duration_cast(frameTimes.back() - frameTimes.front()).count(); return (frameTimes.size() - 1) * 1000.0 / duration; } private: std::queue frameTimes; }; double rgbWeight = 0.4; double thermalWeight = 0.6; int staticDepthPlane = 0; void updateBlendWeights(int percentRgb, void*) { rgbWeight = static_cast(percentRgb) / 100.0; thermalWeight = 1.0 - rgbWeight; } void updateDepthPlane(int depth, void*) { staticDepthPlane = depth; } cv::Mat createNaNMask(const cv::Mat& frame) { cv::Mat nanMask = cv::Mat::zeros(frame.size(), CV_8UC1); for(int r = 0; r < frame.rows; ++r) { for(int c = 0; c < frame.cols; ++c) { if(std::isnan(frame.at(r, c))) { nanMask.at(r, c) = 255; } } } return nanMask; } int main() { dai::Device device; int thermalWidth = -1, thermalHeight = -1; bool thermalFound = false; dai::CameraBoardSocket thermalSocket; for(const auto& features : device.getConnectedCameraFeatures()) { if(std::find(features.supportedTypes.begin(), features.supportedTypes.end(), dai::CameraSensorType::THERMAL) != features.supportedTypes.end()) { thermalFound = true; thermalSocket = features.socket; thermalWidth = features.width; thermalHeight = features.height; break; } } if(!thermalFound) { throw std::runtime_error("No thermal camera found!"); } dai::Pipeline pipeline; // Define sources and outputs auto camRgb = pipeline.create(); auto thermalCam = pipeline.create(); auto sync = pipeline.create(); auto out = pipeline.create(); auto align = pipeline.create(); auto cfgIn = pipeline.create(); thermalCam->setBoardSocket(thermalSocket); thermalCam->setFps(FPS); camRgb->setBoardSocket(RGB_SOCKET); camRgb->setResolution(COLOR_RESOLUTION); camRgb->setFps(FPS); camRgb->setIspScale(1, 3); out->setStreamName("out"); sync->setSyncThreshold(std::chrono::milliseconds(static_cast((1 / FPS) * 1000.0 * 0.5))); cfgIn->setStreamName("config"); align->outputAligned.link(sync->inputs["aligned"]); camRgb->isp.link(sync->inputs["rgb"]); camRgb->isp.link(align->inputAlignTo); thermalCam->raw.link(align->input); sync->out.link(out->input); cfgIn->out.link(align->inputConfig); device.startPipeline(pipeline); auto queue = device.getOutputQueue("out", 8, false); auto cfgQ = device.getInputQueue("config"); FPSCounter fpsCounter; std::string windowName = "rgb-thermal"; cv::namedWindow(windowName, cv::WINDOW_NORMAL); cv::resizeWindow(windowName, 1280, 720); cv::createTrackbar("RGB Weight %", windowName, nullptr, 100, updateBlendWeights); cv::createTrackbar("Static Depth Plane [mm]", windowName, nullptr, 2000, updateDepthPlane); while(true) { auto messageGroup = queue->get(); fpsCounter.tick(); auto frameRgb = messageGroup->get("rgb"); auto thermalAligned = messageGroup->get("aligned"); if(frameRgb && thermalAligned) { auto frameRgbCv = frameRgb->getCvFrame(); auto thermalFrame = thermalAligned->getFrame(true); // Colorize the aligned depth cv::Mat mask; cv::Mat colormappedFrame; cv::Mat thermalFrameFloat; thermalFrame.convertTo(thermalFrameFloat, CV_32F); // Get a mask for the nan values mask = createNaNMask(thermalFrameFloat); auto meanValue = cv::mean(thermalFrameFloat, ~mask); thermalFrameFloat.setTo(meanValue, mask); cv::normalize(thermalFrameFloat, thermalFrameFloat, 0, 255, cv::NORM_MINMAX, CV_8U); cv::applyColorMap(thermalFrameFloat, colormappedFrame, cv::COLORMAP_MAGMA); colormappedFrame.setTo(cv::Scalar(0, 0, 0), mask); cv::Mat blended; cv::addWeighted(frameRgbCv, rgbWeight, colormappedFrame, thermalWeight, 0, blended); cv::putText(blended, "FPS: " + std::to_string(fpsCounter.getFps()), cv::Point(10, 30), cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(255, 255, 255), 2); cv::imshow(windowName, blended); } int key = cv::waitKey(1); if(key == 'q' || key == 'Q') { break; } auto cfg = align->initialConfig.get(); cfg.staticDepthPlane = staticDepthPlane; auto alignConfig = std::make_shared(); alignConfig->set(cfg); cfgQ->send(alignConfig); } return 0; } ``` ## Pipeline ### examples/thermal_align.pipeline.json ```json { "pipeline": { "connections": [ { "node1Id": 4, "node1Output": "outputAligned", "node1OutputGroup": "", "node2Id": 2, "node2Input": "aligned", "node2InputGroup": "inputs" }, { "node1Id": 0, "node1Output": "isp", "node1OutputGroup": "", "node2Id": 2, "node2Input": "rgb", "node2InputGroup": "inputs" }, { "node1Id": 5, "node1Output": "out", "node1OutputGroup": "", "node2Id": 4, "node2Input": "inputConfig", "node2InputGroup": "" }, { "node1Id": 0, "node1Output": "isp", "node1OutputGroup": "", "node2Id": 4, "node2Input": "inputAlignTo", "node2InputGroup": "" }, { "node1Id": 1, "node1Output": "raw", "node1OutputGroup": "", "node2Id": 4, "node2Input": "input", "node2InputGroup": "" }, { "node1Id": 2, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "in", "node2InputGroup": "" } ], "globalProperties": { "calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1 }, "nodes": [ [ 0, { "id": 0, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 1, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 6, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "still" ], { "blocking": false, "group": "", "id": 7, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 2, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "video" ], { "blocking": false, "group": "", "id": 3, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "isp" ], { "blocking": false, "group": "", "id": 4, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "preview" ], { "blocking": false, "group": "", "id": 5, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 8, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ColorCamera", "properties": { "boardSocket": 0, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 25.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": true, "isp3aFps": 0, "ispScale": { "horizDenominator": 3, "horizNumerator": 1, "vertDenominator": 3, "vertNumerator": 1 }, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 300, "previewKeepAspectRatio": true, "previewWidth": 300, "rawPacked": null, "resolution": 0, "sensorCropX": -1.0, "sensorCropY": -1.0, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1 } } ], [ 1, { "id": 1, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 9, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 14, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "still" ], { "blocking": false, "group": "", "id": 15, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 10, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "video" ], { "blocking": false, "group": "", "id": 11, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "isp" ], { "blocking": false, "group": "", "id": 12, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "preview" ], { "blocking": false, "group": "", "id": 13, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 16, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "Camera", "properties": { "boardSocket": -1, "calibAlpha": null, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 25.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": true, "isp3aFps": 0, "ispScale": { "horizDenominator": 0, "horizNumerator": 0, "vertDenominator": 0, "vertNumerator": 0 }, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 300, "previewKeepAspectRatio": true, "previewWidth": 300, "rawPacked": null, "resolutionHeight": -1, "resolutionWidth": -1, "sensorCropX": -1.0, "sensorCropY": -1.0, "sensorType": -1, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1, "warpMeshHeight": 0, "warpMeshSource": -1, "warpMeshStepHeight": 32, "warpMeshStepWidth": 32, "warpMeshUri": "", "warpMeshWidth": 0 } } ], [ 2, { "id": 2, "ioInfo": [ [ [ "inputs", "aligned" ], { "blocking": true, "group": "inputs", "id": 17, "name": "aligned", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "inputs", "rgb" ], { "blocking": true, "group": "inputs", "id": 18, "name": "rgb", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 19, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "Sync", "properties": { "syncAttempts": -1, "syncThresholdNs": 20000000 } } ], [ 3, { "id": 3, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 20, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "out" } } ], [ 4, { "id": 4, "ioInfo": [ [ [ "", "inputAlignTo" ], { "blocking": false, "group": "", "id": 21, "name": "inputAlignTo", "queueSize": 1, "type": 3, "waitForMessage": true } ], [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 22, "name": "inputConfig", "queueSize": 4, "type": 3, "waitForMessage": false } ], [ [ "", "input" ], { "blocking": false, "group": "", "id": 23, "name": "input", "queueSize": 4, "type": 3, "waitForMessage": true } ], [ [ "", "outputAligned" ], { "blocking": false, "group": "", "id": 24, "name": "outputAligned", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "passthroughInput" ], { "blocking": false, "group": "", "id": 25, "name": "passthroughInput", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ImageAlign", "properties": { "alignHeight": 0, "alignWidth": 0, "initialConfig": { "staticDepthPlane": 0 }, "interpolation": -1, "numFramesPool": 4, "numShaves": 2, "outKeepAspectRatio": true, "warpHwIds": [] } } ], [ 5, { "id": 5, "ioInfo": [ [ [ "", "out" ], { "blocking": false, "group": "", "id": 26, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "XLinkIn", "properties": { "maxDataSize": 5242880, "numFrames": 8, "streamName": "config" } } ] ] } } ``` ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.