# Feature Tracker This example showscases the [FeatureTracker](https://docs.luxonis.com/software-v3/depthai/depthai-components/nodes/feature_tracker.md) node. It detects features and tracks them between consecutive frames using optical flow by assigning unique ID to matching features. ## Demo This example requires the DepthAI v3 API, see [installation instructions](https://docs.luxonis.com/software-v3/depthai.md). ## Pipeline ### examples/feature_tracker.pipeline.json ```json {"pipeline": {"connections": [{"node1Id": 5, "node1Output": "out", "node1OutputGroup": "", "node2Id": 2, "node2Input": "inputConfig", "node2InputGroup": ""}, {"node1Id": 2, "node1Output": "outputFeatures", "node1OutputGroup": "", "node2Id": 8, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 1, "node1Output": "out", "node1OutputGroup": "", "node2Id": 2, "node2Input": "inputImage", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 6, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 1, "node2Input": "inputImage", "node2InputGroup": ""}], "globalProperties": {"calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1}, "nodes": [[8, {"alias": "", "id": 8, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 13, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_2_outputFeatures"}}], [6, {"alias": "", "id": 6, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 12, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_0_0"}}], [5, {"alias": "", "id": 5, "ioInfo": [[["", "out"], {"blocking": false, "group": "", "id": 11, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkIn", "parentId": -1, "properties": {"maxDataSize": 5242880, "numFrames": 8, "streamName": "__x_2__inputConfig"}}], [2, {"alias": "", "id": 2, "ioInfo": [[["", "passthroughInputImage"], {"blocking": false, "group": "", "id": 10, "name": "passthroughInputImage", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "outputFeatures"], {"blocking": false, "group": "", "id": 9, "name": "outputFeatures", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "inputImage"], {"blocking": false, "group": "", "id": 8, "name": "inputImage", "queueSize": 4, "type": 3, "waitForMessage": true}], [["", "inputConfig"], {"blocking": false, "group": "", "id": 7, "name": "inputConfig", "queueSize": 4, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "FeatureTracker", "parentId": -1, "properties": {"initialConfig": {"cornerDetector": {"cellGridDimension": 4, "enableSobel": true, "enableSorting": true, "numMaxFeatures": 256, "numTargetFeatures": 256, "thresholds": {"decreaseFactor": 0.8999999761581421, "increaseFactor": 1.100000023841858, "initialValue": 20000.0, "max": 0.0, "min": 0.0}, "type": 0}, "featureMaintainer": {"enable": true, "lostFeatureErrorThreshold": 50000.0, "minimumDistanceBetweenFeatures": 50.0, "trackedFeatureThreshold": 200000.0}, "motionEstimator": {"enable": true, "opticalFlow": {"epsilon": 0.009999999776482582, "maxIterations": 9, "pyramidLevels": 0, "searchWindowHeight": 5, "searchWindowWidth": 5}, "type": 0}}, "numMemorySlices": 2, "numShaves": 2}}], [1, {"alias": "", "id": 1, "ioInfo": [[["", "out"], {"blocking": false, "group": "", "id": 6, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "inputImage"], {"blocking": true, "group": "", "id": 5, "name": "inputImage", "queueSize": 3, "type": 3, "waitForMessage": false}], [["", "inputConfig"], {"blocking": true, "group": "", "id": 4, "name": "inputConfig", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "ImageManip", "parentId": -1, "properties": {"backend": 0, "initialConfig": {"base": {"background": 0, "backgroundB": 0, "backgroundG": 0, "backgroundR": 0, "center": true, "colormap": 0, "operations": [], "outputHeight": 0, "outputWidth": 0, "resizeMode": 0, "undistort": false}, "outputFrameType": 30, "reusePreviousImage": false, "skipCurrentImage": false}, "numFramesPool": 4, "outputFrameSize": 1048576, "performanceMode": 0}}], [0, {"alias": "", "id": 0, "ioInfo": [[["dynamicOutputs", "0"], {"blocking": false, "group": "dynamicOutputs", "id": 3, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "raw"], {"blocking": false, "group": "", "id": 2, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "mockIsp"], {"blocking": true, "group": "", "id": 1, "name": "mockIsp", "queueSize": 8, "type": 3, "waitForMessage": false}], [["", "inputControl"], {"blocking": true, "group": "", "id": 0, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "Camera", "parentId": -1, "properties": {"boardSocket": 0, "cameraName": "", "fps": -1.0, "imageOrientation": -1, "initialControl": {"aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": {"height": 0, "priority": 0, "width": 0, "x": 0, "y": 0}, "afRegion": {"height": 0, "priority": 0, "width": 0, "x": 0, "y": 0}, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "enableHdr": false, "expCompensation": 0, "expManual": {"exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0}, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "miscControls": [], "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": {"activeLevel": 0, "enable": 0, "gpioNumber": 0}, "strobeTimings": {"durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0}, "wbColorTemp": 0}, "isp3aFps": 0, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [{"enableUndistortion": null, "fps": {"value": null}, "resizeMode": 0, "size": {"value": {"index": 0, "value": [640, 640]}}, "type": 22}], "resolutionHeight": -1, "resolutionWidth": -1}}]]}} ``` ## Source code #### Python ```python import cv2 from collections import deque import depthai as dai class FeatureTrackerDrawer: lineColor = (200, 0, 200) pointColor = (0, 0, 255) circleRadius = 2 maxTrackedFeaturesPathLength = 30 trackedFeaturesPathLength = 10 trackedIDs = None trackedFeaturesPath = None def onTrackBar(self, val): FeatureTrackerDrawer.trackedFeaturesPathLength = val pass def trackFeaturePath(self, features): newTrackedIDs = set() for currentFeature in features: currentID = currentFeature.id newTrackedIDs.add(currentID) if currentID not in self.trackedFeaturesPath: self.trackedFeaturesPath[currentID] = deque() path = self.trackedFeaturesPath[currentID] path.append(currentFeature.position) while(len(path) > max(1, FeatureTrackerDrawer.trackedFeaturesPathLength)): path.popleft() self.trackedFeaturesPath[currentID] = path featuresToRemove = set() for oldId in self.trackedIDs: if oldId not in newTrackedIDs: featuresToRemove.add(oldId) for id in featuresToRemove: self.trackedFeaturesPath.pop(id) self.trackedIDs = newTrackedIDs def drawFeatures(self, img): cv2.setTrackbarPos(self.trackbarName, self.windowName, FeatureTrackerDrawer.trackedFeaturesPathLength) for featurePath in self.trackedFeaturesPath.values(): path = featurePath for j in range(len(path) - 1): src = (int(path[j].x), int(path[j].y)) dst = (int(path[j + 1].x), int(path[j + 1].y)) cv2.line(img, src, dst, self.lineColor, 1, cv2.LINE_AA, 0) j = len(path) - 1 cv2.circle(img, (int(path[j].x), int(path[j].y)), self.circleRadius, self.pointColor, -1, cv2.LINE_AA, 0) def __init__(self, trackbarName, windowName): self.trackbarName = trackbarName self.windowName = windowName cv2.namedWindow(windowName) cv2.createTrackbar(trackbarName, windowName, FeatureTrackerDrawer.trackedFeaturesPathLength, FeatureTrackerDrawer.maxTrackedFeaturesPathLength, self.onTrackBar) self.trackedIDs = set() self.trackedFeaturesPath = dict() print("Press 'm' to enable/disable motion estimation!") inputConfigQueue = None def on_trackbar(val): try: cfg = dai.FeatureTrackerConfig() cornerDetector = dai.FeatureTrackerConfig.CornerDetector() cornerDetector.numMaxFeatures = cv2.getTrackbarPos('numMaxFeatures', 'Features') cornerDetector.numTargetFeatures = cornerDetector.numMaxFeatures thresholds = dai.FeatureTrackerConfig.CornerDetector.Thresholds() thresholds.initialValue = cv2.getTrackbarPos('harrisScore','Features') cornerDetector.thresholds = thresholds except cv2.error as e: pass cfg.setCornerDetector(cornerDetector) if inputConfigQueue: inputConfigQueue.send(cfg) cv2.namedWindow('Features', cv2.WINDOW_NORMAL) cv2.resizeWindow('Features', 1080, 800) cv2.createTrackbar('harrisScore','Features',20000,25000, on_trackbar) cv2.createTrackbar('numMaxFeatures','Features',256,1024, on_trackbar) # Create pipeline with dai.Pipeline() as pipeline: camera = pipeline.create(dai.node.Camera).build() camOutput = camera.requestOutput((640, 640), dai.ImgFrame.Type.NV12) manip = pipeline.create(dai.node.ImageManip) manip.initialConfig.setFrameType(dai.ImgFrame.Type.GRAY8) camOutput.link(manip.inputImage) featureTracker = pipeline.create(dai.node.FeatureTracker) featureTracker.initialConfig.setCornerDetector(dai.FeatureTrackerConfig.CornerDetector.Type.HARRIS) featureTracker.initialConfig.setMotionEstimator(False) featureTracker.initialConfig.setNumTargetFeatures(256) motionEstimator = dai.FeatureTrackerConfig.MotionEstimator() motionEstimator.enable = True featureTracker.initialConfig.setMotionEstimator(motionEstimator) cornerDetector = dai.FeatureTrackerConfig.CornerDetector() cornerDetector.numMaxFeatures = 256 cornerDetector.numTargetFeatures = cornerDetector.numMaxFeatures # RVC2 specific setting to allow for more features featureTracker.setHardwareResources(2,2) outputFeaturePassthroughQueue = camOutput.createOutputQueue() outputFeatureQueue = featureTracker.outputFeatures.createOutputQueue() manip.out.link(featureTracker.inputImage) inputConfigQueue = featureTracker.inputConfig.createInputQueue() thresholds = dai.FeatureTrackerConfig.CornerDetector.Thresholds() thresholds.initialValue = cv2.getTrackbarPos('harrisScore','Features') cornerDetector.thresholds = thresholds featureTracker.initialConfig.setCornerDetector(cornerDetector) leftWindowName = "Features" leftFeatureDrawer = FeatureTrackerDrawer("Feature tracking duration (frames)", leftWindowName) pipeline.start() while pipeline.isRunning(): outputPassthroughImage : dai.ImgFrame = outputFeaturePassthroughQueue.get() passthroughImage = outputPassthroughImage.getCvFrame() trackedFeaturesLeft = outputFeatureQueue.get().trackedFeatures leftFeatureDrawer.trackFeaturePath(trackedFeaturesLeft) leftFeatureDrawer.drawFeatures(passthroughImage) # Show the frame cv2.imshow(leftWindowName, passthroughImage) key = cv2.waitKey(1) if key == ord('q'): break elif key == ord('m'): cfg = dai.FeatureTrackerConfig() cornerDetector = dai.FeatureTrackerConfig.CornerDetector() cornerDetector.numMaxFeatures = cv2.getTrackbarPos('numMaxFeatures', 'Features') cornerDetector.numTargetFeatures = cornerDetector.numMaxFeatures thresholds = dai.FeatureTrackerConfig.CornerDetector.Thresholds() thresholds.initialValue = cv2.getTrackbarPos('harrisScore','Features') cornerDetector.thresholds = thresholds cfg.setCornerDetector(cornerDetector) cfg.setMotionEstimator(motionEstimator) if motionEstimator.enable == False: motionEstimator.enable = True cfg.setMotionEstimator(motionEstimator) print("Enabling motionEstimator") else: motionEstimator.enable = False cfg.setMotionEstimator(motionEstimator) print("Disabling motionEstimator") inputConfigQueue.send(cfg) ``` #### C++ ```cpp #include #include #include #include #include #include #include "depthai/depthai.hpp" class FeatureTrackerDrawer { public: static const cv::Scalar lineColor; static const cv::Scalar pointColor; static const int circleRadius = 2; static const int maxTrackedFeaturesPathLength = 30; static int trackedFeaturesPathLength; FeatureTrackerDrawer(const std::string& trackbarName, const std::string& windowName) : trackbarName(trackbarName), windowName(windowName) { cv::namedWindow(windowName); cv::createTrackbar(trackbarName, windowName, &trackedFeaturesPathLength, maxTrackedFeaturesPathLength, onTrackBar, this); } static void onTrackBar(int val, void* userdata) { trackedFeaturesPathLength = val; } void trackFeaturePath(const std::vector& features) { std::set newTrackedIDs; for(const auto& currentFeature : features) { int currentID = currentFeature.id; newTrackedIDs.insert(currentID); if(trackedFeaturesPath.find(currentID) == trackedFeaturesPath.end()) { trackedFeaturesPath[currentID] = std::deque(); } auto& path = trackedFeaturesPath[currentID]; path.push_back(currentFeature.position); while(path.size() > std::max(1, trackedFeaturesPathLength)) { path.pop_front(); } } // Remove features that are no longer tracked std::set featuresToRemove; for(const auto& oldId : trackedIDs) { if(newTrackedIDs.find(oldId) == newTrackedIDs.end()) { featuresToRemove.insert(oldId); } } for(const auto& id : featuresToRemove) { trackedFeaturesPath.erase(id); } trackedIDs = newTrackedIDs; } void drawFeatures(cv::Mat& img) { cv::setTrackbarPos(trackbarName.c_str(), windowName.c_str(), trackedFeaturesPathLength); for(const auto& [id, path] : trackedFeaturesPath) { for(size_t j = 0; j < path.size() - 1; j++) { cv::Point src(static_cast(path[j].x), static_cast(path[j].y)); cv::Point dst(static_cast(path[j + 1].x), static_cast(path[j + 1].y)); cv::line(img, src, dst, lineColor, 1, cv::LINE_AA, 0); } if(!path.empty()) { size_t j = path.size() - 1; cv::Point point(static_cast(path[j].x), static_cast(path[j].y)); cv::circle(img, point, circleRadius, pointColor, -1, cv::LINE_AA, 0); } } } private: std::string trackbarName; std::string windowName; std::set trackedIDs; std::map> trackedFeaturesPath; }; // Initialize static members const cv::Scalar FeatureTrackerDrawer::lineColor(200, 0, 200); const cv::Scalar FeatureTrackerDrawer::pointColor(0, 0, 255); int FeatureTrackerDrawer::trackedFeaturesPathLength = 10; void onTrackbar(int val, void* userdata) { auto* inputConfigQueue = static_cast*>(userdata); try { auto cfg = std::make_shared(); auto cornerDetector = dai::FeatureTrackerConfig::CornerDetector(); cornerDetector.numMaxFeatures = cv::getTrackbarPos("numMaxFeatures", "Features"); cornerDetector.numTargetFeatures = cornerDetector.numMaxFeatures; auto thresholds = dai::FeatureTrackerConfig::CornerDetector::Thresholds(); thresholds.initialValue = cv::getTrackbarPos("harrisScore", "Features"); cornerDetector.thresholds = thresholds; cfg->setCornerDetector(cornerDetector); if(*inputConfigQueue) { (*inputConfigQueue)->send(cfg); } } catch(const cv::Exception& e) { // Ignore OpenCV errors } } int main() { std::cout << "Press 'm' to enable/disable motion estimation!" << std::endl; // Create device std::shared_ptr device = std::make_shared(); // Create pipeline dai::Pipeline pipeline(device); // Create nodes auto camera = pipeline.create()->build(); auto camOutput = camera->requestOutput(std::make_pair(640, 640), dai::ImgFrame::Type::NV12); auto manip = pipeline.create(); manip->initialConfig->setFrameType(dai::ImgFrame::Type::GRAY8); camOutput->link(manip->inputImage); auto featureTracker = pipeline.create(); featureTracker->initialConfig->setCornerDetector(dai::FeatureTrackerConfig::CornerDetector::Type::HARRIS); featureTracker->initialConfig->setMotionEstimator(false); featureTracker->initialConfig->setNumTargetFeatures(256); auto motionEstimator = dai::FeatureTrackerConfig::MotionEstimator(); motionEstimator.enable = true; featureTracker->initialConfig->setMotionEstimator(motionEstimator); auto cornerDetector = dai::FeatureTrackerConfig::CornerDetector(); cornerDetector.numMaxFeatures = 256; cornerDetector.numTargetFeatures = cornerDetector.numMaxFeatures; // RVC2 specific setting to allow for more features featureTracker->setHardwareResources(2, 2); auto outputFeaturePassthroughQueue = camOutput->createOutputQueue(); auto outputFeatureQueue = featureTracker->outputFeatures.createOutputQueue(); auto inputConfigQueue = featureTracker->inputConfig.createInputQueue(); manip->out.link(featureTracker->inputImage); auto thresholds = dai::FeatureTrackerConfig::CornerDetector::Thresholds(); thresholds.initialValue = 20000; // Default value cornerDetector.thresholds = thresholds; featureTracker->initialConfig->setCornerDetector(cornerDetector); // Create window and trackbars cv::namedWindow("Features", cv::WINDOW_NORMAL); cv::resizeWindow("Features", 1080, 800); cv::createTrackbar("harrisScore", "Features", nullptr, 25000, onTrackbar, &inputConfigQueue); cv::createTrackbar("numMaxFeatures", "Features", nullptr, 1024, onTrackbar, &inputConfigQueue); cv::setTrackbarPos("harrisScore", "Features", 20000); cv::setTrackbarPos("numMaxFeatures", "Features", 256); std::string leftWindowName = "Features"; FeatureTrackerDrawer leftFeatureDrawer("Feature tracking duration (frames)", leftWindowName); // Start pipeline pipeline.start(); while(true) { auto outputPassthroughImage = outputFeaturePassthroughQueue->get(); if(outputPassthroughImage == nullptr) continue; cv::Mat passthroughImage = outputPassthroughImage->getCvFrame(); auto trackedFeaturesLeft = outputFeatureQueue->get()->trackedFeatures; leftFeatureDrawer.trackFeaturePath(trackedFeaturesLeft); leftFeatureDrawer.drawFeatures(passthroughImage); cv::imshow(leftWindowName, passthroughImage); int key = cv::waitKey(1); if(key == 'q') { break; } else if(key == 'm') { auto cfg = std::make_shared(); auto cornerDetector = dai::FeatureTrackerConfig::CornerDetector(); cornerDetector.numMaxFeatures = cv::getTrackbarPos("numMaxFeatures", "Features"); cornerDetector.numTargetFeatures = cornerDetector.numMaxFeatures; auto thresholds = dai::FeatureTrackerConfig::CornerDetector::Thresholds(); thresholds.initialValue = cv::getTrackbarPos("harrisScore", "Features"); cornerDetector.thresholds = thresholds; cfg->setCornerDetector(cornerDetector); cfg->setMotionEstimator(motionEstimator); if(!motionEstimator.enable) { motionEstimator.enable = true; cfg->setMotionEstimator(motionEstimator); std::cout << "Enabling motionEstimator" << std::endl; } else { motionEstimator.enable = false; cfg->setMotionEstimator(motionEstimator); std::cout << "Disabling motionEstimator" << std::endl; } inputConfigQueue->send(cfg); } } return 0; } ``` ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.