# Feature Tracker Example shows capabilities of [FeatureTracker](https://docs.luxonis.com/software/depthai/examples/feature_tracker.md). It detects features and tracks them between consecutive frames using optical flow by assigning unique ID to matching features. [Feature Detector](https://docs.luxonis.com/software/depthai/examples/feature_detector.md) example only detects features. ## Demo ## Setup Please run the [install script](https://github.com/luxonis/depthai-python/blob/main/examples/install_requirements.py) to download all required dependencies. Please note that this script must be ran from git context, so you have to download the [depthai-python](https://github.com/luxonis/depthai-python) repository first and then run the script ```bash git clone https://github.com/luxonis/depthai-python.git cd depthai-python/examples python3 install_requirements.py ``` For additional information, please follow the [installation guide](https://docs.luxonis.com/software/depthai/manual-install.md). ## Source code #### Python ```python #!/usr/bin/env python3 import cv2 import depthai as dai from collections import deque class FeatureTrackerDrawer: lineColor = (200, 0, 200) pointColor = (0, 0, 255) circleRadius = 2 maxTrackedFeaturesPathLength = 30 # for how many frames the feature is tracked trackedFeaturesPathLength = 10 trackedIDs = None trackedFeaturesPath = None def onTrackBar(self, val): FeatureTrackerDrawer.trackedFeaturesPathLength = val pass def trackFeaturePath(self, features): newTrackedIDs = set() for currentFeature in features: currentID = currentFeature.id newTrackedIDs.add(currentID) if currentID not in self.trackedFeaturesPath: self.trackedFeaturesPath[currentID] = deque() path = self.trackedFeaturesPath[currentID] path.append(currentFeature.position) while(len(path) > max(1, FeatureTrackerDrawer.trackedFeaturesPathLength)): path.popleft() self.trackedFeaturesPath[currentID] = path featuresToRemove = set() for oldId in self.trackedIDs: if oldId not in newTrackedIDs: featuresToRemove.add(oldId) for id in featuresToRemove: self.trackedFeaturesPath.pop(id) self.trackedIDs = newTrackedIDs def drawFeatures(self, img): cv2.setTrackbarPos(self.trackbarName, self.windowName, FeatureTrackerDrawer.trackedFeaturesPathLength) for featurePath in self.trackedFeaturesPath.values(): path = featurePath for j in range(len(path) - 1): src = (int(path[j].x), int(path[j].y)) dst = (int(path[j + 1].x), int(path[j + 1].y)) cv2.line(img, src, dst, self.lineColor, 1, cv2.LINE_AA, 0) j = len(path) - 1 cv2.circle(img, (int(path[j].x), int(path[j].y)), self.circleRadius, self.pointColor, -1, cv2.LINE_AA, 0) def __init__(self, trackbarName, windowName): self.trackbarName = trackbarName self.windowName = windowName cv2.namedWindow(windowName) cv2.createTrackbar(trackbarName, windowName, FeatureTrackerDrawer.trackedFeaturesPathLength, FeatureTrackerDrawer.maxTrackedFeaturesPathLength, self.onTrackBar) self.trackedIDs = set() self.trackedFeaturesPath = dict() # Create pipeline pipeline = dai.Pipeline() # Define sources and outputs monoLeft = pipeline.create(dai.node.MonoCamera) monoRight = pipeline.create(dai.node.MonoCamera) featureTrackerLeft = pipeline.create(dai.node.FeatureTracker) featureTrackerRight = pipeline.create(dai.node.FeatureTracker) xoutPassthroughFrameLeft = pipeline.create(dai.node.XLinkOut) xoutTrackedFeaturesLeft = pipeline.create(dai.node.XLinkOut) xoutPassthroughFrameRight = pipeline.create(dai.node.XLinkOut) xoutTrackedFeaturesRight = pipeline.create(dai.node.XLinkOut) xinTrackedFeaturesConfig = pipeline.create(dai.node.XLinkIn) xoutPassthroughFrameLeft.setStreamName("passthroughFrameLeft") xoutTrackedFeaturesLeft.setStreamName("trackedFeaturesLeft") xoutPassthroughFrameRight.setStreamName("passthroughFrameRight") xoutTrackedFeaturesRight.setStreamName("trackedFeaturesRight") xinTrackedFeaturesConfig.setStreamName("trackedFeaturesConfig") # Properties monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_720_P) monoLeft.setCamera("left") monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_720_P) monoRight.setCamera("right") # Linking monoLeft.out.link(featureTrackerLeft.inputImage) featureTrackerLeft.passthroughInputImage.link(xoutPassthroughFrameLeft.input) featureTrackerLeft.outputFeatures.link(xoutTrackedFeaturesLeft.input) xinTrackedFeaturesConfig.out.link(featureTrackerLeft.inputConfig) monoRight.out.link(featureTrackerRight.inputImage) featureTrackerRight.passthroughInputImage.link(xoutPassthroughFrameRight.input) featureTrackerRight.outputFeatures.link(xoutTrackedFeaturesRight.input) xinTrackedFeaturesConfig.out.link(featureTrackerRight.inputConfig) # By default the least mount of resources are allocated # increasing it improves performance numShaves = 2 numMemorySlices = 2 featureTrackerLeft.setHardwareResources(numShaves, numMemorySlices) featureTrackerRight.setHardwareResources(numShaves, numMemorySlices) featureTrackerConfig = featureTrackerRight.initialConfig.get() print("Press 's' to switch between Lucas-Kanade optical flow and hardware accelerated motion estimation!") # Connect to device and start pipeline with dai.Device(pipeline) as device: # Output queues used to receive the results passthroughImageLeftQueue = device.getOutputQueue("passthroughFrameLeft", 8, False) outputFeaturesLeftQueue = device.getOutputQueue("trackedFeaturesLeft", 8, False) passthroughImageRightQueue = device.getOutputQueue("passthroughFrameRight", 8, False) outputFeaturesRightQueue = device.getOutputQueue("trackedFeaturesRight", 8, False) inputFeatureTrackerConfigQueue = device.getInputQueue("trackedFeaturesConfig") leftWindowName = "left" leftFeatureDrawer = FeatureTrackerDrawer("Feature tracking duration (frames)", leftWindowName) rightWindowName = "right" rightFeatureDrawer = FeatureTrackerDrawer("Feature tracking duration (frames)", rightWindowName) while True: inPassthroughFrameLeft = passthroughImageLeftQueue.get() passthroughFrameLeft = inPassthroughFrameLeft.getFrame() leftFrame = cv2.cvtColor(passthroughFrameLeft, cv2.COLOR_GRAY2BGR) inPassthroughFrameRight = passthroughImageRightQueue.get() passthroughFrameRight = inPassthroughFrameRight.getFrame() rightFrame = cv2.cvtColor(passthroughFrameRight, cv2.COLOR_GRAY2BGR) trackedFeaturesLeft = outputFeaturesLeftQueue.get().trackedFeatures leftFeatureDrawer.trackFeaturePath(trackedFeaturesLeft) leftFeatureDrawer.drawFeatures(leftFrame) trackedFeaturesRight = outputFeaturesRightQueue.get().trackedFeatures rightFeatureDrawer.trackFeaturePath(trackedFeaturesRight) rightFeatureDrawer.drawFeatures(rightFrame) # Show the frame cv2.imshow(leftWindowName, leftFrame) cv2.imshow(rightWindowName, rightFrame) key = cv2.waitKey(1) if key == ord('q'): break elif key == ord('s'): if featureTrackerConfig.motionEstimator.type == dai.FeatureTrackerConfig.MotionEstimator.Type.LUCAS_KANADE_OPTICAL_FLOW: featureTrackerConfig.motionEstimator.type = dai.FeatureTrackerConfig.MotionEstimator.Type.HW_MOTION_ESTIMATION print("Switching to hardware accelerated motion estimation") else: featureTrackerConfig.motionEstimator.type = dai.FeatureTrackerConfig.MotionEstimator.Type.LUCAS_KANADE_OPTICAL_FLOW print("Switching to Lucas-Kanade optical flow") cfg = dai.FeatureTrackerConfig() cfg.set(featureTrackerConfig) inputFeatureTrackerConfigQueue.send(cfg) ``` #### C++ ```cpp #include // Includes common necessary includes for development using depthai library #include "depthai/depthai.hpp" #include "deque" #include "unordered_map" #include "unordered_set" static const auto lineColor = cv::Scalar(200, 0, 200); static const auto pointColor = cv::Scalar(0, 0, 255); class FeatureTrackerDrawer { private: static const int circleRadius = 2; static const int maxTrackedFeaturesPathLength = 30; // for how many frames the feature is tracked static int trackedFeaturesPathLength; using featureIdType = decltype(dai::TrackedFeature::id); std::unordered_set trackedIDs; std::unordered_map> trackedFeaturesPath; std::string trackbarName; std::string windowName; public: void trackFeaturePath(std::vector& features) { std::unordered_set newTrackedIDs; for(auto& currentFeature : features) { auto currentID = currentFeature.id; newTrackedIDs.insert(currentID); if(!trackedFeaturesPath.count(currentID)) { trackedFeaturesPath.insert({currentID, std::deque()}); } std::deque& path = trackedFeaturesPath.at(currentID); path.push_back(currentFeature.position); while(path.size() > std::max(1, trackedFeaturesPathLength)) { path.pop_front(); } } std::unordered_set featuresToRemove; for(auto& oldId : trackedIDs) { if(!newTrackedIDs.count(oldId)) { featuresToRemove.insert(oldId); } } for(auto& id : featuresToRemove) { trackedFeaturesPath.erase(id); } trackedIDs = newTrackedIDs; } void drawFeatures(cv::Mat& img) { cv::setTrackbarPos(trackbarName.c_str(), windowName.c_str(), trackedFeaturesPathLength); for(auto& featurePath : trackedFeaturesPath) { std::deque& path = featurePath.second; unsigned int j = 0; for(j = 0; j < path.size() - 1; j++) { auto src = cv::Point(path[j].x, path[j].y); auto dst = cv::Point(path[j + 1].x, path[j + 1].y); cv::line(img, src, dst, lineColor, 1, cv::LINE_AA, 0); } cv::circle(img, cv::Point(path[j].x, path[j].y), circleRadius, pointColor, -1, cv::LINE_AA, 0); } } FeatureTrackerDrawer(std::string trackbarName, std::string windowName) : trackbarName(trackbarName), windowName(windowName) { cv::namedWindow(windowName.c_str()); cv::createTrackbar(trackbarName.c_str(), windowName.c_str(), &trackedFeaturesPathLength, maxTrackedFeaturesPathLength, nullptr); } }; int FeatureTrackerDrawer::trackedFeaturesPathLength = 10; int main() { using namespace std; // Create pipeline dai::Pipeline pipeline; // Define sources and outputs auto monoLeft = pipeline.create(); auto monoRight = pipeline.create(); auto featureTrackerLeft = pipeline.create(); auto featureTrackerRight = pipeline.create(); auto xoutPassthroughFrameLeft = pipeline.create(); auto xoutTrackedFeaturesLeft = pipeline.create(); auto xoutPassthroughFrameRight = pipeline.create(); auto xoutTrackedFeaturesRight = pipeline.create(); auto xinTrackedFeaturesConfig = pipeline.create(); xoutPassthroughFrameLeft->setStreamName("passthroughFrameLeft"); xoutTrackedFeaturesLeft->setStreamName("trackedFeaturesLeft"); xoutPassthroughFrameRight->setStreamName("passthroughFrameRight"); xoutTrackedFeaturesRight->setStreamName("trackedFeaturesRight"); xinTrackedFeaturesConfig->setStreamName("trackedFeaturesConfig"); // Properties monoLeft->setResolution(dai::MonoCameraProperties::SensorResolution::THE_720_P); monoLeft->setCamera("left"); monoRight->setResolution(dai::MonoCameraProperties::SensorResolution::THE_720_P); monoRight->setCamera("right"); // Linking monoLeft->out.link(featureTrackerLeft->inputImage); featureTrackerLeft->passthroughInputImage.link(xoutPassthroughFrameLeft->input); featureTrackerLeft->outputFeatures.link(xoutTrackedFeaturesLeft->input); xinTrackedFeaturesConfig->out.link(featureTrackerLeft->inputConfig); monoRight->out.link(featureTrackerRight->inputImage); featureTrackerRight->passthroughInputImage.link(xoutPassthroughFrameRight->input); featureTrackerRight->outputFeatures.link(xoutTrackedFeaturesRight->input); xinTrackedFeaturesConfig->out.link(featureTrackerRight->inputConfig); // By default the least mount of resources are allocated // increasing it improves performance when optical flow is enabled auto numShaves = 2; auto numMemorySlices = 2; featureTrackerLeft->setHardwareResources(numShaves, numMemorySlices); featureTrackerRight->setHardwareResources(numShaves, numMemorySlices); auto featureTrackerConfig = featureTrackerRight->initialConfig.get(); printf("Press 's' to switch between Lucas-Kanade optical flow and hardware accelerated motion estimation! \n"); // Connect to device and start pipeline dai::Device device(pipeline); // Output queues used to receive the results auto passthroughImageLeftQueue = device.getOutputQueue("passthroughFrameLeft", 8, false); auto outputFeaturesLeftQueue = device.getOutputQueue("trackedFeaturesLeft", 8, false); auto passthroughImageRightQueue = device.getOutputQueue("passthroughFrameRight", 8, false); auto outputFeaturesRightQueue = device.getOutputQueue("trackedFeaturesRight", 8, false); auto inputFeatureTrackerConfigQueue = device.getInputQueue("trackedFeaturesConfig"); const auto leftWindowName = "left"; auto leftFeatureDrawer = FeatureTrackerDrawer("Feature tracking duration (frames)", leftWindowName); const auto rightWindowName = "right"; auto rightFeatureDrawer = FeatureTrackerDrawer("Feature tracking duration (frames)", rightWindowName); while(true) { auto inPassthroughFrameLeft = passthroughImageLeftQueue->get(); cv::Mat passthroughFrameLeft = inPassthroughFrameLeft->getFrame(); cv::Mat leftFrame; cv::cvtColor(passthroughFrameLeft, leftFrame, cv::COLOR_GRAY2BGR); auto inPassthroughFrameRight = passthroughImageRightQueue->get(); cv::Mat passthroughFrameRight = inPassthroughFrameRight->getFrame(); cv::Mat rightFrame; cv::cvtColor(passthroughFrameRight, rightFrame, cv::COLOR_GRAY2BGR); auto trackedFeaturesLeft = outputFeaturesLeftQueue->get()->trackedFeatures; leftFeatureDrawer.trackFeaturePath(trackedFeaturesLeft); leftFeatureDrawer.drawFeatures(leftFrame); auto trackedFeaturesRight = outputFeaturesRightQueue->get()->trackedFeatures; rightFeatureDrawer.trackFeaturePath(trackedFeaturesRight); rightFeatureDrawer.drawFeatures(rightFrame); // Show the frame cv::imshow(leftWindowName, leftFrame); cv::imshow(rightWindowName, rightFrame); int key = cv::waitKey(1); if(key == 'q') { break; } else if(key == 's') { if(featureTrackerConfig.motionEstimator.type == dai::FeatureTrackerConfig::MotionEstimator::Type::LUCAS_KANADE_OPTICAL_FLOW) { featureTrackerConfig.motionEstimator.type = dai::FeatureTrackerConfig::MotionEstimator::Type::HW_MOTION_ESTIMATION; printf("Switching to hardware accelerated motion estimation \n"); } else { featureTrackerConfig.motionEstimator.type = dai::FeatureTrackerConfig::MotionEstimator::Type::LUCAS_KANADE_OPTICAL_FLOW; printf("Switching to Lucas-Kanade optical flow \n"); } auto cfg = dai::FeatureTrackerConfig(); cfg.set(featureTrackerConfig); inputFeatureTrackerConfigQueue->send(cfg); } } return 0; } ``` ## Pipeline ### examples/feature_tracker.pipeline.json ```json {"pipeline": {"connections": [{"node1Id": 5, "node1Output": "out", "node1OutputGroup": "", "node2Id": 2, "node2Input": "inputConfig", "node2InputGroup": ""}, {"node1Id": 2, "node1Output": "outputFeatures", "node1OutputGroup": "", "node2Id": 8, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 1, "node1Output": "out", "node1OutputGroup": "", "node2Id": 2, "node2Input": "inputImage", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 6, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 1, "node2Input": "inputImage", "node2InputGroup": ""}], "globalProperties": {"calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1}, "nodes": [[8, {"alias": "", "id": 8, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 13, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_2_outputFeatures"}}], [6, {"alias": "", "id": 6, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 12, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_0_0"}}], [5, {"alias": "", "id": 5, "ioInfo": [[["", "out"], {"blocking": false, "group": "", "id": 11, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkIn", "parentId": -1, "properties": {"maxDataSize": 5242880, "numFrames": 8, "streamName": "__x_2__inputConfig"}}], [2, {"alias": "", "id": 2, "ioInfo": [[["", "passthroughInputImage"], {"blocking": false, "group": "", "id": 10, "name": "passthroughInputImage", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "outputFeatures"], {"blocking": false, "group": "", "id": 9, "name": "outputFeatures", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "inputImage"], {"blocking": false, "group": "", "id": 8, "name": "inputImage", "queueSize": 4, "type": 3, "waitForMessage": true}], [["", "inputConfig"], {"blocking": false, "group": "", "id": 7, "name": "inputConfig", "queueSize": 4, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "FeatureTracker", "parentId": -1, "properties": {"initialConfig": {"cornerDetector": {"cellGridDimension": 4, "enableSobel": true, "enableSorting": true, "numMaxFeatures": 256, "numTargetFeatures": 256, "thresholds": {"decreaseFactor": 0.8999999761581421, "increaseFactor": 1.100000023841858, "initialValue": 20000.0, "max": 0.0, "min": 0.0}, "type": 0}, "featureMaintainer": {"enable": true, "lostFeatureErrorThreshold": 50000.0, "minimumDistanceBetweenFeatures": 50.0, "trackedFeatureThreshold": 200000.0}, "motionEstimator": {"enable": true, "opticalFlow": {"epsilon": 0.009999999776482582, "maxIterations": 9, "pyramidLevels": 0, "searchWindowHeight": 5, "searchWindowWidth": 5}, "type": 0}}, "numMemorySlices": 2, "numShaves": 2}}], [1, {"alias": "", "id": 1, "ioInfo": [[["", "out"], {"blocking": false, "group": "", "id": 6, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "inputImage"], {"blocking": true, "group": "", "id": 5, "name": "inputImage", "queueSize": 3, "type": 3, "waitForMessage": false}], [["", "inputConfig"], {"blocking": true, "group": "", "id": 4, "name": "inputConfig", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "ImageManip", "parentId": -1, "properties": {"backend": 0, "initialConfig": {"base": {"background": 0, "backgroundB": 0, "backgroundG": 0, "backgroundR": 0, "center": true, "colormap": 0, "operations": [], "outputHeight": 0, "outputWidth": 0, "resizeMode": 0, "undistort": false}, "outputFrameType": 30, "reusePreviousImage": false, "skipCurrentImage": false}, "numFramesPool": 4, "outputFrameSize": 1048576, "performanceMode": 0}}], [0, {"alias": "", "id": 0, "ioInfo": [[["dynamicOutputs", "0"], {"blocking": false, "group": "dynamicOutputs", "id": 3, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "raw"], {"blocking": false, "group": "", "id": 2, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "mockIsp"], {"blocking": true, "group": "", "id": 1, "name": "mockIsp", "queueSize": 8, "type": 3, "waitForMessage": false}], [["", "inputControl"], {"blocking": true, "group": "", "id": 0, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "Camera", "parentId": -1, "properties": {"boardSocket": 0, "cameraName": "", "fps": -1.0, "imageOrientation": -1, "initialControl": {"aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": {"height": 0, "priority": 0, "width": 0, "x": 0, "y": 0}, "afRegion": {"height": 0, "priority": 0, "width": 0, "x": 0, "y": 0}, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "enableHdr": false, "expCompensation": 0, "expManual": {"exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0}, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "miscControls": [], "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": {"activeLevel": 0, "enable": 0, "gpioNumber": 0}, "strobeTimings": {"durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0}, "wbColorTemp": 0}, "isp3aFps": 0, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [{"enableUndistortion": null, "fps": {"value": null}, "resizeMode": 0, "size": {"value": {"index": 0, "value": [640, 640]}}, "type": 22}], "resolutionHeight": -1, "resolutionWidth": -1}}]]}} ``` ### Need assistance? Head over to [Discussion Forum](https://discuss.luxonis.com/) for technical support or any other questions you might have.