diff --git a/README.md b/README.md index 2b77fae..5d019dd 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,83 @@ # Lightly Studio Plugins -A collection of plugins for [Lightly Studio](https://github.com/lightly-ai/lightly-studio). Each plugin is independently pip-installable and auto-discovered via Python entry points. +A collection of installable plugins that extend the base functionality of [Lightly Studio](https://github.com/lightly-ai/lightly-studio). -| Plugin | Description | Maintainer | Install | -|---|---|---|---| -| [BBox auto propagation nano tracker](plugins/bbox_auto_propagation_nano_tracker/)|Auto bbox propagation using nano tracker|Lightly| `pip install git+https://github.com/lightly-ai/lightly-studio-plugins.git#subdirectory=plugins/bbox_auto_propagation_nano_tracker/`| -| [SAM3 Segmentation](plugins/sam3_segmentation/)|Automatic instance segmentation using SAM3 with a text prompt. Requires HuggingFace access to `facebook/sam3`.|Lightly| `pip install git+https://github.com/lightly-ai/lightly-studio-plugins.git#subdirectory=plugins/sam3_segmentation/`| -| [LightlyTrain object detection inference](plugins/lightly_train_object_detection_inference/)|LightlyTrain inference operator for object detection auto-labeling|Lightly| `pip install git+https://github.com/lightly-ai/lightly-studio-plugins.git#subdirectory=plugins/lightly_train_object_detection_inference/`| +Each plugin in this repository is packaged independently, installs in a single command, and is auto-discovered by Lightly Studio via Python entry points. -## Adding a New Plugin +![SAM3 Segmentation Plugin](plugins/sam3_segmentation/sam3_plugin.gif) + +

SAM3 Segmentation Plugin

+ +Each plugin entry below includes the exact copy-paste install command. After installation, the plugin is available in Lightly Studio automatically. + +## Available Plugins + +- [BBox auto propagation nano tracker](plugins/bbox_auto_propagation_nano_tracker/) + Propagates boxes from one annotated video frame to other frames in the same video. + +
+ Details + + If triggered from a frame, all bounding box annotations on that frame are + propagated. If triggered from an annotation, only the selected annotation is + propagated. + + - Scope: video only, within a single video + - Entry points: frame or annotation + - Controls: forward and backward propagation windows in seconds + - Tradeoff: uses OpenCV NanoTracker, which is lightweight and fast on many + machines but less robust on difficult motion, occlusion, or scale changes + - Maintainer: Lightly + - Install: + `pip install git+https://github.com/lightly-ai/lightly-studio-plugins.git#subdirectory=plugins/bbox_auto_propagation_nano_tracker/` + +
+ +- [SAM3 Segmentation](plugins/sam3_segmentation/) + Segments all instances matching a text prompt in a single image or across the current view. + +
+ Details + + This is designed for dataset-wide prompt-based labeling workflows with + class-like prompts such as `person`, `car`, or `dog`. + + - Scope: single image or images in the current view + - Input: text prompt + - Output: segmentation masks + - Labels: the prompt text is used as the annotation class name + - Requirement: Hugging Face access to `facebook/sam3` + - Maintainer: Lightly + - Install: + `pip install git+https://github.com/lightly-ai/lightly-studio-plugins.git#subdirectory=plugins/sam3_segmentation/` + +
+ +- [LightlyTrain object detection inference](plugins/lightly_train_object_detection_inference/) + Runs LightlyTrain object detection inference on one image or the current view for auto-labeling. + +
+ Details + + You can use built-in LightlyTrain models for quick bootstrapping or provide a + path to your own LightlyTrain checkpoint. + + - Scope: single image or images in the current view + - Input: LightlyTrain model name or local path to a LightlyTrain checkpoint + - Output: object detection annotations + - Labels: class labels are read from the loaded model and created in the + dataset if they do not exist yet + - Recommended models: + `dinov3/convnext-large-ltdetr-coco` for best performance, + `dinov3/vits16-ltdetr-coco` for a speed/quality balance, + `picodet-l-coco` for resource-constrained environments + - Maintainer: Lightly + - Install: + `pip install git+https://github.com/lightly-ai/lightly-studio-plugins.git#subdirectory=plugins/lightly_train_object_detection_inference/` + +
+ +## Contributing Plugins 1. Create a new directory under `plugins/`: ``` diff --git a/plugins.toml b/plugins.toml index 5d9f5df..a773371 100644 --- a/plugins.toml +++ b/plugins.toml @@ -1,7 +1,7 @@ [[plugins]] id = "lightly_plugins_bbox_auto_propagation_nano_tracker" name = "BBox auto propagation nano tracker" -description = "Auto bbox propagation using nano tracker" +description = "Propagates bounding boxes across frames in the same video using OpenCV NanoTracker" source = "local:plugins/bbox_auto_propagation_nano_tracker" maintainer = "lightly" tags = ["auto-labeling", "tracking"] @@ -9,7 +9,7 @@ tags = ["auto-labeling", "tracking"] [[plugins]] id = "lightly_plugins_sam3_segmentation" name = "SAM3 Segmentation" -description = "Automatic instance segmentation using SAM3 with a text prompt" +description = "Segments all instances matching a text prompt in a single image or the current image view" source = "local:plugins/sam3_segmentation" maintainer = "lightly" tags = ["auto-labeling", "segmentation"] @@ -17,7 +17,7 @@ tags = ["auto-labeling", "segmentation"] [[plugins]] id = "lightly_plugins_lightly_train_object_detection_inference" name = "LightlyTrain object detection inference" -description = "LightlyTrain inference operator for object detection auto-labeling" +description = "Runs LightlyTrain object detection inference on a single image or the current filtered image view" source = "local:plugins/lightly_train_object_detection_inference" maintainer = "lightly" tags = ["auto-labeling", "object-detection", "inference"] diff --git a/plugins/sam3_segmentation/sam3_plugin.gif b/plugins/sam3_segmentation/sam3_plugin.gif new file mode 100644 index 0000000..e748ebf Binary files /dev/null and b/plugins/sam3_segmentation/sam3_plugin.gif differ