diff --git a/.github/workflows/docs_and_notebooks_checks.yml b/.github/workflows/docs_and_notebooks_checks.yml index b35b68eda7..5e36962123 100644 --- a/.github/workflows/docs_and_notebooks_checks.yml +++ b/.github/workflows/docs_and_notebooks_checks.yml @@ -11,7 +11,7 @@ permissions: jobs: staleness: - name: Docs and notebooks scan (read-only) + name: Docs and notebooks scan (changed docs only) runs-on: ubuntu-latest timeout-minutes: 5 @@ -31,29 +31,75 @@ jobs: python -m pip install --upgrade pip python -m pip install "pydantic>=2,<3" pyyaml "nbformat>=5" + - name: Collect changed .md/.ipynb files + id: changed_docs + shell: bash + run: | + set -euo pipefail + mkdir -p tmp/docs_nb_checks + + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + base="${{ github.event.pull_request.base.sha }}" + head="${{ github.event.pull_request.head.sha }}" + else + base="${{ github.event.before }}" + head="${{ github.sha }}" + fi + + git diff --name-only --diff-filter=ACMR "$base" "$head" \ + | { grep -iE '\.(md|ipynb)$' || true; } \ + | { grep -E '^(docs/|tools/|examples/COLAB/|examples/JUPYTER/)' || true; } \ + | sort -u > tmp/docs_nb_checks/changed_docs.txt + + count=$(wc -l < tmp/docs_nb_checks/changed_docs.txt | tr -d ' ') + echo "count=$count" >> "$GITHUB_OUTPUT" + + echo "Changed docs files:" + if [[ "$count" -eq 0 ]]; then + echo "(none)" + else + sed 's/^/- /' tmp/docs_nb_checks/changed_docs.txt + fi + - name: Run staleness report (read-only) + if: steps.changed_docs.outputs.count != '0' + shell: bash run: | - python tools/docs_and_notebooks_check.py \ - --config tools/docs_and_notebooks_report_config.yml \ - --out-dir tmp/docs_nb_checks \ - report + set -euo pipefail + mapfile -t targets < tmp/docs_nb_checks/changed_docs.txt + python tools/docs_and_notebooks_check.py \ + --config tools/docs_and_notebooks_report_config.yml \ + --out-dir tmp/docs_nb_checks \ + report \ + --targets "${targets[@]}" - # Optional: run check mode (will fail only once you populate allowlists in config) - name: Run staleness policy check (optional gate) + if: steps.changed_docs.outputs.count != '0' continue-on-error: true + shell: bash run: | + set -euo pipefail + mapfile -t targets < tmp/docs_nb_checks/changed_docs.txt + python tools/docs_and_notebooks_check.py \ - --config tools/docs_and_notebooks_report_config.yml \ - --out-dir tmp/docs_nb_checks \ - --no-step-summary \ - check + --config tools/docs_and_notebooks_report_config.yml \ + --out-dir tmp/docs_nb_checks \ + --no-step-summary \ + check \ + --targets "${targets[@]}" + + - name: No changed docs to scan + if: steps.changed_docs.outputs.count == '0' + run: echo "No changed .md or .ipynb files found in the repo. Skipping scan." - name: Upload staleness artifacts + if: steps.changed_docs.outputs.count != '0' uses: actions/upload-artifact@v4 with: name: staleness-report path: | tmp/docs_nb_checks/*.json tmp/docs_nb_checks/*.md + tmp/docs_nb_checks/changed_docs.txt if-no-files-found: error diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a5956fde1e..b2c419b7a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -84,6 +84,15 @@ repos: args: [--check, --diff] stages: [manual] + - repo: https://github.com/hukkin/mdformat + rev: 1.0.0 + hooks: + - id: mdformat + additional_dependencies: + - mdformat-myst + files: ^docs/.*\.md$ + stages: [pre-commit] + # check only, no modifications - repo: local hooks: @@ -93,6 +102,7 @@ repos: language: python pass_filenames: true files: ^(docs/|examples/(JUPYTER|COLAB)/|tools/).*(\.md|\.ipynb)$ + exclude: ^tools/docs_audits/ args: - --config - tools/docs_and_notebooks_report_config.yml diff --git a/README.md b/README.md index 5c177728ad..3c2ca193cc 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,6 @@ - - - - - - [📚Documentation](https://deeplabcut.github.io/DeepLabCut/README.html) | [🛠️ Installation](https://deeplabcut.github.io/DeepLabCut/docs/installation.html) | [🌎 Home Page](https://www.deeplabcut.org) | @@ -57,7 +51,7 @@ **DeepLabCut™️** is a toolbox for state-of-the-art markerless pose estimation of animals performing various behaviors. As long as you can see (label) what you want to track, you can use this toolbox, as it is animal and object agnostic. [Read a short development and application summary below](https://github.com/DeepLabCut/DeepLabCut#why-use-deeplabcut). -# [Installation: how to install DeepLabCut](https://deeplabcut.github.io/DeepLabCut/docs/installation.html) +# [Installation](https://deeplabcut.github.io/DeepLabCut/docs/installation.html) Please click the link above for all the information you need to get started! Please note that currently we support only Python 3.10+ (see conda files for guidance). @@ -80,39 +74,46 @@ pip install --pre "deeplabcut[gui]" or `pip install --pre "deeplabcut"` (headless version with PyTorch)! -To use the TensorFlow (TF) engine (requires Python 3.10; TF up to v2.10 supported on Windows, -up to v2.12 on other platforms): you'll need to run `pip install "deeplabcut[gui,tf]"` -(which includes all functions plus GUIs) or `pip install "deeplabcut[tf]"` (headless -version with PyTorch and TensorFlow). We aim to depreciate the TF part in 2027. +To use the TensorFlow (TF) engine: you'll need to run `pip install "deeplabcut[gui,tf]"` or `pip install "deeplabcut[tf]"` (headless version with TF). Alternatively, we also offer more targeted optional TensorFlow installs for specific CUDA setups, e.g. `deeplabcut[tf-cu11]` or `deeplabcut[tf-cu12]`. Please refer to our [installation instructions](https://deeplabcut.github.io/DeepLabCut/docs/installation.html) for more detailed information on Python version, CUDA compatibility, etc. +We aim to **deprecate the tensorflow backend** in version 3.2 (release date TBD). -We recommend using our conda file, see [here](https://github.com/DeepLabCut/DeepLabCut/blob/main/conda-environments/README.md) or the [`deeplabcut-docker` package](https://github.com/DeepLabCut/DeepLabCut/tree/main/docker). -# [Documentation: The DeepLabCut Process](https://deeplabcut.github.io/DeepLabCut/README.html) + +# Documentation: The DeepLabCut Process Our docs walk you through using DeepLabCut, and key API points. For an overview of the toolbox and workflow for project management, see our step-by-step at [Nature Protocols paper](https://doi.org/10.1038/s41596-019-0176-0). -For a deeper understanding and more resources for you to get started with Python and DeepLabCut, please check out our free online course! https://deeplabcut.github.io/DeepLabCut/docs/course.html + +

-# [DEMO the code](examples/README.md) +# [Code demo](examples/README.md) -🐭 pose tracking of single animals demo [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/DeepLabCut/DeepLabCut/blob/master/examples/COLAB/COLAB_DEMO_mouse_openfield.ipynb) +🐭 Pose tracking of single animals demo [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/DeepLabCut/DeepLabCut/blob/master/examples/COLAB/COLAB_DEMO_mouse_openfield.ipynb) -See [more demos here](https://github.com/DeepLabCut/DeepLabCut/blob/main/examples/README.md). We provide data and several Jupyter Notebooks: one that walks you through a demo dataset to test your installation, and another Notebook to run DeepLabCut from the beginning on your own data. We also show you how to use the code in Docker, and on Google Colab. +See [more demos here](https://github.com/DeepLabCut/DeepLabCut/blob/main/examples/README.md). +We provide data and several Jupyter Notebooks, walking you through a demo dataset to test your installation, and another to run DeepLabCut from the start on your own data. +We also show how to use the code in Docker, and on Google Colab. # Why use DeepLabCut? -DeepLabCut continues to be actively maintained and we strive to provide a user-friendly `GUI` and `API` for computer vision researchers and life scientists alike. This means we integrate state-of-the-art models and frameworks, while providing our "best-guess" defaults for life scientists. We highly encourage you to read our papers to get a better understanding of what to use and how to modify the models for your setting. +DeepLabCut continues to be actively maintained and we strive to provide a user-friendly `GUI` and `API` for computer vision researchers and life scientists alike. This means we integrate state-of-the-art models and frameworks, while providing our "best-guess" defaults for life scientists. +We highly encourage you to read our papers to get a better understanding of what to use and how to modify the models for your setting. ## Performance 🔥 In general, we provide all the tooling for you to train and use custom models with various high-performance backbones. -We also provide two foundation pretrained animal models: `SuperAnimal-Quadruped`, `SuperAnimal-TopViewMouse`. To gauge their *out-of-distribution* performance, we provide the following tables. -These models are trained on the [SuperAnimal-Quadruped with AP-10K held out for out-of-domain testing]([https://cocodataset.org/](https://www.nature.com/articles/s41467-024-48792-2)) and the [SuperAnimal-TopViewMouse with DLC-openfield held out for out-of-distribution testing](https://www.nature.com/articles/s41467-024-48792-2). We provide models that include AP-10K in the API (and GUI). +## Pretrained Models + +We also provide two foundation pretrained animal models: `SuperAnimal-Quadruped` & `SuperAnimal-TopViewMouse`. +To gauge their *out-of-distribution* performance, we provide the following tables. + +These models are trained on the [SuperAnimal-Quadruped dataset](https://doi.org/10.5281/zenodo.10619172) with *AP-10K* held out for out-of-domain testing and the [SuperAnimal-TopViewMouse dataset](https://doi.org/10.5281/zenodo.13757509) with *DLC-openfield* held out for out-of-distribution testing (see [Ye et al. 2024](https://www.nature.com/articles/s41467-024-48792-2)). +We provide models that include AP-10K in the API (and GUI). Note, there are many different models to select from in DeepLabCut 3.0. We strongly recommend you check [this Guide](https://deeplabcut.github.io/DeepLabCut/docs/pytorch/architectures.html) for more details. This table, and those below, give you a sense of performance in real-world complex in-the-wild and lab mouse data, respectively. This [link provides the model weights](https://huggingface.co/mwmathis/DeepLabCutModelZoo-SuperAnimal-Quadruped) to reproduce the numbers; but please note, our `full` models are in our DLClibrary and released in the API. @@ -132,8 +133,13 @@ This [link provides the model weights](https://huggingface.co/mwmathis/DeepLabCu ## The History -In 2018, we demonstrated the capabilities for [trail tracking](https://vnmurthylab.org/), [reaching in mice](http://www.mousemotorlab.org/) and various Drosophila behaviors during egg-laying (see [Mathis et al.](https://www.nature.com/articles/s41593-018-0209-y) for details). There is, however, nothing specific that makes the toolbox only applicable to these tasks and/or species. The toolbox has already been successfully applied (by us and others) to [rats](http://www.mousemotorlab.org/deeplabcut), humans, various fish species, bacteria, leeches, various robots, cheetahs, [mouse whiskers](http://www.mousemotorlab.org/deeplabcut) and [race horses](http://www.mousemotorlab.org/deeplabcut). DeepLabCut utilized the feature detectors (ResNets + readout layers) of one of the state-of-the-art algorithms for human pose estimation by Insafutdinov et al., called DeeperCut, which inspired the name for our toolbox (see references below). Since this time, the package has changed substantially. The code has been re-tooled and re-factored since 2.1+: We have added faster and higher performance variants with MobileNetV2s, EfficientNets, and our own DLCRNet backbones (see [Pretraining boosts out-of-domain robustness for pose estimation](https://arxiv.org/abs/1909.11229) and [Lauer et al 2022](https://www.nature.com/articles/s41592-022-01443-0)). Additionally, we have improved the inference speed and provided both additional and novel augmentation methods, added real-time, and multi-animal support. -In v3.0+ we have changed the backend to support PyTorch. This brings not only an easier installation process for users, but performance gains, developer flexibility, and a lot of new tools! Importantly, the high-level API stays the same, so it will be a seamless transition for users 💜! +### Development and Applications + +In 2018, we demonstrated the capabilities for [trail tracking](https://vnmurthylab.org/), [reaching in mice](http://www.mousemotorlab.org/) and various Drosophila behaviors during egg-laying (see [Mathis et al.](https://www.nature.com/articles/s41593-018-0209-y) for details). There is, however, nothing specific that makes the toolbox only applicable to these tasks and/or species. +The toolbox has already been successfully applied (by us and others) to [rats](http://www.mousemotorlab.org/deeplabcut), humans, various fish species, bacteria, leeches, various robots, cheetahs, [mouse whiskers](http://www.mousemotorlab.org/deeplabcut) and [race horses](http://www.mousemotorlab.org/deeplabcut). +DeepLabCut utilized the feature detectors (ResNets + readout layers) of one of the state-of-the-art algorithms for human pose estimation by Insafutdinov et al., called DeeperCut, which inspired the name for our toolbox (see references below). Since this time, the package has changed substantially. +The code has been re-tooled and re-factored since 2.1+: We have added faster and higher performance variants with MobileNetV2s, EfficientNets, and our own DLCRNet backbones (see [Pretraining boosts out-of-domain robustness for pose estimation](https://arxiv.org/abs/1909.11229) and [Lauer et al 2022](https://www.nature.com/articles/s41592-022-01443-0)). Additionally, we have improved the inference speed and provided both additional and novel augmentation methods, added real-time, and multi-animal support. +In v3.0+ we have updated the backend to support PyTorch. This brings not only an easier installation process for users, but performance gains, developer flexibility, and a lot of new tools! Importantly, the high-level API stays the same, so it will be a seamless transition for users 💜! We currently provide state-of-the-art performance for animal pose estimation and the labs (M. Mathis Lab and A. Mathis Group) have both top journal and computer vision conference papers.

@@ -145,49 +151,51 @@ We currently provide state-of-the-art performance for animal pose estimation and **Left:** Due to transfer learning it requires **little training data** for multiple, challenging behaviors (see [Mathis et al. 2018](https://www.nature.com/articles/s41593-018-0209-y) for details). **Mid Left:** The feature detectors are robust to video compression (see [Mathis/Warren](https://www.biorxiv.org/content/early/2018/10/30/457242) for details). **Mid Right:** It allows 3D pose estimation with a single network and camera (see [Mathis/Warren](https://www.biorxiv.org/content/early/2018/10/30/457242)). **Right:** It allows 3D pose estimation with a single network trained on data from multiple cameras together with standard triangulation methods (see [Nath* and Mathis* et al. 2019](https://doi.org/10.1038/s41596-019-0176-0)). -**DeepLabCut** is embedding in a larger open-source eco-system, providing behavioral tracking for neuroscience, ecology, medical, and technical applications. Moreover, many new tools are being actively developed. See [DLC-Utils](https://github.com/DeepLabCut/DLCutils) for some helper code. +### Ecosystem + +**DeepLabCut** is part of a larger open-source eco-system, providing behavioral tracking for neuroscience, ecology, medical, and technical applications. +Moreover, many new tools are being actively developed. See [DLC-Utils](https://github.com/DeepLabCut/DLCutils) for some helper code.

-## Code contributors: +### Code contributors -DLC code was originally developed by [Alexander Mathis](https://github.com/AlexEMG) & [Mackenzie Mathis](https://github.com/MMathisLab), and was extended in 2.0 with the core dev team consisting of [Tanmay Nath](https://github.com/meet10may) (2.0-2.1), [Jessy Lauer](https://github.com/jeylau) (2.1-2.4), and [Niels Poulsen](https://github.com/n-poulsen) (2.3-3.0). -DeepLabCut is an open-source tool and has benefited from suggestions and edits by many individuals including early contributors: Mert Yuksekgonul, Tom Biasi, Richard Warren, Ronny Eichler, Hao Wu, Federico Claudi, Gary Kane and Jonny Saunders as well as the [100+ contributors](https://github.com/DeepLabCut/DeepLabCut/graphs/contributors). Please see [AUTHORS](https://github.com/DeepLabCut/DeepLabCut/blob/master/AUTHORS) for more details! +DeepLabCut was originally developed by [Alexander Mathis](https://github.com/AlexEMG) & [Mackenzie Mathis](https://github.com/MMathisLab), and was extended in 2.0 with the core dev team consisting of [Tanmay Nath](https://github.com/meet10may) (2.0-2.1), [Jessy Lauer](https://github.com/jeylau) (2.1-2.4), and [Niels Poulsen](https://github.com/n-poulsen) (2.3-3.0). +DeepLabCut is an open-source tool and has benefited from suggestions and edits by many individuals including early contributors: Mert Yuksekgonul, Tom Biasi, Richard Warren, Ronny Eichler, Hao Wu, Federico Claudi, Gary Kane and Jonny Saunders as well as the [100+ contributors](https://github.com/DeepLabCut/DeepLabCut/graphs/contributors). +Please see [AUTHORS](https://github.com/DeepLabCut/DeepLabCut/blob/master/AUTHORS) for more details! 🤩 This is an actively developed package and we welcome community development and involvement: [![Contributors](https://contrib.rocks/image?repo=DeepLabCut/DeepLabCut)](https://github.com/DeepLabCut/DeepLabCut/graphs/contributors) - - -# Get Assistance & be part of the DLC Community✨: +# Get Assistance & be part of the DLC Community✨ | 🚉 Platform | 🎯 Goal | ⏱️ Estimated Response Time | 📢 Support Squad | |------------------------------------------------------------|-----------------------------------------------------------------------------|---------------------------|----------------------------------------| -| GitHub DeepLabCut/[Issues](https://github.com/DeepLabCut/DeepLabCut/issues) | To report bugs and code issues🐛 (we encourage you to search issues first) | 2-5 days | DLC Core Dev Team | -| GitHub DeepLabCut/[Contributing](https://github.com/DeepLabCut/DeepLabCut/blob/master/CONTRIBUTING.md) | To contribute your expertise and experience🙏💯 | 2-5 days | DLC Core Dev Team | -| 🚧 GitHub DeepLabCut/[Roadmap](https://github.com/DeepLabCut/DeepLabCut/blob/master/docs/roadmap.md) | To learn more about our journey✈️ | N/A | N/A +| GitHub - [Issues](https://github.com/DeepLabCut/DeepLabCut/issues) | To report bugs and code issues🐛 (we encourage you to search issues first) | 2-5 days | DLC Core Dev Team | +| GitHub - [Contributing](https://github.com/DeepLabCut/DeepLabCut/blob/master/CONTRIBUTING.md) | To contribute your expertise and experience🙏💯 | 2-5 days | DLC Core Dev Team | | [![Image.sc forum](https://img.shields.io/badge/dynamic/json.svg?label=forum&url=https%3A%2F%2Fforum.image.sc%2Ftag%2Fdeeplabcut.json&query=%24.topic_list.tags.0.topic_count&colorB=brightgreen&&suffix=%20topics&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA4AAAAOCAYAAAAfSC3RAAABPklEQVR42m3SyyqFURTA8Y2BER0TDyExZ+aSPIKUlPIITFzKeQWXwhBlQrmFgUzMMFLKZeguBu5y+//17dP3nc5vuPdee6299gohUYYaDGOyyACq4JmQVoFujOMR77hNfOAGM+hBOQqB9TjHD36xhAa04RCuuXeKOvwHVWIKL9jCK2bRiV284QgL8MwEjAneeo9VNOEaBhzALGtoRy02cIcWhE34jj5YxgW+E5Z4iTPkMYpPLCNY3hdOYEfNbKYdmNngZ1jyEzw7h7AIb3fRTQ95OAZ6yQpGYHMMtOTgouktYwxuXsHgWLLl+4x++Kx1FJrjLTagA77bTPvYgw1rRqY56e+w7GNYsqX6JfPwi7aR+Y5SA+BXtKIRfkfJAYgj14tpOF6+I46c4/cAM3UhM3JxyKsxiOIhH0IO6SH/A1Kb1WBeUjbkAAAAAElFTkSuQmCC)](https://forum.image.sc/tag/deeplabcut)
🐭Tag: DeepLabCut | To ask help and support questions 👋 | Promptly🔥 | The DLC Community | |[![Gitter](https://badges.gitter.im/DeepLabCut/community.svg)](https://gitter.im/DeepLabCut/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) | To discuss with other users, share ideas and collaborate💡 | 2-5 days | The DLC Community | -| [BluSky🦋](https://bsky.app/profile/deeplabcut.bsky.social) | To keep up with our latest news and updates 📢 | 2-5 days | DLC Team | +| [![BlueSky](https://img.shields.io/badge/BlueSky-%40deeplabcut-blue?logo=bluesky)](https://bsky.app/profile/deeplabcut.bsky.social) | To keep up with our latest news and updates 📢 | 2-5 days | DLC Team | | [![Twitter Follow](https://img.shields.io/twitter/follow/DeepLabCut.svg?label=DeepLabCut&style=social)](https://x.com/DeepLabCut) | To keep up with our latest news and updates 📢 | 2-5 days | DLC Team | -| The DeepLabCut [AI Residency Program](https://www.deeplabcutairesidency.org/) | To come and work with us next summer👏 | Annually | DLC Team | + + -## References \& Citations: +## References \& Citations Please see our [dedicated page](https://deeplabcut.github.io/DeepLabCut/docs/citation.html) on how to **cite DeepLabCut** 🙏 and our suggestions for your Methods section! -## License: +## License This project is primarily licensed under the GNU Lesser General Public License v3.0. Note that the software is provided "as is", without warranty of any kind, express or implied. If you use the code or data, please cite us! Note, artwork (DeepLabCut logo) and images are copyrighted; please do not take or use these images without written permission. SuperAnimal models are provided for research use only (non-commercial use). -## Major Versions: +## Major Versions **For all versions, please see [here](https://github.com/DeepLabCut/DeepLabCut/releases).** @@ -202,18 +210,21 @@ This package includes graphical user interfaces to label your data, and take you VERSION 1.0: The initial, Nature Neuroscience version of [DeepLabCut](https://www.nature.com/articles/s41593-018-0209-y) can be found in the history of git, or here: https://github.com/DeepLabCut/DeepLabCut/releases/tag/1.11 -# News (and in the news): +# News + +## Major releases -:purple_heart: We released a major update, moving from 2.x --> 3.x with the backend change to PyTorch +💜 We released a major update, moving from 2.x --> 3.x with the backend change to PyTorch -:purple_heart: The DeepLabCut Model Zoo launches SuperAnimals, see more [here](https://deeplabcut.github.io/DeepLabCut/docs/ModelZoo.html). +💜 The DeepLabCut Model Zoo launches SuperAnimals, see more [here](https://deeplabcut.github.io/DeepLabCut/docs/ModelZoo.html). -:purple_heart: **DeepLabCut supports multi-animal pose estimation!** maDLC is out of beta/rc mode and beta is deprecated, thanks to the testers out there for feedback! Your labeled data will be backwards compatible, but not all other steps. Please see the [new `2.2+` releases](https://github.com/DeepLabCut/DeepLabCut/releases) for what's new & how to install it, please see our new [paper, Lauer et al 2022](https://www.nature.com/articles/s41592-022-01443-0), and the [new docs]( https://deeplabcut.github.io/DeepLabCut) on how to use it! +💜 **DeepLabCut supports multi-animal pose estimation!** maDLC is out of beta/rc mode and beta is deprecated, thanks to the testers out there for feedback! Your labeled data will be backwards compatible, but not all other steps. Please see the [new `2.2+` releases](https://github.com/DeepLabCut/DeepLabCut/releases) for what's new & how to install it, please see our new [paper, Lauer et al 2022](https://www.nature.com/articles/s41592-022-01443-0), and the [new docs]( https://deeplabcut.github.io/DeepLabCut) on how to use it! -:purple_heart: We support multi-animal re-identification, see [Lauer et al 2022](https://www.nature.com/articles/s41592-022-01443-0). +💜 We support multi-animal re-identification, see [Lauer et al 2022](https://www.nature.com/articles/s41592-022-01443-0). -:purple_heart: We have a **real-time** package available! http://DLClive.deeplabcut.org +💜 We have a **real-time** package available! [DLC-live on GitHub](https://github.com/DeepLabCut/DeepLabCut-live) and [DLC-live-GUI](https://github.com/DeepLabCut/DeepLabCut-live-GUI) +## In the news - June 2024: Our second DLC paper ['Using DeepLabCut for 3D markerless pose estimation across species and behaviors'](https://www.nature.com/articles/s41596-019-0176-0) in Nature Protocols has surpassed 1,000 Google Scholar citations! - May 2024: DeepLabCut was featured in Nature: ['DeepLabCut: the motion-tracking tool that went viral'](https://www.nature.com/articles/d41586-024-01474-x) @@ -251,6 +262,8 @@ importing a project into the new data format for DLC 2.0 - July 2018: Ed Yong covered DeepLabCut and interviewed several users for the [Atlantic](https://www.theatlantic.com/science/archive/2018/07/deeplabcut-tracking-animal-movements/564338). - April 2018: first DeepLabCut preprint on [arXiv.org](https://arxiv.org/abs/1804.03142) - ## Funding + # Funding - We are grateful for the follow support over the years! This software project was supported in part by the Essential Open Source Software for Science (EOSS) program at Chan Zuckerberg Initiative (cycles 1, 3, 3-DEI, 4), and jointly with the Kavli Foundation for EOSS Cycle 6! We also thank the Rowland Institute at Harvard for funding from 2017-2020, and EPFL from 2020-present. +We are grateful for the following support and funding over the years! +This software project was supported in part by the **Essential Open Source Software for Science (EOSS)** program at **Chan Zuckerberg Initiative** (cycles 1, 3, 3-DEI, 4), and jointly with the **Kavli Foundation** for **EOSS Cycle 6**! +We also thank the **Rowland Institute** at **Harvard** for funding from 2017-2020, and **EPFL** from 2020-present. diff --git a/_toc.yml b/_toc.yml index 6bb51801ff..6da12c3416 100644 --- a/_toc.yml +++ b/_toc.yml @@ -2,125 +2,120 @@ format: jb-book root: README parts: -- caption: Getting Started - chapters: - - file: docs/UseOverviewGuide - - file: docs/course - -- caption: Installation - chapters: - - file: docs/installation - - file: docs/recipes/installTips - - file: docs/docker - -- caption: Main User Guides - chapters: - - file: docs/standardDeepLabCut_UserGuide - - file: docs/maDLC_UserGuide - - file: docs/Overviewof3D - - file: docs/HelperFunctions - -- caption: Graphical User Interfaces (GUIs) - chapters: - - file: docs/gui/PROJECT_GUI - - file: docs/gui/napari_GUI - sections: - - file: docs/gui/napari/basic_usage - - file: docs/gui/napari/advanced_usage - -- caption: DLC3 PyTorch Specific Docs - chapters: - - file: docs/pytorch/user_guide.md - - file: docs/pytorch/pytorch_config.md - - file: docs/pytorch/architectures.md - -- caption: Quick Start Tutorials - chapters: - - file: docs/quick-start/single_animal_quick_guide - - file: docs/quick-start/tutorial_maDLC - -- caption: "🚀 Beginner's Guide to DeepLabCut" - chapters: - - file: docs/beginner-guides/beginners-guide - - file: docs/beginner-guides/manage-project - - file: docs/beginner-guides/labeling - - file: docs/beginner-guides/Training-Evaluation - - file: docs/beginner-guides/video-analysis - -- caption: "🚀 Main Demo Notebooks" - chapters: - - file: examples/COLAB/COLAB_DEMO_SuperAnimal - - file: examples/COLAB/COLAB_DEMO_mouse_openfield - - file: examples/COLAB/COLAB_3miceDemo - - file: examples/COLAB/COLAB_HumanPose_with_RTMPose - -- caption: "🚀 Notebooks For Your Data" - chapters: - - file: examples/COLAB/COLAB_YOURDATA_SuperAnimal - - file: examples/COLAB/COLAB_YOURDATA_TrainNetwork_VideoAnalysis - - file: examples/COLAB/COLAB_YOURDATA_maDLC_TrainNetwork_VideoAnalysis - -- caption: "🚀 Special Feature Demos" - chapters: - - file: examples/COLAB/COLAB_transformer_reID - - file: examples/COLAB/COLAB_BUCTD_and_CTD_tracking - - file: examples/JUPYTER/Demo_3D_DeepLabCut - - file: examples/COLAB/COLAB_DLC_ModelZoo - -- caption: "🧑‍🍳 Cookbook (detailed helper guides)" - chapters: - - file: docs/convert_maDLC - - file: docs/recipes/OtherData - - file: docs/recipes/io - - file: docs/recipes/nn - - file: docs/recipes/post - - file: docs/recipes/BatchProcessing - - file: docs/recipes/DLCMethods - - file: docs/recipes/ClusteringNapari - - file: docs/recipes/OpenVINO - - file: docs/recipes/flip_and_rotate - - file: docs/recipes/pose_cfg_file_breakdown - - file: docs/recipes/publishing_notebooks_into_the_DLC_main_cookbook - -- caption: Hardware Tips - chapters: - - file: docs/recipes/TechHardware - -- caption: DeepLabCut-Live! - chapters: - - file: docs/dlc-live/deeplabcutlive - - file: docs/dlc-live/dlc-live-gui/index - sections: - - file: docs/dlc-live/dlc-live-gui/quickstart/install - - file: docs/dlc-live/dlc-live-gui/user_guide/overview - - file: docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/camera_support + - caption: Getting started + chapters: + - file: docs/UseOverviewGuide + - file: docs/installation sections: - - file: docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/opencv_backend - - file: docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/basler_backend - - file: docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/aravis_backend - - file: docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/gentl_backend - - file: docs/dlc-live/dlc-live-gui/user_guide/misc/misc_landing + # - file: docs/recipes/installTips + - file: docs/docker + # - file: docs/quick-start/index + # sections: + + - caption: Main workflows overview + chapters: + - file: docs/main-workflows/user-guide sections: - - file: docs/dlc-live/dlc-live-gui/user_guide/misc/modelzoo_downloads - - file: docs/dlc-live/dlc-live-gui/user_guide/misc/timestamp_format - -- caption: "🦄 DeepLabCut Model Zoo" - chapters: - - file: docs/ModelZoo - - file: docs/recipes/UsingModelZooPupil - -- caption: DeepLabCut Benchmarking - chapters: - - file: docs/benchmark - - file: docs/pytorch/Benchmarking_shuffle_guide - -- caption: "Mission & Contribute" - chapters: - - file: docs/MISSION_AND_VALUES - - file: docs/roadmap - - file: docs/Governance - - file: CONTRIBUTING + - file: docs/quick-start/single_animal_quick_guide + - file: docs/quick-start/tutorial_maDLC + - file: docs/main-workflows/multi-animal-tracking + # - file: docs/standardDeepLabCut_UserGuide + # - file: docs/maDLC_UserGuide + - file: docs/Overviewof3D + + - caption: GUI workflow + chapters: + - file: docs/gui/PROJECT_GUI + sections: + - file: docs/beginner-guides/beginners-guide + - file: docs/beginner-guides/manage-project + - file: docs/beginner-guides/labeling + - file: docs/beginner-guides/Training-Evaluation + - file: docs/beginner-guides/video-analysis + - file: docs/gui/napari_GUI + sections: + - file: docs/gui/napari/basic_usage + - file: docs/gui/napari/advanced_usage + - file: docs/gui/napari/tracking/basic_usage -- caption: Citations for DeepLabCut - chapters: - - file: docs/citation + - caption: Notebooks & Demos + chapters: + - file: docs/notebooks/your_data + sections: + - file: examples/COLAB/COLAB_YOURDATA_SuperAnimal + - file: examples/COLAB/COLAB_YOURDATA_TrainNetwork_VideoAnalysis + - file: examples/COLAB/COLAB_YOURDATA_maDLC_TrainNetwork_VideoAnalysis + - file: docs/notebooks/main_demos + sections: + - file: examples/COLAB/COLAB_DEMO_SuperAnimal + - file: examples/COLAB/COLAB_DEMO_mouse_openfield + - file: examples/COLAB/COLAB_3miceDemo + - file: examples/COLAB/COLAB_HumanPose_with_RTMPose + - file: docs/notebooks/extra + sections: + - file: examples/COLAB/COLAB_transformer_reID + - file: examples/COLAB/COLAB_BUCTD_and_CTD_tracking + - file: examples/JUPYTER/Demo_3D_DeepLabCut + - file: examples/COLAB/COLAB_DLC_ModelZoo + + - caption: DeepLabCut 3.0 - PyTorch guides + chapters: + - file: docs/pytorch/index + sections: + - file: docs/pytorch/user_guide.md + - file: docs/pytorch/pytorch_config.md + - file: docs/pytorch/architectures.md + - file: docs/pytorch/Benchmarking_shuffle_guide + + - caption: Advanced, Performance & Live + chapters: + - file: docs/ModelZoo + sections: + - file: docs/recipes/UsingModelZooPupil + - file: docs/dlc-live/deeplabcutlive + - file: docs/dlc-live/dlc-live-gui/index + sections: + - file: docs/dlc-live/dlc-live-gui/quickstart/install + - file: docs/dlc-live/dlc-live-gui/user_guide/overview + - file: docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/camera_support + sections: + - file: docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/opencv_backend + - file: docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/basler_backend + - file: docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/aravis_backend + - file: docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/gentl_backend + - file: docs/dlc-live/dlc-live-gui/user_guide/misc/misc_landing + sections: + - file: docs/dlc-live/dlc-live-gui/user_guide/misc/modelzoo_downloads + - file: docs/dlc-live/dlc-live-gui/user_guide/misc/timestamp_format + - file: docs/benchmark + - file: docs/recipes/TechHardware + + - caption: Additional guides (Recipes) + chapters: + - file: docs/recipes/index + sections: + - file: docs/recipes/publishing_notebooks_into_the_DLC_main_cookbook + - file: docs/HelperFunctions + - file: docs/convert_maDLC + - file: docs/recipes/OtherData + - file: docs/recipes/io + - file: docs/recipes/nn + - file: docs/recipes/post + - file: docs/recipes/BatchProcessing + - file: docs/recipes/DLCMethods + - file: docs/recipes/ClusteringNapari + - file: docs/recipes/OpenVINO + - file: docs/recipes/flip_and_rotate + - file: docs/recipes/pose_cfg_file_breakdown + # - file: docs/course + - file: docs/dlc-utils/index + sections: + - file: docs/dlc-utils/XROMM/usage + + - caption: Project & Community + chapters: + - file: docs/MISSION_AND_VALUES + - file: docs/roadmap + - file: docs/Governance + - file: CONTRIBUTING + - file: docs/citation diff --git a/docs/Governance.md b/docs/Governance.md index 2ee2b1b172..204d253608 100644 --- a/docs/Governance.md +++ b/docs/Governance.md @@ -4,8 +4,11 @@ deeplabcut: last_metadata_updated: '2026-03-06' ignore: false --- + (governance-model)= + # Governance Model of DeepLabCut + (adapted from https://napari.org/stable/community/governance.html) ## Abstract @@ -112,7 +115,7 @@ DeepLabCut uses a “consensus seeking” process for making decisions. The grou tries to find a resolution that has no open objections among core developers. Core developers are expected to distinguish between fundamental objections to a proposal and minor perceived flaws that they can live with, and not hold up the -decision-making process for the latter. If no option can be found without +decision-making process for the latter. If no option can be found without objections, the decision is escalated to the SC, which will itself use consensus seeking to come to a resolution. In the unlikely event that there is still a deadlock, the proposal will move forward if it has the support of a @@ -139,7 +142,7 @@ are made according to the following rules: decision-making process outlined above. - **Changes to this governance model or our mission, vision, and values** - require a dedicated issue on our [issue tracker](https://github.com/DeepLabCut/DeepLabCut/issues) + require a dedicated issue on our [issue tracker](https://github.com/DeepLabCut/DeepLabCut/issues) and follow the decision-making process outlined above, *unless* there is unanimous agreement from core developers on the change in which case it can move forward faster. diff --git a/docs/HelperFunctions.md b/docs/HelperFunctions.md index aa90f91975..05c08f8ad0 100644 --- a/docs/HelperFunctions.md +++ b/docs/HelperFunctions.md @@ -3,27 +3,32 @@ deeplabcut: last_content_updated: '2025-06-30' last_metadata_updated: '2026-03-06' ignore: false + visibility: online + status: outdated + recommendation: archive + notes: I would suggest using API docs over pages like this to avoid drift. The advice below promises updates that are not being made, and the content is already quite outdated. Automating API docs generation and putting usage info for obtaining commands info in ipython in a basic 'evergreen' page would be more sustainable than trying to maintain this page. --- + (helper-functions)= + # Helper & Advanced Optional Function Documentation There are additional functions that are not required, but can be extremely helpful. First off, if you are new to Python, you might not know this handy trick: you can see -ALL the functions in deeplabcut by typing ``deeplabcut.`` then hitting "tab." You will see a massive list! +ALL the functions in deeplabcut by typing `deeplabcut.` then hitting "tab." You will see a massive list!

-Or perhaps you sort of know the name of the function, but not fully, then you can start typing the command, i.e. as in ``deeplabcut.a `` then hit tab: +Or perhaps you sort of know the name of the function, but not fully, then you can start typing the command, i.e. as in `deeplabcut.a ` then hit tab:

- -Now, for any of these functions, you type ``deeplabcut.analyze_videos_converth5_to_csv?`` you get: +Now, for any of these functions, you type `deeplabcut.analyze_videos_converth5_to_csv?` you get: ```text Signature: deeplabcut.analyze_videos_converth5_to_csv(videopath, videotype='.avi') @@ -52,8 +57,7 @@ Only videos with this extension are analyzed. The default is ``.avi`` While some of the names are ridiculously long, we wanted them to be "self-explanatory." Here is a list (that is bound to be continually updated) of currently available helper functions. To see information about any of them, including HOW -to use them, use the ``?`` at the end of the call, as described above. - +to use them, use the `?` at the end of the call, as described above. ```python deeplabcut.analyze_videos_converth5_to_csv @@ -108,18 +112,23 @@ In order to label with epipolar lines, you must complete two additional sets of steps 1-3 in [3D Overview](3D-overview). - Second, you must extract imagr from `camera_1` first; here you would have run the standard `deeplabcut.extract_frames(config_path, userfeedback=True)`, but just extract files from 1 camera. Next, you need to extract matching frames from `camera_2`: + ```python deeplabcut.extract_frames(config_path, mode = 'match', config3d=config_path3d, extracted_cam=0) ``` + You can set `extracted_cam=0` to match all other camera images to the frame numbers in the `camera_1` folder, or change this to match to other cameras. If you `deeplabcut.extract_frames` with `mode='automatic'` before, it shouldn't matter which camera you pick. If you already extracted from both cameras, be warned this will overwrite the images for `camera_2`. - Three, now you can label with epipolar lines: - - Here, label `camera_1` as you would normally, i.e.: - ```python - deeplabcut.label_frames(config_path) - ``` - - Then for `camera_2` (now it will compute the epipolar lines based on camera_1 labels and project them onto the GUI): - ```python - deeplabcut.label_frames(config_path, config3d=config_path3d) - ``` + - Here, label `camera_1` as you would normally, i.e.: + + ```python + deeplabcut.label_frames(config_path) + ``` + + - Then for `camera_2` (now it will compute the epipolar lines based on camera_1 labels and project them onto the GUI): + + ```python + deeplabcut.label_frames(config_path, config3d=config_path3d) + ``` diff --git a/docs/MISSION_AND_VALUES.md b/docs/MISSION_AND_VALUES.md index bc6623a6af..80630ed2fc 100644 --- a/docs/MISSION_AND_VALUES.md +++ b/docs/MISSION_AND_VALUES.md @@ -4,7 +4,9 @@ deeplabcut: last_metadata_updated: '2026-03-06' ignore: false --- + (mission-and-values)= + # Mission and Values of DeepLabCut This document is meant to help guide decisions about the future of `DeepLabCut`, be it in terms of @@ -25,7 +27,6 @@ estimation framework that is: - fast (GPU-powered) - scalable (project focused for ease of portability and sharability) - As the project has grown we've turned these original principles into the mission statement and set of values that we described below. @@ -36,44 +37,44 @@ pose estimation for people to use in their daily work** without the need to be a framework. We hope to accomplish this by: - being **easy to use and install**. We are careful in taking on new dependencies, sometimes making them optional, and -aim support a fully (Python) packaged installation that works cross-platform. + aim support a fully (Python) packaged installation that works cross-platform. - being **well-documented** with **comprehensive tutorials and examples**. All functions in our API have thorough -docstrings clarifying expected inputs and outputs, and we maintain a separate -[tutorials and information website](http://deeplabcut.org). + docstrings clarifying expected inputs and outputs, and we maintain a separate + [tutorials and information website](http://deeplabcut.org). - providing **GUI access** to all critical functionality so DeepLabCut can be used by people without coding experience. - being **interactive** and **highly performant** in order to support large data pipelines. - providing a **consistent and stable API** to enable plugin developers to build on top of DeepLabCut without their -code constantly breaking and to enable advanced users to build out sophisticated Python workflows, if needed. + code constantly breaking and to enable advanced users to build out sophisticated Python workflows, if needed. - **ensuring correctness**. We strive for complete test coverage of both the code and GUI, with all code reviewed by a -core developer before being included in the repository. + core developer before being included in the repository. ## Our values - We are **inclusive**. We welcome newcomers who are making their first contribution and strive to grow our most -dedicated contributors into [core developers](https://github.com/orgs/DeepLabCut/teams/core-developers). -We have a [Code of Conduct](https://github.com/DeepLabCut/DeepLabCut/blob/main/CODE_OF_CONDUCT.md) to make DeepLabCut -a welcoming place for all. + dedicated contributors into [core developers](https://github.com/orgs/DeepLabCut/teams/core-developers). + We have a [Code of Conduct](https://github.com/DeepLabCut/DeepLabCut/blob/main/CODE_OF_CONDUCT.md) to make DeepLabCut + a welcoming place for all. - We are **community-engaged**. We respond to feature requests and proposals on our + - [issue tracker](https://github.com/DeepLabCut/DeepLabCut/issues). - We serve **scientific applications** primarily, over “consumer or commercial” pose estimation tools. This often means -prioritizing core functionality support, and rejecting implementations of “flashy” features that have little -scientific value. + prioritizing core functionality support, and rejecting implementations of “flashy” features that have little + scientific value. - We are **domain agnostic** within the sciences. Functionality that is highly specific to particular scientific -domains belongs in plugins, whereas functionality that cuts across many domains and is likely to be widely used belongs -inside DeepLabCut. + domains belongs in plugins, whereas functionality that cuts across many domains and is likely to be widely used belongs + inside DeepLabCut. - We value **education and documentation**. All functions should have docstrings, preferably with examples, and major -functionality should be explained in our [tutorials](http://deeplabcut.org). Core developers can take an active role -in finishing documentation examples. - + functionality should be explained in our [tutorials](http://deeplabcut.org). Core developers can take an active role + in finishing documentation examples. ## Acknowledgements diff --git a/docs/ModelZoo.md b/docs/ModelZoo.md index 9d37486afc..c10220c313 100644 --- a/docs/ModelZoo.md +++ b/docs/ModelZoo.md @@ -4,25 +4,26 @@ deeplabcut: last_metadata_updated: '2026-03-06' ignore: false --- + (file:model-zoo)= + # The DeepLabCut Model Zoo! ![image](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/8957c690-4f27-4430-8581-4161fd58d052/68747470733a2f2f696d616765732e73717561726573706163652d63646e2e636f6d2f636f6e74656e742f76312f3537663664353163396637343536366635356563663237312f313631363439323337333730302d50474f41433732494f4236415545343756544a582f6b6531375a77644742546f646449.png?format=450w) - ## 🏠 [Home page](http://modelzoo.deeplabcut.org/) - Started in 2020, expanded in 2022 with PhD student [Shaokai Ye et al.](https://arxiv.org/abs/2203.07436v1), and the first proper [SuperAnimal Foundation Models](#about-the-superanimal-models) published in 2024 🔥, the Model Zoo is four things: - (1) a collection of models that are trained on diverse data across (typically) large datasets, which means you do not need to train models yourself, rather you can use them in your research applications. - (2) a contribution website for community crowd sourcing of expertly labeled keypoints to improve models! You can get involved here: [contrib.deeplabcut.org](https://contrib.deeplabcut.org/). - (3) a no-install DeepLabCut that you can use on ♾[Google Colab](https://github.com/DeepLabCut/DeepLabCut/blob/main/examples/COLAB/COLAB_DEMO_SuperAnimal.ipynb), -test our models in 🕸[the browser](https://contrib.deeplabcut.org/), or on our 🤗[HuggingFace](https://huggingface.co/spaces/DeepLabCut/DeepLabCutModelZoo-SuperAnimals) app! + test our models in 🕸[the browser](https://contrib.deeplabcut.org/), or on our 🤗[HuggingFace](https://huggingface.co/spaces/DeepLabCut/DeepLabCutModelZoo-SuperAnimals) app! - (4) new methods to make SuperAnimal Foundation Models that combine data across different labs/datasets, keypoints, animals/species, and use on your data! ## Quick Start: + ``` pip install deeplabcut[gui,modelzoo] ``` @@ -34,52 +35,54 @@ Animal pose estimation is critical in applications ranging from neuroscience to To provide the community with easy access to such high performance models across diverse environments and species, we present a new paradigm for building pre-trained animal pose models -- which we call SuperAnimal models -- and the ability to use them for transfer learning (e.g., fine-tune them if needed). ## SuperAnimal members: -- Models are based on what they are trained on, for example `superanimal_quadruped_x` is trained on [SuperAnimal-Quadruped-80K](https://zenodo.org/records/10619173). Each model class is described below: - +- Models are based on what they are trained on, for example `superanimal_quadruped_x` is trained on [SuperAnimal-Quadruped-80K](https://zenodo.org/records/10619173). Each model class is described below: ### SuperAnimal-Quadruped: - - `superanimal_quadruped_x` models aim to work across a large range of quadruped animals, from horses, dogs, sheep, rodents, to elephants. The camera perspective is orthogonal to the animal ("side view"), and most of the data includes the animals face (thus the front and side of the animal). You will note we have several variants that differ in speed vs. performance, so please do test them out on your data to see which is best suited for your application. Also note we have a "video adaptation" feature, which lets you adapt your data to the model in a self-supervised way. No labeling needed! + - [Please see the full datasheet here](https://zenodo.org/records/10619173) + - [More details on the models (detector, pose estimators)](https://huggingface.co/mwmathis/DeepLabCutModelZoo-SuperAnimal-Quadruped) -- We provide several models: - - `superanimal_quadruped_hrnetw32` (pytorch engine) - - `superanimal_quadruped_hrnetw32` is a top-down model that is paired with a detector. That means it takes a cropped image from an object detector and predicts the keypoints. The object detector is currently a trained [ResNet50-based Faster-RCNN](https://pytorch.org/vision/stable/models/faster_rcnn.html). - - `superanimal_quadruped_dlcrnet` (tensorflow engine) - - `superanimal_quadruped_dlcrnet` is a bottom-up model that predicts all keypoints, then groups them into individuals. This can be faster, but more error prone. - - `superanimal_quadruped` -> This is the same as `superanimal_quadruped_dlcrnet`, this was the old naming and being depreciated. - - For all models, they are automatically downloaded to modelzoo/checkpoints when used. -- Here are example images of what the model is trained on: -![SA_Q](https://user-images.githubusercontent.com/28102185/209957688-954fb616-7750-4521-bb52-20a51c3a7718.png) +- We provide several models: + - `superanimal_quadruped_hrnetw32` (pytorch engine) + - `superanimal_quadruped_hrnetw32` is a top-down model that is paired with a detector. That means it takes a cropped image from an object detector and predicts the keypoints. The object detector is currently a trained [ResNet50-based Faster-RCNN](https://pytorch.org/vision/stable/models/faster_rcnn.html). + - `superanimal_quadruped_dlcrnet` (tensorflow engine) + - `superanimal_quadruped_dlcrnet` is a bottom-up model that predicts all keypoints, then groups them into individuals. This can be faster, but more error prone. + - `superanimal_quadruped` -> This is the same as `superanimal_quadruped_dlcrnet`, this was the old naming and being depreciated. + - For all models, they are automatically downloaded to modelzoo/checkpoints when used. +- Here are example images of what the model is trained on: + ![SA_Q](https://user-images.githubusercontent.com/28102185/209957688-954fb616-7750-4521-bb52-20a51c3a7718.png) ### SuperAnimal-TopViewMouse: +- `superanimal_topviewmouse_x` aims to work across lab mice in different lab settings from a top-view perspective; this is very polar in many behavioral assays in freely moving mice. -- `superanimal_topviewmouse_x` aims to work across lab mice in different lab settings from a top-view perspective; this is very polar in many behavioral assays in freely moving mice. - [Please see the full datasheet here](https://zenodo.org/records/10618947) + - [More details on the models (detector, pose estimators)](https://huggingface.co/mwmathis/DeepLabCutModelZoo-SuperAnimal-TopViewMouse) + - We provide several models: - - `superanimal_topviewmouse_hrnetw32` (pytorch engine) - - `superanimal_topviewmouse_hrnetw32` is a top-down model that is paired with a detector. That means it takes a cropped image from an object detector and predicts the keypoints. The object detector is currently a trained [ResNet50-based Faster-RCNN](https://pytorch.org/vision/stable/models/faster_rcnn.html). - - `superanimal_topviewmouse_dlcrnet` (tensorflow engine) - - `superanimal_topviewmouse_dlcrnet` is a bottom-up model that predicts all keypoints then groups them into individuals. This can be faster, but more error prone. - - `superanimal_topviewmouse` -> This is the same as `superanimal_topviewmouse_dlcrnet`, this was the old naming and being depreciated. - - For all models, they are automatically downloaded to modelzoo/checkpoints when used. -- Here are example images of what the model is trained on: -![SA-TVM](https://user-images.githubusercontent.com/28102185/209957260-c0db72e0-4fdf-434c-8579-34bc5f27f907.png) + - `superanimal_topviewmouse_hrnetw32` (pytorch engine) + - `superanimal_topviewmouse_hrnetw32` is a top-down model that is paired with a detector. That means it takes a cropped image from an object detector and predicts the keypoints. The object detector is currently a trained [ResNet50-based Faster-RCNN](https://pytorch.org/vision/stable/models/faster_rcnn.html). + - `superanimal_topviewmouse_dlcrnet` (tensorflow engine) + - `superanimal_topviewmouse_dlcrnet` is a bottom-up model that predicts all keypoints then groups them into individuals. This can be faster, but more error prone. + - `superanimal_topviewmouse` -> This is the same as `superanimal_topviewmouse_dlcrnet`, this was the old naming and being depreciated. + - For all models, they are automatically downloaded to modelzoo/checkpoints when used. + +- Here are example images of what the model is trained on: + ![SA-TVM](https://user-images.githubusercontent.com/28102185/209957260-c0db72e0-4fdf-434c-8579-34bc5f27f907.png) ### SuperAnimal-Human: - `superanimal_humanbody` models aim to work across human body pose estimation from various camera perspectives and environments. The models are designed to handle different human poses, activities, and lighting conditions commonly found in human motion analysis, sports analysis, and behavioral studies. - - `superanimal_humanbody_rtmpose_x` (pytorch engine) - - `superanimal_humanbody_rtmpose_x` is a top-down model that is paired with a detector pretrained from `torchvision`. That means it takes a cropped image from an object detector and predicts the keypoints. This model uses 17 body parts in the COCO body7 format. - + - `superanimal_humanbody_rtmpose_x` (pytorch engine) + - `superanimal_humanbody_rtmpose_x` is a top-down model that is paired with a detector pretrained from `torchvision`. That means it takes a cropped image from an object detector and predicts the keypoints. This model uses 17 body parts in the COCO body7 format. ### Practical example: Using SuperAnimal models for inference without training. @@ -126,7 +129,6 @@ result = deeplabcut.video_inference_superanimal( df_3d = result[video_path]["df_3d"] ``` - ### Practical example: Using SuperAnimal model bottom up, considering video/animal size. In our work we introduced a spatial-pyramid for smartly rescaling images. Imagine if you frames are much larger than what we trained on, it would be hard for the model to find the animal! Here, you can simply guide the model with the `scale_list`: @@ -148,12 +150,13 @@ deeplabcut.video_inference_superanimal([video_path], ``` ### Practical example: Using transfer learning with superanimal weights. + In the `deeplabcut.train_network` function, the `superanimal_transfer_learning` option plays a pivotal role. If it's set to __True__, it uses a new decoding layer and allows you to use superanimal weights in any project, no matter the number of keypoints. However, if it's set to __False__, you are doing fine-tuning. So, make sure your dataset has the right number of keypoints. Specifically: - * `superanimal_quadruped_x` uses 39 keypoints - * `superanimal_topviewmouse_x` uses 27 keypoints - * `superanimal_humanbody_x` uses 17 keypoints +\* `superanimal_quadruped_x` uses 39 keypoints +\* `superanimal_topviewmouse_x` uses 27 keypoints +\* `superanimal_humanbody_x` uses 17 keypoints ```python import os @@ -191,8 +194,6 @@ Pixel statistics domain shift: The brightness of your video might look very diff This might either result in jittering predictions in the video or fail modes for lab mice videos (if the brightness of the mice is unusual compared to our training dataset). You can use our "video adaptation" model to counter this. - - ### Our longer term perspective ... Via DeepLabCut Model Zoo, we aim to provide plug and play models that do not need any labeling and will just work diff --git a/docs/Overviewof3D.md b/docs/Overviewof3D.md index 07c9384a90..4351e78f87 100644 --- a/docs/Overviewof3D.md +++ b/docs/Overviewof3D.md @@ -3,8 +3,14 @@ deeplabcut: last_content_updated: '2025-10-14' last_metadata_updated: '2026-03-06' ignore: false + visibility: online + status: review_needed + recommendation: update + notes: Contents seem up-to-date as the codebase has not evolved drastically for 3D, but formatting and organization could be improved. Separate basic/advanced sections could help, as well as more admonitions/dropdowns to streamline. --- + (3D-overview)= + # 3D DeepLabCut In this repo we directly support 2-camera based 3D pose estimation. If you want n camera support, plus nicer @@ -14,32 +20,30 @@ link you will find how we optimize 6+ camera DLC output data for cheetahs (and s DLC 3D - ## **ATTENTION: Our code base in this repo assumes you:** A. You have 2D videos and a DeepLabCut network to analyze them as described in the [main documentation](overview). This can be with multiple separate networks for each camera (less recommended), or one network trained on all views - recommended! (See -[Nath*, Mathis* et al., 2019](https://www.biorxiv.org/content/10.1101/476531v1)). We also support multi-animal 3D with this code (please see +[Nath\*, Mathis\* et al., 2019](https://www.biorxiv.org/content/10.1101/476531v1)). We also support multi-animal 3D with this code (please see [Lauer et al. 2022](https://doi.org/10.1038/s41592-022-01443-0)). -B. You are using 2 cameras, in a [stereo configuration](https://github.com/DeepLabCut/DeepLabCut/blob/5ac4c8cb6bcf2314a3abfcf979b8dd170608e094/deeplabcut/pose_estimation_3d/camera_calibration.py#L223), for 3D*. +B. You are using 2 cameras, in a [stereo configuration](https://github.com/DeepLabCut/DeepLabCut/blob/5ac4c8cb6bcf2314a3abfcf979b8dd170608e094/deeplabcut/pose_estimation_3d/camera_calibration.py#L223), for 3D\*. C. You have calibration images taken (see details below!). +### \***If you need more than 2 camera support:** -### ***If you need more than 2 camera support:** Here are other excellent options for you to use that extend DeepLabCut: DLC 3D - **[AcinoSet](https://github.com/African-Robotics-Unit/AcinoSet)**; **n**-camera support with triangulation, extended Kalman filtering, and trajectory optimization -code (see video to the right for a min demo, courtesy of Prof. Patel), plus a GUI to visualize 3D data. It is built to -work directly with DeepLabCut (but currently tailored to cheetah's, thus some coding skills are required at this time). - + code (see video to the right for a min demo, courtesy of Prof. Patel), plus a GUI to visualize 3D data. It is built to + work directly with DeepLabCut (but currently tailored to cheetah's, thus some coding skills are required at this time). - **[anipose.org](https://anipose.readthedocs.io/en/latest/)**; a wrapper for 3D deeplabcut that provides >3 camera support and is built to work directly with -DeepLabCut. You can `pip install anipose` into your DLC conda environment. + DeepLabCut. You can `pip install anipose` into your DLC conda environment. - **Argus, easywand or DLTdv** w/DeepLabCut see https://github.com/backyardbiomech/DLCconverterDLT; this can be used with the the highly popular Argus or DLTdv tools for wand calibration. As of Summer, 2025, [Argus](https://github.com/kilmoretrout/argus_gui) now supports direct import and export of DeepLabCut output files in the GUI with new [workflow documentation](https://github.com/kilmoretrout/argus_gui/blob/master/docs/deeplabcut.md) @@ -57,11 +61,10 @@ DeepLabCut. You can `pip install anipose` into your DLC conda environment. Watch a [DEMO VIDEO](https://youtu.be/Eh6oIGE4dwI) on how to use this code, and check out the Notebook [here](https://github.com/DeepLabCut/DeepLabCut/blob/main/examples/JUPYTER/Demo_3D_DeepLabCut.ipynb)! - You will run this function **one** time per project; a project is defined as a given set of cameras and calibration images. You can always analyze new videos within this project. -The function **create\_new\_project\_3d** creates a new project directory specifically for converting the 2D pose to 3D +The function **create_new_project_3d** creates a new project directory specifically for converting the 2D pose to 3D pose, required subdirectories, and a basic 3D project configuration file. Each project is identified by the name of the project (e.g. Task1), name of the experimenter (e.g. YourName), as well as the date at creation. @@ -70,9 +73,11 @@ cameras to be used. Currently, DeepLabCut supports triangulation using 2 cameras in a future version. To start a 3D project type the following in ipython: + ```python deeplabcut.create_new_project_3d("ProjectName", "NameofLabeler", num_cameras=2) ``` + TIP 1: you can also pass `working_directory="Full path of the working directory"` if you want to place this folder somewhere beside the current directory you are working in. If the optional argument `working_directory` is unspecified, the project directory is created in the current working directory. @@ -83,7 +88,7 @@ easy use. Please note that `config_path3d='Full path of the 3D project configura This function will create a project directory with the name **Name of the project+name of the experimenter+date of creation of the project+3d** in the **Working directory**. The project directory will have subdirectories: -**calibration_images**, **camera_matrix**, **corners**, and **undistortion**. All the outputs generated during the +**calibration_images**, **camera_matrix**, **corners**, and **undistortion**. All the outputs generated during the course of a project will be stored in one of these subdirectories, thus allowing each project to be curated in separation from other projects. @@ -98,7 +103,7 @@ pickle files contain the intrinsic and extrinsic camera parameters. While the in transformation from 3-D camera's coordinates into the image coordinates, the extrinsic parameters represent a rigid transformation from world coordinate system to the 3-D camera's coordinate system. -**corners:** As a part of camera calibration, the checkerboard pattern is detected in the calibration images and these +**corners:** As a part of camera calibration, the checkerboard pattern is detected in the calibration images and these patterns will be stored in this directory. Each row of the checkerboard grid is marked with a unique color. **undistortion:** In order to check for calibration, the calibration images and the corresponding corner points are @@ -115,9 +120,10 @@ Here is an overview of the calibration and triangulation workflow that follows: (**CRITICAL!**) You must take images of a checkerboard to calibrate your images. Here are example boards you could print and use (mount it on a flat, hard surface!): https://markhedleyjones.com/projects/calibration-checkerboard-collection. + - You must save the image pairs as .jpg files. - They should be named with the **camera-#** as the prefix, i.e. **camera-1-01.jpg** and **camera-2-01.jpg** for the -first pair of images. Please note, this cannot be changed after the project is created. + first pair of images. Please note, this cannot be changed after the project is created. **TIP:** If you want to take a short video (vs. snapping pairs of frames) while you move the checkerboard around, you can use this command inside your conda environment (but outside of ipython!) to convert the video to **.jpg** frames @@ -126,20 +132,20 @@ can use this command inside your conda environment (but outside of ipython!) to ```python ffmpeg -i videoname.mp4 -vframes 20 camera-1-%03d.jpg ``` + - While taking the images: - Keep the orientation of the checkerboard the same and do not rotate it more than 30 degrees. Rotating the - checkerboard circular will change the origin across the frames and may result in incorrect order of detected corners. + checkerboard circular will change the origin across the frames and may result in incorrect order of detected corners. - Cover several distances, and within each distance, cover all parts of the image view (all corners and center). - Use a checkerboard as big as possible, ideally with at least 8x6 squares. - Aim for taking at least 30-70 pair of images, as after corner detection, some of the images might need to be - discarded due to either incorrect corner detection or incorrect order of detected corners. + discarded due to either incorrect corner detection or incorrect order of detected corners. - You can take the images as a series of .jpg images, or a video where you post-hoc pair sync'd frames (see tip - above). - + above). The camera calibration is an **iterative process**, where the user needs to select a set of calibration images where the grid pattern is correctly detected. The function `deeplabcut.calibrate_cameras(config_path)` @@ -175,7 +181,6 @@ Here is what they might look like:

- Once all the set of images has been selected (namely, delete from the folder any bad pairs!) where the corners and their orders are detected correctly, then the two cameras can be calibrated using: @@ -227,15 +232,15 @@ video filename must contain this naming, i.e. this could be named as `rig-1-mous information for the 2D views. - Of critical importance is that you need to input the **same** body part names as in the config.yaml file of the 2D -project. + project. - You must set the snapshot to use inside the 2D config file (default is -1, namely the last training snapshot of the -network). + network). - You need to set a "scorer 3D" name; this will point to the project file and be set in future 3D output file names. - You should define a "skeleton" here as well (note, this is not rigid, it just connects the points in the plotting -step). Not every point needs to be "skeletonized", i.e. these points can be a subset of the full body parts list. The -other points will just be plotted into the 3D space. Here is how the config.yaml looks with some example inputs: + step). Not every point needs to be "skeletonized", i.e. these points can be a subset of the full body parts list. The + other points will just be plotted into the 3D space. Here is how the config.yaml looks with some example inputs: -

+

@@ -253,8 +258,9 @@ deeplabcut.triangulate( filterpredictions=True/False ) ``` -NOTE: Windows users, you must input paths as: ``r`C:\Users\computername\videofolder'`` or -``C:\\Users\\computername\\videofolder'``. + +NOTE: Windows users, you must input paths as: `` r`C:\Users\computername\videofolder' `` or +`C:\\Users\\computername\\videofolder'`. **TIP:** Here are all the parameters you can pass: @@ -289,6 +295,7 @@ save_as_csv: bool, optional track_method: str, optional Method used for tracking: "box" or "ellipse" ``` + The **triangulated file** is now saved under the same directory where the video files reside (or the destination folder you set)! This can be used for future analysis. This step can be run at anytime as you collect new videos, and easily added to your automated analysis pipeline, i.e. such as **replacing** @@ -320,7 +327,7 @@ deeplabcut.create_labeled_video_3d( variables `xlim`, `ylim`, `zlim` and `view`. Your checkerboard_3d.png image which was created above will show you the axis ranges. Here is an example: -

+

@@ -331,6 +338,7 @@ the values, and start again! **Other optional parameters include:** here + ```python videofolder: string Full path of the folder where the videos are stored. Use this if the videos are stored in a different location other than where the triangulation files are stored. By default is ``None`` and therefore looks for video files in the directory where the triangulation file is stored. @@ -373,5 +381,5 @@ dpi: int, optional, default=300 ### If you use this code: -We kindly ask that you cite [Mathis et al, 2018](https://www.nature.com/articles/s41593-018-0209-y) **&** [Nath*, Mathis*, et al., 2019](https://doi.org/10.1038/s41596-019-0176-0). If you use 3D +We kindly ask that you cite [Mathis et al, 2018](https://www.nature.com/articles/s41593-018-0209-y) **&** [Nath\*, Mathis\*, et al., 2019](https://doi.org/10.1038/s41596-019-0176-0). If you use 3D multi-animal: [Lauer et al. 2022](https://doi.org/10.1038/s41592-022-01443-0). diff --git a/docs/README.md b/docs/README.md index 2812d8f001..00b76cff14 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,6 +4,11 @@ deeplabcut: last_metadata_updated: '2026-03-06' ignore: false --- + + + + + Please see https://deeplabcut.github.io/DeepLabCut for documentation on how to use this software. This directory contains the source code for the docs. diff --git a/docs/UseOverviewGuide.md b/docs/UseOverviewGuide.md index 77cf3eda53..f9ebc3a1a7 100644 --- a/docs/UseOverviewGuide.md +++ b/docs/UseOverviewGuide.md @@ -4,80 +4,94 @@ deeplabcut: last_metadata_updated: '2026-03-06' ignore: false --- + (overview)= + # 🥳 Get started with DeepLabCut: our key recommendations Below we will first outline what you need to get started, the different ways you can use DeepLabCut, and then the full workflow. Note, we highly recommend you also read and follow our [Nature Protocols paper](https://www.nature.com/articles/s41596-019-0176-0), which is (still) fully relevant to standard DeepLabCut. -```{Hint} -💡📚 If you are new to Python and DeepLabCut, you might consider checking our [beginner guide](https://deeplabcut.github.io/DeepLabCut/docs/beginner-guides/beginners-guide.html) once you are ready to jump into using the DeepLabCut App! +```{hint} +💡📚 If you are new to Python and DeepLabCut, you might consider checking our {ref}`beginner guide ` once you are ready to jump into using the DeepLabCut App! ``` +## Introduction + +**DeepLabCut** is a software package for markerless pose estimation of animals performing various tasks. The software can manage multiple projects for various tasks. Each project is identified by the name of the project (e.g. TheBehavior), name of the experimenter (e.g. YourName), as well as the date at creation. This project folder holds a `config.yaml` (a text document) file containing various (project) parameters as well as links the data of the project. + +

+ +

+ +

+ +

-## [How to install DeepLabCut](how-to-install) +## {ref}`Installing DeepLabCut` We don't cover installation in depth on this page, so click on the link above if that is what you are looking for. See below for details on getting started with DeepLabCut! -## What we support: +## What we support We are primarily a package that enables deep learning-based pose estimation. We have a lot of models and options, but don't get overwhelmed -- the developer team has tried our best to "set the best defaults we possibly can"! -- Decide on your needs: there are **two main modes, standard DeepLabCut or multi-animal DeepLabCut**. We highly recommend carefully considering which one is best for your needs. For example, a white mouse + black mouse would call for standard, while two black mice would use multi-animal. **[Important Information on how to use DLC in different scenarios (single vs multi animal)](important-info-regd-usage)** Then pick a user guide: +### Main modes of DeepLabCut + +- Decide on your needs: there are **two main modes, standard DeepLabCut or multi-animal DeepLabCut**. + + - We highly recommend carefully considering which one is best for your needs. + - For example, a white mouse + black mouse would call for standard, while two black mice would use multi-animal. See {ref}`important-info-regd-usage`. + - Then pick a user guide: + 1. {ref}`How to use standard DeepLabCut ` + 1. {ref}`How to use multi-animal DeepLabCut ` - - (1) [How to use standard DeepLabCut](single-animal-userguide) - - (2) [How to use multi-animal DeepLabCut](multi-animal-userguide) +- To note, as of DLC3+ the single and multi-animal code bases are more integrated and we support **top-down**, **bottom-up**, and a new "hybrid" approach that is state-of-the-art, called **BUCTD** (bottom-up conditional top down) -- To note, as of DLC3+ the single and multi-animal code bases are more integrated and we support **top-down**, **bottom-up**, and a new "hybrid" approach that is state-of-the-art, called **BUCTD** (bottom-up conditional top down), models. - If these terms are new to you, check out our [Primer on Motion Capture with Deep Learning!](https://www.sciencedirect.com/science/article/pii/S0896627320307170). In brief, both work for single or multiple animals and each method can be better or worse on your data.

- +

- - Here is more information on BUCTD: +- Here is more information on BUCTD: +

- +

- **Additional Learning Resources:** - - - [TUTORIALS:](https://www.youtube.com/channel/UC2HEbWpC_1v6i9RnDMy-dfA?view_as=subscriber) video tutorials that demonstrate various aspects of using the code base. - - [HOW-TO-GUIDES:](overview) step-by-step user guidelines for using DeepLabCut on your own datasets (see below) - - [EXPLANATIONS:](https://github.com/DeepLabCut/DeepLabCut-Workshop-Materials) resources on understanding how DeepLabCut works - - [REFERENCES:](https://github.com/DeepLabCut/DeepLabCut#references) read the science behind DeepLabCut - - [BEGINNER GUIDE TO THE GUI](https://deeplabcut.github.io/DeepLabCut/docs/beginner-guides/beginners-guide.html) +### Additional learning resources -Getting Started: [a video tutorial on navigating the documentation!](https://www.youtube.com/watch?v=A9qZidI7tL8) +- [Video tutorials:](https://www.youtube.com/channel/UC2HEbWpC_1v6i9RnDMy-dfA?view_as=subscriber) video tutorials that demonstrate various aspects of using the code base. + -### What you need to get started: + - - **a set of videos that span the types of behaviors you want to track.** Having 10 videos that include different backgrounds, different individuals, and different postures is MUCH better than 1 or 2 videos of 1 or 2 different individuals (i.e. 10-20 frames from each of 10 videos is **much better** than 50-100 frames from 2 videos). +- [Explanations:](https://github.com/DeepLabCut/DeepLabCut-Workshop-Materials) resources on understanding how DeepLabCut works +- [References:](https://github.com/DeepLabCut/DeepLabCut#references) read the science behind DeepLabCut +- {ref}`Beginner's guide to the GUI`: a step-by-step walkthrough of the GUI for new users. - - **minimally, a computer w/a CPU.** If you want to use DeepLabCut on your own computer for many experiments, then you should get an NVIDIA GPU. See technical specs [here](https://github.com/DeepLabCut/DeepLabCut/wiki/FAQ). You can also use cloud computing resources, including COLAB ([see how](https://github.com/DeepLabCut/DeepLabCut/blob/master/examples/README.md)). + + -### What you DON'T need to get started: +### What you need to get started - - no specific cameras/videos are required; color, monochrome, etc., is all fine. If you can see what you want to measure, then this will work for you (given enough labeled data). +- **A set of videos that span the types of behaviors you want to track.** Having 10 videos that include different backgrounds, different individuals, and different postures is MUCH better than 1 or 2 videos of 1 or 2 different individuals (i.e. 10-20 frames from each of 10 videos is **much better** than 50-100 frames from 2 videos). - - no specific computer is required (but see recommendations above), our software works on Linux, Windows, and MacOS. +- **Ideally, a computer with a GPU.** If you want to use DeepLabCut on your own computer for training and/or for many experiments, then you should get an NVIDIA GPU. +- You can also use cloud computing resources, including COLAB ([see how](https://github.com/DeepLabCut/DeepLabCut/blob/master/examples/README.md)). -### Overview: -**DeepLabCut** is a software package for markerless pose estimation of animals performing various tasks. The software can manage multiple projects for various tasks. Each project is identified by the name of the project (e.g. TheBehavior), name of the experimenter (e.g. YourName), as well as the date at creation. This project folder holds a ``config.yaml`` (a text document) file containing various (project) parameters as well as links the data of the project. +### What you DON'T need to get started +- No specific cameras/videos are required; color, monochrome, etc., is all fine. If you can see what you want to measure, then this will work for you (given enough labeled data). -

- -

+- No specific computer is required (but see recommendations above), our software works on Linux, Windows, and MacOS. -

- -

+## Workflow overview -### Overview of the workflow: -This page contains a list of the essential functions of DeepLabCut as well as demos. There are many optional parameters with each described function. For detailed function documentation, please refer to the main user guides or API documentation. For additional assistance, you can use the [help](UseOverviewGuide.md#help) function to better understand what each function does. +This page contains a list of the essential functions of DeepLabCut as well as demos. There are many optional parameters with each described function. For detailed function documentation, please refer to the main user guides or API documentation. For additional assistance, you can use the `help` function to better understand what each function does.

@@ -87,89 +101,142 @@ This page contains a list of the essential functions of DeepLabCut as well as de

-You can have as many projects on your computer as you wish. You can have DeepLabCut installed in an [environment](../conda-environments/README.md) and always exit and return to this environment to run the code. You just need to point to the correct ``config.yaml`` file to [jump back in](/docs/UseOverviewGuide.md#tips-for-daily-use)! The documentation below will take you through the individual steps. +You can have as many projects on your computer as you wish. +You can have DeepLabCut installed in a {ref}`conda environment`; once you are finished, exit your terminal, and later re-activate your environment. + +When working on a given project, you just need to point to the correct `config.yaml` file to resume work; the documentation below will take you through the individual steps.

- +

+(sec:important-info-regd-usage)= -(important-info-regd-usage)= +## Usage advice & project types -# Specific Advice for Using DeepLabCut: +```{tip} +We recommend first using **DeepLabCut for a single animal scenario** to understand the workflow - even if it's just our demo data. Multi-animal tracking is more complex - i.e. it has several decisions the user needs to make. Then, when you are ready you can jump into multi-animal mode. +``` -## Important information on using DeepLabCut: +### First project: single or multi-animal? -We recommend first using **DeepLabCut for a single animal scenario** to understand the workflow - even if it's just our demo data. Multi-animal tracking is more complex - i.e. it has several decisions the user needs to make. Then, when you are ready you can jump into multi-animals... +*Which scenario do you have?* -### Additional information for getting started with maDeepLabCut: +- **I have single animal videos:** -We highly recommend using it first in the Project Manager GUI ([Option 3](docs/functionDetails.md#deeplabcut-project-manager-gui)). This will allow you to get used to the additional steps by being walked through the process. Then, you can always use all the functions in your favorite IDE, notebooks, etc. + - Quick start: when you `create_new_project` (and leave the default flag to False in `multianimal=False`). This is the typical work path for a single animal project. -### *What scenario do you have?* +- **I have single animal videos, but I want to use the updated network capabilities introduced for multi-animal projects:** -- **I have single animal videos:** - - quick start: when you `create_new_project` (and leave the default flag to False in `multianimal=False`). This is the typical work path for many of you. + - Quick start: when you `create_new_project` just set the flag `multianimal=True`. -- **I have single animal videos, but I want to use the updated network capabilities introduced for multi-animal projects:** - - quick start: when you `create_new_project` just set the flag `multianimal=True`. This enables you to use maDLC features even though you have only one animal. To note, this is rarely required for single animal projects, and not the recommended path. Some tips for when you might want to use this: this is good for say, a hand or a mouse if you feel the "skeleton" during training would increase performance. DON'T do this for things that could be identified an individual objects. i.e., don't do whisker 1, whisker 2, whisker 3 as 3 individuals. Each whisker always has a specific spatial location, and by calling them individuals you will do WORSE than in single animal mode. + - This enables you to use maDLC features even though you have only one animal. To note, this is rarely required for single animal projects, and not the recommended path. + - Some tips for when you might want to use this: + - This is good for e.g. a hand or a mouse if you feel the "skeleton" during training would increase performance. + - Do not do this for things that could be identified as an individual objects. i.e., don't do whisker 1, whisker 2, whisker 3 as 3 individuals. + Each whisker always has a specific spatial location, and by calling them individuals the network will perform worse than in single animal mode. -[VIDEO TUTORIAL AVAILABLE!](https://youtu.be/JDsa8R5J0nQ) + - [VIDEO TUTORIAL AVAILABLE!](https://youtu.be/JDsa8R5J0nQ) - **I have multiple *identical-looking animals* in my videos:** - - quick start: when you `create_new_project` set the flag `multianimal=True`. If you can't tell them apart, you can assign the "individual" ID to any animal in each frame. See this [labeling w/2.2 demo video](https://www.youtube.com/watch?v=_qbEqNKApsI) -[VIDEO TUTORIAL AVAILABLE!](https://youtu.be/Kp-stcTm77g) + - Quick start: when you `create_new_project` set the flag `multianimal=True`. + - If you can't tell them apart, you can assign the "individual" ID to any animal in each frame. See this [labeling w/2.2 demo video](https://www.youtube.com/watch?v=_qbEqNKApsI) + - [VIDEO TUTORIAL AVAILABLE!](https://youtu.be/Kp-stcTm77g) - **I have multiple animals, *but I can tell them apart,* in my videos and want to use DLC2.2:** - - quick start: when you `create_new_project` set the flag `multianimal=True`. And always label the "individual" ID name the same; i.e. if you have mouse1 and mouse2 but mouse2 always has a miniscope, in every frame label mouse2 consistently. See this [labeling w/2.2 demo video](https://www.youtube.com/watch?v=_qbEqNKApsI). Then, you MUST put the following in the config.yaml file: `identity: true` -[VIDEO TUTORIAL AVAILABLE!](https://youtu.be/Kp-stcTm77g) - ALSO, if you can tell them apart, label animals them consistently! + - Quick start: when you `create_new_project` set the flag `multianimal=True`. + - Always label the "individual" ID name the same; i.e. if you have mouse1 and mouse2 but mouse2 always has a miniscope, in every frame label mouse2 consistently. See this [labeling w/2.2 demo video](https://www.youtube.com/watch?v=_qbEqNKApsI). + - Then, you MUST put the following in the config.yaml file: `identity: true` + - [VIDEO TUTORIAL AVAILABLE!](https://youtu.be/Kp-stcTm77g) + +```{important} +If you can tell them apart, label your animals consistently! +``` - **I have a pre-2.2 single animal project, but I want to use 2.2:** + - Please read [the conversion to maDLC guide](convert-maDLC) -Please read [this convert 2 maDLC guide](convert-maDLC) +### Getting started with multi-animal (ma) DeepLabCut -# The options for using DeepLabCut: +We highly recommend using it first in the {ref}`Project Manager GUI `. +This will allow you to get used to the additional steps by being walked through the process. Then, you can always use all the functions in your favorite IDE, notebooks, etc. -Great - now that you get the overall workflow let's jump in! Here, you have several options. +## How to run DeepLabCut -[**Option 1**](using-demo-notebooks) DEMOs: for a quick introduction to DLC on our data. +There are several options to use DeepLabCut, and we recommend you pick the one that best suits your needs and experience level. You can always switch between them, so don't worry about picking the "wrong" one. -[**Option 2**](using-project-manager-gui) Standalone GUI: is the perfect place for -beginners who want to start using DeepLabCut on your own data. +- **Option 1**: [Demo notebooks](using-demo-notebooks): for a quick introduction to DLC on our data. -[**Option 3**](using-the-terminal) In the terminal: is best for more advanced users, as -with the terminal interface you get the most versatility and options. +- **Option 2**: [Standalone GUI](using-project-manager-gui): is the perfect place for + beginners who want to start using DeepLabCut on your own data. + +- **Option 3**: [In the terminal](using-the-terminal): is best for more advanced users, as + with the terminal interface you get the most versatility and options. (using-demo-notebooks)= -## Option 1: Demo Notebooks: + +### Option 1: Demo Jupyter notebooks + [VIDEO TUTORIAL AVAILABLE!](https://www.youtube.com/watch?v=DRT-Cq2vdWs) We provide Jupyter and COLAB notebooks for using DeepLabCut on both a pre-labeled dataset, and on the end user's -own dataset. See all the demo's [here!](../examples/README.md) Please note that GUIs are not easily supported in Jupyter in MacOS, as you need a framework build of python. While it's possible to launch them with a few tweaks, we recommend using the Project Manager GUI or terminal, so please follow the instructions below. +own dataset. See all the demo's [here!](../examples/README.md) +Please note that GUIs are not easily supported in Jupyter in MacOS, as you need a framework build of python. While it's possible to launch them with a few tweaks, we recommend using the Project Manager GUI or terminal, so please follow the instructions below. (using-project-manager-gui)= -## Option 2: using the Project Manager GUI: + +### Option 2: using the Project Manager GUI + [VIDEO TUTORIAL!](https://www.youtube.com/watch?v=KcXogR-p5Ak) [VIDEO TUTORIAL#2!](https://youtu.be/Kp-stcTm77g) -Start Python by typing ``ipython`` or ``python`` in the terminal (note: using pythonw for Mac users was depreciated in 2022). -If you are using DeepLabCut on the cloud, you cannot use the GUIs. If you use Windows, please always open the terminal with administrator privileges. Please read more in our Nature Protocols paper [here](https://www.nature.com/articles/s41596-019-0176-0). And, see our [troubleshooting wiki](https://github.com/DeepLabCut/DeepLabCut/wiki/Troubleshooting-Tips). + + + + +If you are using DeepLabCut on the cloud, you cannot use the GUIs. + +```{warning} +On **Windows**: Open the terminal/cmd/anaconda prompt as **Administrator** (right click and select "Run as administrator") to avoid permission issues during usage when downloading models, and for symlink support when videos are not copied into the project folder. +Admin mode is not required for installation. +``` Simply open the terminal and type: + ```python python -m deeplabcut ``` + That's it! Follow the GUI for details (using-the-terminal)= -## Option 3: using the program terminal, Start iPython*: + +### Option 3: using the terminal + +1. Start iPython: + + ```bash + ipython + ``` + +1. Import DeepLabCut: + + ```python + import deeplabcut + ``` + +1. Follow the instructions in the user guides for either standard or multi-animal DeepLabCut (see below). [VIDEO TUTORIAL AVAILABLE!](https://www.youtube.com/watch?v=7xwOhUcIGio) -Please decide with mode you want to use DeepLabCut, and follow one of the following: +Please decide which mode you want to use DeepLabCut with, and follow one of: + +- (1) {ref}`How to use standard DeepLabCut ` +- (2) {ref}`How to use multi-animal DeepLabCut ` + +## Useful links -- (1) [How to use standard DeepLabCut](single-animal-userguide) -- (2) [How to use multi-animal DeepLabCut](multi-animal-userguide) +Please read more in our Nature Protocols paper [here](https://www.nature.com/articles/s41596-019-0176-0). diff --git a/docs/_static/custom.css b/docs/_static/custom.css index c8df32f9cf..d0f1793c1a 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -9,6 +9,17 @@ html[data-theme="light"] { --logo-filter: none; --button-color: #fff; --footer-text-color: #000000; + + /* Workflow dropdown/admonition colors */ + --single-animal-border: #9b5de5; + --single-animal-bg: #f5edff; + --single-animal-title-bg: #ead7ff; + --single-animal-title-text: #4b236f; + + --multi-animal-border: #57c4be; + --multi-animal-bg: #e8f8f7; + --multi-animal-title-bg: #21a197; + --multi-animal-title-text: #000000; } html[data-theme="dark"] { @@ -20,6 +31,17 @@ html[data-theme="dark"] { /* --logo-filter: grayscale(100%) brightness(20); */ --button-color: #fff; --footer-text-color: #b9b9b9; + + /* Workflow dropdown/admonition colors */ + --single-animal-border: #c084fc; + --single-animal-bg: rgba(72, 86, 107, 0.16); + --single-animal-title-bg: rgba(241, 83, 255, 0.32); + --single-animal-title-text: #f3e8ff; + + --multi-animal-border: #57c4be; + --multi-animal-bg: rgba(72, 86, 107, 0.16); + --multi-animal-title-bg: #21a197; + --multi-animal-title-text: #000000; } /* Sidebar */ @@ -101,3 +123,257 @@ html[data-theme="dark"] { opacity: 0.8; z-index: 1; } + +/* ============================================================ + Workflow-specific dropdown/admonition styling + Single animal = light purple + Multi animal = light green + + Recommended MyST usage: + + ```{dropdown} + :class-container: single-animal + :open: + + Single-animal-specific instructions. + ``` + + ```{dropdown} + :class-container: multi-animal + :open: + + Multi-animal-specific instructions. + ``` + ============================================================ */ + +/* ------------------------------------------------------------ + Sphinx-design dropdowns + Usually rendered with .sd-dropdown + ------------------------------------------------------------ */ + +.sd-dropdown.single-animal { + border-left: 0.35rem solid var(--single-animal-border); + background-color: var(--single-animal-bg); + border-radius: 0.5rem; + margin: 1rem 0; + overflow: hidden; +} + +.sd-dropdown.single-animal>.sd-summary-title { + background-color: var(--single-animal-title-bg); + color: var(--single-animal-title-text); + font-weight: 700; +} + +.sd-dropdown.multi-animal { + border-left: 0.35rem solid var(--multi-animal-border); + background-color: var(--multi-animal-bg); + border-radius: 0.5rem; + margin: 1rem 0; + overflow: hidden; +} + +.sd-dropdown.multi-animal>.sd-summary-title { + background-color: var(--multi-animal-title-bg); + color: var(--multi-animal-title-text); + font-weight: 700; +} + +/* ------------------------------------------------------------ + Regular Sphinx/MyST admonitions as fallback + + ```{admonition} Configuration + :class: single-animal + + Single-animal-specific instructions. + ``` + ------------------------------------------------------------ */ + +div.admonition.single-animal { + border-left: 0.35rem solid var(--single-animal-border); + background-color: var(--single-animal-bg); + border-radius: 0.5rem; + overflow: hidden; +} + +div.admonition.single-animal>.admonition-title { + background-color: var(--single-animal-title-bg); + color: var(--single-animal-title-text); + font-weight: 700; +} + +div.admonition.multi-animal { + border-left: 0.35rem solid var(--multi-animal-border); + background-color: var(--multi-animal-bg); + border-radius: 0.5rem; + overflow: hidden; +} + +div.admonition.multi-animal>.admonition-title { + background-color: var(--multi-animal-title-bg); + color: var(--multi-animal-title-text); + font-weight: 700; +} + +/* ------------------------------------------------------------ + Raw HTML
fallback + +
+ Configuration + + Single-animal-specific instructions. +
+ ------------------------------------------------------------ */ + +details.workflow-dropdown { + border: 1px solid transparent; + border-left-width: 0.35rem; + border-radius: 0.5rem; + margin: 1rem 0; + padding: 0; + overflow: hidden; +} + +details.workflow-dropdown>summary { + cursor: pointer; + font-weight: 700; + padding: 0.6rem 0.9rem; + list-style-position: inside; +} + +details.workflow-dropdown>*:not(summary) { + padding-left: 1rem; + padding-right: 1rem; +} + +details.workflow-dropdown.single-animal { + border-left-color: var(--single-animal-border); + background-color: var(--single-animal-bg); +} + +details.workflow-dropdown.single-animal>summary { + background-color: var(--single-animal-title-bg); + color: var(--single-animal-title-text); +} + +details.workflow-dropdown.multi-animal { + border-left-color: var(--multi-animal-border); + background-color: var(--multi-animal-bg); +} + +details.workflow-dropdown.multi-animal>summary { + background-color: var(--multi-animal-title-bg); + color: var(--multi-animal-title-text); +} + +/* ------------------------------------------------------------ + Compatibility fallback for dropdowns rendered as div.dropdown + ------------------------------------------------------------ */ + +div.dropdown.single-animal { + border-left: 0.35rem solid var(--single-animal-border); + background-color: var(--single-animal-bg); + border-radius: 0.5rem; + overflow: hidden; +} + +div.dropdown.single-animal>.admonition-title, +div.dropdown.single-animal>.sd-summary-title { + background-color: var(--single-animal-title-bg); + color: var(--single-animal-title-text); + font-weight: 700; +} + +div.dropdown.multi-animal { + border-left: 0.35rem solid var(--multi-animal-border); + background-color: var(--multi-animal-bg); + border-radius: 0.5rem; + overflow: hidden; +} + +div.dropdown.multi-animal>.admonition-title, +div.dropdown.multi-animal>.sd-summary-title { + background-color: var(--multi-animal-title-bg); + color: var(--multi-animal-title-text); + font-weight: 700; +} + +/* ============================================================ + Workflow dropdowns: title from CSS, default alignment, + custom left stripe, no hover color shift + ============================================================ */ + +/* Hide sphinx-design's no-title kebab icon */ +details.sd-dropdown.sd-card.single-animal .sd-summary-text>svg.no-title, +details.sd-dropdown.sd-card.multi-animal .sd-summary-text>svg.no-title { + display: none !important; +} + +/* CSS-generated titles inside the default title span */ +details.sd-dropdown.sd-card.single-animal .sd-summary-text::before { + content: "🐁 Single animal"; +} + +details.sd-dropdown.sd-card.multi-animal .sd-summary-text::before { + content: "🐀🐀🐀 Multi animal"; +} + +/* Single animal colors */ +details.sd-dropdown.sd-card.single-animal>summary.sd-card-header { + --pst-sd-dropdown-color: var(--single-animal-border); + --pst-sd-dropdown-bg-color: var(--single-animal-title-bg); + + color: var(--single-animal-title-text) !important; + background-color: var(--single-animal-title-bg) !important; +} + +/* The body also needs the variable because pydata sets border-left there too */ +details.sd-dropdown.sd-card.single-animal>summary.sd-card-header+div.sd-summary-content { + --pst-sd-dropdown-color: var(--single-animal-border); + + background-color: var(--single-animal-bg) !important; +} + +/* Multi animal colors */ +details.sd-dropdown.sd-card.multi-animal>summary.sd-card-header { + --pst-sd-dropdown-color: var(--multi-animal-border); + --pst-sd-dropdown-bg-color: var(--multi-animal-title-bg); + + color: var(--multi-animal-title-text) !important; + background-color: var(--multi-animal-title-bg) !important; +} + +/* The body also needs the variable because pydata sets border-left there too */ +details.sd-dropdown.sd-card.multi-animal>summary.sd-card-header+div.sd-summary-content { + --pst-sd-dropdown-color: var(--multi-animal-border); + + background-color: var(--multi-animal-bg) !important; +} + +/* Keep text, emoji, chevron color stable */ +details.sd-dropdown.sd-card.single-animal .sd-summary-text, +details.sd-dropdown.sd-card.single-animal .sd-summary-state-marker, +details.sd-dropdown.sd-card.single-animal .sd-summary-state-marker svg { + color: var(--single-animal-title-text) !important; + fill: currentColor !important; +} + +details.sd-dropdown.sd-card.multi-animal .sd-summary-text, +details.sd-dropdown.sd-card.multi-animal .sd-summary-state-marker, +details.sd-dropdown.sd-card.multi-animal .sd-summary-state-marker svg { + color: var(--multi-animal-title-text) !important; + fill: currentColor !important; +} + +/* Disable pydata hover darken/lighten effect */ +details.sd-dropdown.sd-card.single-animal>summary.sd-card-header:hover, +details.sd-dropdown.sd-card.single-animal>summary.sd-card-header:focus { + color: var(--single-animal-title-text) !important; + background-color: var(--single-animal-title-bg) !important; +} + +details.sd-dropdown.sd-card.multi-animal>summary.sd-card-header:hover, +details.sd-dropdown.sd-card.multi-animal>summary.sd-card-header:focus { + color: var(--multi-animal-title-text) !important; + background-color: var(--multi-animal-title-bg) !important; +} diff --git a/docs/beginner-guides/Training-Evaluation.md b/docs/beginner-guides/Training-Evaluation.md index 802a22aec5..c106b8982c 100644 --- a/docs/beginner-guides/Training-Evaluation.md +++ b/docs/beginner-guides/Training-Evaluation.md @@ -3,18 +3,35 @@ deeplabcut: last_content_updated: '2025-02-28' last_metadata_updated: '2026-03-06' ignore: false + visibility: online + status: viable + recommendation: move + notes: As mentioned on other beginner-guides/ docs, this should be part of the GUI section. --- -# Neural Network training and evaluation in the GUI + +(file:training-evaluation-gui)= + +# Neural network training and evaluation in the GUI + DLC LIVE! +## Network training + +### Creating a training dataset Before training your model, the first step is to assemble your training dataset. +This involves: -**Create Training Dataset:** Move to the corresponding tab and click **`Create Training Dataset`**. For starters, the default settings will do just fine. While there are more powerful models and data augmentations you might want to consider, you can trust that for most projects the defaults are an ideal place to start. +- Splitting labeled data into training and evaluation subsets +- Creating each shuffle folder with the model configuration ready for training. -> 💡 **Note:** This guide assumes you have a GPU on your local machine. If you're CPU-bound and finding training challenging, consider using Google Colab. Our [Colab Guide](https://colab.research.google.com/github/DeepLabCut/DeepLabCut/blob/master/examples/COLAB/COLAB_YOURDATA_TrainNetwork_VideoAnalysis.ipynb) can help you get started! +**Create Training Dataset:** Move to the corresponding tab and click **`Create Training Dataset`**. For starters, the default settings will do just fine. While there are more powerful models and data augmentations you might want to consider, you can trust that for most projects the defaults are a good place to start. -## Kickstarting the Training Process +```{note} +This guide assumes you have a (CUDA-enabled) GPU on your local machine. If you're CPU-bound and training is not feasible, consider using Google Colab. Our [Colab Guide](https://colab.research.google.com/github/DeepLabCut/DeepLabCut/blob/master/examples/COLAB/COLAB_YOURDATA_TrainNetwork_VideoAnalysis.ipynb) can help you get started! +``` + +### Starting the training process With your training dataset ready, it's time to train your model. @@ -29,31 +46,34 @@ You can keep an eye on the training progress via your terminal window. This will ![DeepLabCut Training in Terminal with TF](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717779598041-DC8UJA2NXJXG65ZWJH1O/training-terminal.png?format=500w) -## Evaluate the Network +## Network evaluation After training, it's time to see how well your model performs. -### Steps to Evaluate the Network +### Step-by-step 1. Find and click on the **`Evaluate Network`** tab. -2. **Choose Evaluation Options:** +1. **Choose Evaluation Options:** - **Plot Predictions:** Select this to visualize the model's predictions, similar to standard DeepLabCut (DLC) evaluations. - **Compare Bodyparts:** Opt to compare all the bodyparts for a comprehensive evaluation. -3. Click the **`Evaluate Network`** button, located on the right side of the main window. - ->💡 Tip: If you wish to evaluate all saved snapshots, go to the configuration file and change the `snapshotindex` parameter to `all`. +1. Click the **`Evaluate Network`** button, located on the right side of the main window. +```{tip} +If you wish to evaluate all saved snapshots, go to the configuration file and change the `snapshotindex` parameter to `all`. +``` -### Understanding the Evaluation Results +### Interpreting the results - **Performance Metrics:** DLC will assess the latest snapshot of your model, generating a `.CSV` file with performance -metrics. This file is stored in the **`evaluation-results`** (for TensorFlow models) or the -**`evaluation-results-pytorch`** (for PyTorch models) folder within your project. + metrics. This file is stored in the **`evaluation-results`** (for TensorFlow models) or the + **`evaluation-results-pytorch`** (for PyTorch models) folder within your project. +![Combined Evaluation Results in DeepLabCut](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717779617667-0RLTM9DVRALN9YIKSHJZ/combined-evaluation-results.png?format=750w) -![Combined Evaluation Results in DeepLabCut](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717779617667-0RLTM9DVRALN9YIKSHJZ/combined-evaluation-results.png?format=750w)) - **Visual Feedback:** Additionally, DLC creates subfolders containing your frames overlaid with both the labeled bodyparts and the model's predictions, allowing you to visually gauge the network's performance. -![Evaluation Example in DeepLabCut](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717779623162-BFDAW37B9TO94EGME2O5/check-labels.png?format=500w)) +![Evaluation Example in DeepLabCut](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717779623162-BFDAW37B9TO94EGME2O5/check-labels.png?format=500w) + +## Next steps -## Next, head over the beginner guide for [using your new neural network for video analysis](video-analysis) +Head over the {ref}`file:video-analysis-gui` section to learn about applying your trained model to videos, and creating labeled videos with the results of your analysis! diff --git a/docs/beginner-guides/beginners-guide.md b/docs/beginner-guides/beginners-guide.md index 784e522c65..9637651f6b 100644 --- a/docs/beginner-guides/beginners-guide.md +++ b/docs/beginner-guides/beginners-guide.md @@ -3,22 +3,34 @@ deeplabcut: last_content_updated: '2026-03-03' last_metadata_updated: '2026-03-06' ignore: false + visibility: online + status: outdated + recommendation: move + notes: Move to GUI section. --- -(beginners-guide)= -# Using DeepLabCut + +(file:beginners-guide)= + +# Setting up a new project + DLC LIVE! -This guide, and related pages, are meant as a very-new-to-python beginner guide to DeepLabCut. After you are comfortable with this material we recommend then jumping into the more detailed User Guides! +This guide and the related pages are intended as a beginner-friendly introduction to DeepLabCut for users who are new to Python. After you are comfortable with this material, we recommend then jumping into the more detailed user guides! + + -- **ProTip:** For even more 'in-depth' understanding, head over to check out the [DeepLabCut Course](https://deeplabcut.github.io/DeepLabCut/docs/course.html), which provides a deeper dive into the science behind DeepLabCut. + ## Installation Before you begin, make sure that DeepLabCut is installed on your system. +Please see the {ref}`installation page` for detailed instructions on how to install DeepLabCut on your computer. -- **ProTip:** For detailed installation instructions, geared towards a bit more advanced users, refer to the [Full Installation Guide](https://deeplabcut.github.io/DeepLabCut/docs/installation.html). + + -## Starting DeepLabCut +## Starting the DeepLabCut GUI + +In the terminal, type: -In the terminal, enter: ```bash python -m deeplabcut ``` -This will open the DeepLabCut App (note, the default is dark mode, but you can click "appearance" to change: + +This will open DeepLabCut. + + ![DeepLabCut GUI Screenshot](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717779625875-5UHPC367I293CBSP8CT6/GUI-screenshot.png?format=500w) -> 💡 **Note:** For a visual guide on navigating through the DeepLabCut GUI, check out our [YouTube tutorial](https://www.youtube.com/watch?v=tr3npnXWoD4). +```{note} +For a visual guide on navigating through the DeepLabCut GUI, check out our [YouTube tutorial](https://www.youtube.com/watch?v=tr3npnXWoD4). +``` -## Starting a New Project +## Starting a new project -### Navigating the GUI on Initial Launch +### Navigating the GUI on initial Launch When you first launch the GUI, you'll find three primary main options: 1. **Create New Project:** Geared towards new initiatives. A good choice if you're here to start something new. -2. **Load Project:** Use this to resume your on-hold or past work. -3. **Model Zoo:** Best suited for those who want to explore Model Zoo. +1. **Load Project:** Use this to resume your on-hold or past work. +1. **Model Zoo:** Best suited for those who want to explore Model Zoo. -### Commencing Your Work: + -- For a first-time or new user, please click on **`Start New Project`**. + -## 🐾 Steps to Start a New Project +### 🐾 New project step-by-step 1. **Launch New Project:** + - When you start a new project, you'll be presented with an empty project window. In DLC3+ you will see a new option "Engine". - - We recommend using the PyTorch Engine: - ![DeepLabCut Engine](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717780414978-17LOVBUJ8JR102QVSFDY/Screen+Shot+2024-06-07+at+7.13.14+PM.png?format=1500w)) + ![DeepLabCut Engine](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717780414978-17LOVBUJ8JR102QVSFDY/Screen+Shot+2024-06-07+at+7.13.14+PM.png?format=1500w) + + ```{note} + For most users, the engine will be PyTorch. See {ref}`sec:deeplabcut-with-tf-install` for TensorFlow support. + ``` + +1. **Filling in Project Details:** -2. **Filling in Project Details:** - **Naming Your Project:** - - Give a specific, well-defined name to your project. - > **💡 Tip:** Avoid empty spaces in your project name. + - Give a specific, easy-to-track name to your project. + + ```{tip} + Avoid spaces in your project name. + ``` + + - **Fill in the name of the scorer/experimenter**. This name is used in data headers and directory names and it remains permanently associated with the project. - - **Naming the Experimenter:** - - Fill in the name of the experimenter. This part of the data remains immutable. +1. **Determine Project Location:** -3. **Determine Project Location:** - By default, your project will be located on the **Desktop**. - - To pick a different home, modify the path as needed. + - To pick a different location, browse as needed. -4. **Multi-Animal or Single-Animal Project:** - - Tick the 'Multi-Animal' option in the menu, but only if that's the mode of the project. +1. **Multi-Animal or Single-Animal Project:** + + - Tick the 'Multi-Animal' option in the menu if relevant to your experiment. - Choose the 'Number of Cameras' as per your experiment. -5. **Adding Videos:** +1. **Adding Videos:** + - First, click on **`Browse Videos`** button on the right side of the window, to search for the video contents. - Once the media selection tool opens, navigate and select the folder with your videos. - - > **💡 Tip:** DeepLabCut supports **`.mp4`**, **`.avi`**, **`.mkv`** and **`.mov`** files. + ```{tip} + DeepLabCut supports **`.mp4`**, **`.avi`**, **`.mkv`** and **`.mov`** files. + ``` - A list will be created with all the videos inside this folder. - Unselect the videos you wish to remove from the project. + - Videos outside the project directory can be automatically copied into the project folder by selecting the "Copy videos to project folder" option. This is the recommended strategy for data management. External videos that are not copied are instead referenced via symbolic links. While using symbolic links avoids duplicating files and reduces storage usage, it is also more prone to issues, for example if the original files are moved or deleted. + - ```{tip} + By default, the GUI will look for a **directory** containing videos. Use the "Select individual files" + checkbox if you want to select individual videos instead of a whole folder. + ``` + +1. **Define bodyparts and individuals:** -6. **Create your project:** - - Click on **`Create`** button on the bottom, right side of the main window. - - A new folder named after your project's name will be created in the location you chose above. + - Enter all the name, numbers or IDs of bodyparts you wish to track. + - **Example:** "head", "tail", "left paw", "right paw", etc. + - Less recommended: "L1", "L2", "L3", etc. + - **If you have multiple animals**: + - Enter the name, numbers or IDs of the individuals in your experiment. + - **Example:** "mouse1", "mouse2", "mouse3", etc. + - **Unique bodyparts**: If you wish to track "landmark" locations, such as the edges of a maze, or a specific object, you can add these as "unique bodyparts". These are not considered part of an individual, but are still tracked as part of the project. + - **Example**: "maze_left_edge", "maze_right_edge", "reward_port", etc. + - **Identity labeling**: if and only if you can tell individuals apart by their appearance (not their location), set this to Yes and consistently label your individuals in the same way across videos. This will allow DeepLabCut to learn to tell them apart, and assign consistent identities across frames and videos. +1. **Create your project:** + + - Click on the **`Create`** button on the bottom, right side of the main window. + - A new folder will be created in the location you chose above. + +## Video tutorial ### 📽 Video Tutorial: Setting Up Your Project in DeepLabCut ![DeepLabCut Create Project GIF](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717779616437-30U5RFYV0OY6ACGDG7F4/create-project.gif?format=500w) -## Next, head over to the beginner guide for [Setting up what keypoints to track](https://deeplabcut.github.io/DeepLabCut/docs/beginner-guides/manage-project.html) +## Next steps + +Next, head over to the beginner guide for {ref}`editing the configuration and managing the project `, which will show you how to edit the configuration file to edit your bodyparts and skeleton structure. diff --git a/docs/beginner-guides/labeling.md b/docs/beginner-guides/labeling.md index 546d76b96e..359e1f3817 100644 --- a/docs/beginner-guides/labeling.md +++ b/docs/beginner-guides/labeling.md @@ -3,73 +3,80 @@ deeplabcut: last_content_updated: '2025-06-30' last_metadata_updated: '2026-03-06' ignore: false + visibility: online + status: viable + recommendation: move + notes: Move to GUI section. Updated to link directly to the napari plugin docs. Making the link specific to the workflow section of the napari docs could help. --- -(labeling)= -# Labeling GUI - -## Selecting Frames to Label - -In DeepLabCut, choosing the right frames for labeling is a key step. The trick is always to select the MOST DIVERSE data you can that your model will see. That means good lighting, bad lighting, anything you want to throw at it. So, first, pick a range of diverse videos! Then, we will help you pick frames. You've got two easy ways to do this: -1. **Let DeepLabCut Choose:** DeepLabCut can extract frames automatically for you. It's got two neat ways to do that: - - **Uniform:** This is like taking a snapshot at regular time intervals. - - **K-means clustering:** This one applies k-means and picks images from different clusters. This is typically better, as it gives you a variety of actions and poses. Note, as it is a clustering tool, it will miss rare events, so ideally run this step, then perhaps consider running the manual GUI to get some rare frames! You can do both within DLC. +(file:labeling-gui)= -2. **Pick Frames Yourself:** Just like flipping through a photo album, you can go through your video and pick the frames that catch your eye - this is great for finding rare frames. Choose the **`manual`** extraction method. +# Labeling GUI -### Here's how to get started: +## Selecting frames to label -- **Step 1:** Click on **`automatic`** in the frame selection area. -- **Step 2:** Choose **`k-means`** for some variety. -- **Step 3:** Hit the **`Extract Frames`** button, usually found at the bottom right corner. +In DeepLabCut, choosing the right frames for labeling is a key step. -By default, DeepLabCut will grab 20 frames from each of your videos and put them into sub-folders, per video, under **labeled-data** in your project. Now, you're all set to start labeling! +```{important} +Always aim to select the **most diverse data** you can for your model to be trained on. This implies picking a variety of good lighting, bad lighting, partial occlusions, and different poses. +If relevant, label data across several experimental sessions, animals, and conditions. +**Labeling 10 frames from several different videos is typically more effective than labeling 100 frames from a single video.** +``` -## Labeling Your First Set of Frames in DeepLabCut +To help you select "different" frames, DeepLabCut provides two main options: -Alright, you've got your extracted frames ready. Now comes the labeling! +1. **Automated frame extraction** DeepLabCut can extract frames automatically for you. -### Entering the Label Frames Area + - **Uniform:** Samples at regular time intervals. Does not guarantee diversity, but is simple and fast. + - **K-means clustering:** This one runs a k-means algorithm and picks images from different clusters. This is typically more robust in extracting a variety of actions and poses. Note, as it is a clustering tool, it will miss rare events, so after running this step, consider using the manual GUI to get some rare frames! You can do both within DLC. -- **Click on `Label Frames`:** This takes you straight to where your frames are, sorted in the **labeled-data** folder, each video in its own sub-folder. -- **Open a Folder:** Click on the first one to start, and then click **`open`**. +1. **Manual frame extraction** Pick frames yourself using the GUI. This is the most time-consuming, but allows you to have full control over the frames you pick, and can be useful to get rare events that automated tools might miss. You can also use this after running automated frame extraction to get some of those "rare" frames. -### The napari DeepLabCut Labeler +### Example workflow -- **Plugin Window Opens:** As soon as you click **`open`**, the napari DeepLabCut plugin window appears, your main stage for labeling. -- **Tutorial Popup:** A quick tutorial window shows up. It's a brief guide, so give it a look to understand the basics. +1. Select **`automatic`** in the frame selection area. +1. Choose **`k-means`** as a good default option for frame extraction, and set the number of frames you want to extract. +1. Hit the **`Extract Frames`** button. -![Labeling Frames in DeepLabCut using Napari Interface](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717779602092-LVR2TI6OADSHEYRCGS6F/labeling-napari.png?format=500w)) +By default, DeepLabCut will grab 20 frames from each of your videos and put them into sub-folders, per video, under **labeled-data** in your project. +With this, you are all set to start labeling! -### Labeling Setup +## Frame labeling workflow -- **Frames on Display:** Your frames are lined up in the middle, with a slider below to shuffle through them. -- **Tools and Keypoints:** To the bottom right, you find a list of bodyparts from your configuration. On the top left, all your labeling tools are ready. +Alright, you've got your extracted frames ready. Now comes the labeling! -### The Labeling Process +### Launching the labeling GUI -- **Start with `Add points`:** Click this to begin placing keypoints on your first frame. If you can't see a bodypart, just move to the next one. -- **Navigate Through Frames:** Use the slider to go from one frame to the next after you're done labeling. -- **Save Progress:** Remember to save your work as you go with **`Command and S`** (or **`Ctrl and S`** on Windows). +- **Click on `Label Frames`:** This takes you straight to where your frames are, sorted in the **labeled-data** folder, each video in its own sub-folder. +- **Open a Folder:** Click on the first unlabeled folder to start, and then click **`Open`**. -> 💡 **Note:** For a detailed walkthrough on using the Napari labeling GUI, have a look at the -[DeepLabCut Napari Guide](file:napari-gui-landing). Additionally, you can watch our instructional -[YouTube video](https://www.youtube.com/watch?v=hsA9IB5r73E) for more insights and tips. +### napari-deeplabcut +Please refer to the {ref}`file:napari-dlc-basic-usage` section for a detailed walkthrough of how to use the napari-DLC plugin for labeling your frames. -### Completing the Set +### Completing the labeling -Work through all the frames in the first folder. Then, proceed to the next, continuing this way until each folder in your **labeled-data** directory is done. +Work through all the frames in the first folder and save them. -## Checking Your Labels +```{tip} +After saving, you can close napari and click **`Label Frames`** again to open the next folder +**OR** +Remove all layers in napari and drag-and-drop the next folder in the same napari session to keep going without needing to close and reopen napari. +``` -After you've labeled all your frames, it's important to ensure they're accurate. +## Checking labels -### How to Check Your Labels +After you've labeled all your frames, you may want to review their accuracy before moving on to training your model. This is a crucial step, as the quality of your labels will directly impact the performance of your model. -- **Return to the Main Window:** Once you're done with labeling, head back to DeepLabCut's main window, and click on **`Check Labels`**. +- **Return to the DeepLabCut GUI:** Once you're done with labeling, head back to DeepLabCut's main window, and click on **`Check Labels`**. - **Review the Labeled Folders:** The system will have created new folders for each labeled set inside your labeled-data folder. These folders contain your original frames overlaid with the keypoints you've added. ![Checking Labels in DeepLabCut](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717779615252-6BNW661XB2ULH85RTAD3/evaluation-example.png?format=500w) -Take the time to go through each folder. Accurate labels are key. If there are mistakes, the model might learn incorrectly and mislabel your videos later on. It's all about setting the right foundation for accurate analysis. +Take the time to go through each folder. Accurate labels are key. +If there are mistakes, the model might learn incorrectly and mislabel your videos later on. +A clean foundation is essential for accurate analysis. + +## Next steps + +Head on to {ref}`file:training-evaluation-gui` to learn about training and evaluating your neural network with the labeled data you created! diff --git a/docs/beginner-guides/manage-project.md b/docs/beginner-guides/manage-project.md index ee0e0f339e..fd091a2c44 100644 --- a/docs/beginner-guides/manage-project.md +++ b/docs/beginner-guides/manage-project.md @@ -3,48 +3,65 @@ deeplabcut: last_content_updated: '2025-06-30' last_metadata_updated: '2026-03-06' ignore: false + visibility: online + status: viable + recommendation: move + notes: Move to a dedicated GUI section. Making the config edit tool slightly easier to work with and updating the docs below to include additional fields would be helpful. --- -# Setting up what keypoints to track + +(file:manage-project-gui)= + +# Editing and working with the configuration file + DLC LIVE! -**Edit the Configuration File** +The configuration file (`config.yaml`) is the central record of files in your project, as well as the settings for your models. +As a YAML file, it can be edited manually, but the GUI provides an easy way to edit it without needing to know the YAML format. In this guide, we will show you how to edit the configuration file using the GUI. -After creating your DeepLabCut project, you'll go to the main GUI window, where you'll start managing your project from the Project Management Tab. +## Editing the configuration -**Accessing the Configuration File** +After creating your DeepLabCut project, you'll be shown the main GUI window, where you can manage your project from the Project Management Tab. - **Locate the Configuration File:** At the top of the main window, you'll find the file path to the configuration file. -- **Edit the File:** Click on **`Edit config.yaml`**. This action allows you to: - - Define the bodyparts you wish to track. - - Outline the skeleton structure (optional!). +- **Edit the File:** Click on **`Edit config.yaml`**. + - A **`Configuration Editor`** window will open, displaying all the configuration details. + - You will need to modify some of these settings to align with your experiment. + - For example: + - Update or define the bodyparts you wish to track. + - *Optional:* Outline the skeleton structure. -A **`Configuration Editor`** window will open, displaying all the configuration details. You'll need to modify some of these settings to align with your research requirements. +## Step-by-step configuration walkthrough -## Steps to Edit the Configuration - -### 1. Defining Bodyparts +### Defining & updating bodyparts - **Locate the Bodyparts Section:** In the Configuration Editor, find the **`bodyparts`** category. - **Modify the List:** Click on the arrow next to **`bodyparts`** to expand the list. Here, you can: - Update the list with the names of the bodyparts relevant to your study. - Add more entries by right-clicking on a row number and selecting **`Insert`**. - ![Editing Bodyparts in DeepLabCut's Config File](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717779624617-CIVZCM23U69NYK9BO3GY/bodyparts.png?format=500w) + -### 2. Defining the Skeleton +### Defining the skeleton - **Navigate to the Skeleton Section:** Scroll down to the **`skeleton`** category. -- **Adjust the Skeleton List:** Click on the arrow to expand this section. You can then: - - Update the pairs of bodyparts to define the skeleton structure of your model. +- **Adjust the Skeleton List:** Click on the arrow to expand this section. + - You can then update the list of bodypart pairs: i.e. the connections that define the skeleton structure of your model. + - In the list of bodypart pairs, each pair has an index. (ranging from 0 to the total number of pairs in the skeleton). + - Each item of the pair (also indexed; 0 or 1) has a value: the name of the bodypart. + - Each pair of two bodyparts represents a connection, where all connections together make the skeleton. ![Defining the Skeleton Structure in Config File](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717779598505-HQNECHIKSQ6XL033JX8M/skeleton.png?format=500w) -> 💡 **Tip:** If you're new to DeepLabCut, spend some time visualizing how the chosen bodyparts can be connected effectively to form a coherent skeleton. +```{tip} +Spend some time visualizing how the chosen bodyparts can be connected effectively to form a coherent, visually helpful skeleton. +``` -### Saving Your Changes +### Saving changes - **Save the Configuration:** Once you're satisfied with the modifications, click **`Save`**. This will store your changes and return you to the main GUI window. -## Next, head over the beginner guide for [Labeling your data](labeling) +## Next steps + +Head over the guide for the {ref}`file:labeling-gui`, which will show you how to label your data using the napari-based labeling GUI. diff --git a/docs/beginner-guides/video-analysis.md b/docs/beginner-guides/video-analysis.md index 8c48d3209c..842ee78506 100644 --- a/docs/beginner-guides/video-analysis.md +++ b/docs/beginner-guides/video-analysis.md @@ -3,22 +3,31 @@ deeplabcut: last_content_updated: '2025-06-30' last_metadata_updated: '2026-03-06' ignore: false + visibility: online + status: viable + recommendation: move + notes: As mentioned on other beginner-guides/ docs, this should be part of the GUI section. --- -# Video Analysis with DeepLabCut -DLC LIVE! +(file:video-analysis-gui)= + +# Video analysis in the GUI + +DLC LIVE! After training and evaluating your model, the next step is to apply it to your videos. -**How to Analyze Videos** +## Analyzing videos with your trained model + +### Step-by-step 1. **Navigate to the 'Analyze Videos' Tab:** Begin applying your trained model to video data here. -2. **Select Your Video Format and Files:** - - **Choose Video Format:** Pick the format of your video (`.mp4`, `.avi`, `.mkv`, or `.mov`). - - **Select Videos:** Click **`Select Videos`** to find and open your video file. -3. **Start Analysis:** Click **`Analyze`**. The analysis time depends on video length and resolution. Track progress in the terminal or Anaconda prompt. +1. **Select Your Video Format and Files:** + - **Choose Video Format:** Pick the format of your video (`.mp4`, `.avi`, `.mkv`, or `.mov`). + - **Select Videos:** Click **`Select Videos`** to find and open your video file. +1. **Start Analysis:** Click **`Analyze`**. The analysis time depends on video length and resolution. Track progress in the terminal or Anaconda prompt. -## Reviewing Analysis Results +### Reviewing analysis results - **Find Results in Your Project Folder:** After analysis, go to your project's video folder. - **Analysis Files:** Look also for a `.metapickle`, an `.h5`, and possibly a `.csv` file for detailed analysis data. @@ -26,16 +35,21 @@ After training and evaluating your model, the next step is to apply it to your v ![Plot poses](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1717779600836-YOWM5T2MBY0JN1LB537B/plot-poses.png?format=500w) -## Creating a Labeled Video +## Generating labeled videos + +### Create a labeled video 1. **Go to 'Create Labeled Video' Tab:** The previously analyzed video should be selected. -2. If not already selected, choose your video. -3. Click **`Create Videos`**. +1. If not already selected, choose your video. +1. Click **`Create Videos`**. -## Viewing the Labeled Video +### View the labeled video - Your labeled video will be in your video folder, named after the original video plus model details and 'labeled'. -- Watch the video to assess the model's labeling accuracy. +- Use it in your results, or perform downstream analyses with it! + +## Next steps + + -## Happy DeepLabCutting! -- Check out the more advanced user guides for even more options! +Check our more advanced guides, and consider reading more about models, augmentations and other parameters to further optimize your model and analysis! diff --git a/docs/benchmark.md b/docs/benchmark.md index 1b2e9a5d42..a9c3b6fc58 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -4,6 +4,7 @@ deeplabcut: last_metadata_updated: '2026-03-06' ignore: false --- + # DeepLabCut benchmark For further information and the leaderboard, see [the official homepage](https://benchmark.deeplabcut.org/). @@ -11,7 +12,7 @@ For further information and the leaderboard, see [the official homepage](https:/ ## High Level API When implementing your own benchmarks, the most important functions are directly accessible -under the ``deeplabcut.benchmark`` package. +under the `deeplabcut.benchmark` package. ```{eval-rst} .. automodule:: deeplabcut.benchmark diff --git a/docs/citation.md b/docs/citation.md index f8a1234ff6..c015514b8c 100644 --- a/docs/citation.md +++ b/docs/citation.md @@ -4,11 +4,11 @@ deeplabcut: last_metadata_updated: '2026-03-06' ignore: false --- + # How to Cite DeepLabCut Thank you for using DeepLabCut! Here are our recommendations for citing and documenting your use of DeepLabCut in your Methods section: - If you use this code or data we kindly ask that you please [cite Mathis et al, 2018](https://www.nature.com/articles/s41593-018-0209-y) and, if you use the Python package (DeepLabCut2.x+) please also cite [Nath, Mathis et al, 2019](https://doi.org/10.1038/s41596-019-0176-0). If you utilize the MobileNetV2s or EfficientNets please cite [Mathis, Biasi et al. 2021](https://openaccess.thecvf.com/content/WACV2021/papers/Mathis_Pretraining_Boosts_Out-of-Domain_Robustness_for_Pose_Estimation_WACV_2021_paper.pdf). @@ -24,79 +24,82 @@ DOIs (#ProTip, for helping you find citations for software, check out [CiteAs.or ## Formatted citations: - @article{Mathisetal2018, - title = {DeepLabCut: markerless pose estimation of user-defined body parts with deep learning}, - author = {Alexander Mathis and Pranav Mamidanna and Kevin M. Cury and Taiga Abe and Venkatesh N. Murthy and Mackenzie W. Mathis and Matthias Bethge}, - journal = {Nature Neuroscience}, - year = {2018}, - url = {https://www.nature.com/articles/s41593-018-0209-y}} - - @article{NathMathisetal2019, - title = {Using DeepLabCut for 3D markerless pose estimation across species and behaviors}, - author = {Nath*, Tanmay and Mathis*, Alexander and Chen, An Chi and Patel, Amir and Bethge, Matthias and Mathis, Mackenzie W}, - journal = {Nature Protocols}, - year = {2019}, - url = {https://doi.org/10.1038/s41596-019-0176-0}} - - @InProceedings{Mathis_2021_WACV, - author = {Mathis, Alexander and Biasi, Thomas and Schneider, Steffen and Yuksekgonul, Mert and Rogers, Byron and Bethge, Matthias and Mathis, Mackenzie W.}, - title = {Pretraining Boosts Out-of-Domain Robustness for Pose Estimation}, - booktitle = {Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision (WACV)}, - month = {January}, - year = {2021}, - pages = {1859-1868}} - - @article{Lauer2022MultianimalPE, - title={Multi-animal pose estimation, identification and tracking with DeepLabCut}, - author={Jessy Lauer and Mu Zhou and Shaokai Ye and William Menegas and Steffen Schneider and Tanmay Nath and Mohammed Mostafizur Rahman and Valentina Di Santo and Daniel Soberanes and Guoping Feng and Venkatesh N. Murthy and George Lauder and Catherine Dulac and M. Mathis and Alexander Mathis}, - journal={Nature Methods}, - year={2022}, - volume={19}, - pages={496 - 504}} - - @article{Ye2024SuperAnimal, - title={SuperAnimal pretrained pose estimation models for behavioral analysis}, - author={Shaokai Ye and Anastasiia Filippova and Jessy Lauer and Steffen Schneider and Maxime Vidal and and Tian Qiu and Alexander Mathis and Mackenzie W. Mathis}, - journal={Nature Communications}, - year={2024}, - volume={15}} - +``` +@article{Mathisetal2018, + title = {DeepLabCut: markerless pose estimation of user-defined body parts with deep learning}, + author = {Alexander Mathis and Pranav Mamidanna and Kevin M. Cury and Taiga Abe and Venkatesh N. Murthy and Mackenzie W. Mathis and Matthias Bethge}, + journal = {Nature Neuroscience}, + year = {2018}, + url = {https://www.nature.com/articles/s41593-018-0209-y}} + + @article{NathMathisetal2019, + title = {Using DeepLabCut for 3D markerless pose estimation across species and behaviors}, + author = {Nath*, Tanmay and Mathis*, Alexander and Chen, An Chi and Patel, Amir and Bethge, Matthias and Mathis, Mackenzie W}, + journal = {Nature Protocols}, + year = {2019}, + url = {https://doi.org/10.1038/s41596-019-0176-0}} + +@InProceedings{Mathis_2021_WACV, + author = {Mathis, Alexander and Biasi, Thomas and Schneider, Steffen and Yuksekgonul, Mert and Rogers, Byron and Bethge, Matthias and Mathis, Mackenzie W.}, + title = {Pretraining Boosts Out-of-Domain Robustness for Pose Estimation}, + booktitle = {Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision (WACV)}, + month = {January}, + year = {2021}, + pages = {1859-1868}} + +@article{Lauer2022MultianimalPE, + title={Multi-animal pose estimation, identification and tracking with DeepLabCut}, + author={Jessy Lauer and Mu Zhou and Shaokai Ye and William Menegas and Steffen Schneider and Tanmay Nath and Mohammed Mostafizur Rahman and Valentina Di Santo and Daniel Soberanes and Guoping Feng and Venkatesh N. Murthy and George Lauder and Catherine Dulac and M. Mathis and Alexander Mathis}, + journal={Nature Methods}, + year={2022}, + volume={19}, + pages={496 - 504}} + +@article{Ye2024SuperAnimal, + title={SuperAnimal pretrained pose estimation models for behavioral analysis}, + author={Shaokai Ye and Anastasiia Filippova and Jessy Lauer and Steffen Schneider and Maxime Vidal and and Tian Qiu and Alexander Mathis and Mackenzie W. Mathis}, + journal={Nature Communications}, + year={2024}, + volume={15}} +``` ### Review & Educational articles: - @article{Mathis2020DeepLT, - title={Deep learning tools for the measurement of animal behavior in neuroscience}, - author={Mackenzie W. Mathis and Alexander Mathis}, - journal={Current Opinion in Neurobiology}, - year={2020}, - volume={60}, - pages={1-11}} - - @article{Mathis2020Primer, - title={A Primer on Motion Capture with Deep Learning: Principles, Pitfalls, and Perspectives}, - author={Alexander Mathis and Steffen Schneider and Jessy Lauer and Mackenzie W. Mathis}, - journal={Neuron}, - year={2020}, - volume={108}, - pages={44-65}} +``` +@article{Mathis2020DeepLT, + title={Deep learning tools for the measurement of animal behavior in neuroscience}, + author={Mackenzie W. Mathis and Alexander Mathis}, + journal={Current Opinion in Neurobiology}, + year={2020}, + volume={60}, + pages={1-11}} + +@article{Mathis2020Primer, + title={A Primer on Motion Capture with Deep Learning: Principles, Pitfalls, and Perspectives}, + author={Alexander Mathis and Steffen Schneider and Jessy Lauer and Mackenzie W. Mathis}, + journal={Neuron}, + year={2020}, + volume={108}, + pages={44-65}} +``` ### Other open-access pre-prints related to our work on DeepLabCut: - @article{MathisWarren2018speed, - author = {Mathis, Alexander and Warren, Richard A.}, - title = {On the inference speed and video-compression robustness of DeepLabCut}, - year = {2018}, - doi = {10.1101/457242}, - publisher = {Cold Spring Harbor Laboratory}, - URL = {https://www.biorxiv.org/content/early/2018/10/30/457242}, - eprint = {https://www.biorxiv.org/content/early/2018/10/30/457242.full.pdf}, - journal = {bioRxiv}} - - +``` +@article{MathisWarren2018speed, + author = {Mathis, Alexander and Warren, Richard A.}, + title = {On the inference speed and video-compression robustness of DeepLabCut}, + year = {2018}, + doi = {10.1101/457242}, + publisher = {Cold Spring Harbor Laboratory}, + URL = {https://www.biorxiv.org/content/early/2018/10/30/457242}, + eprint = {https://www.biorxiv.org/content/early/2018/10/30/457242.full.pdf}, + journal = {bioRxiv}} +``` ## Methods Suggestion: -For body part tracking we used DeepLabCut (version 2.X.X)* [Mathis et al, 2018, Nath et al, 2019, Lauer et al. 2022]. Specifically, we labeled X number of frames taken from X videos/animals (then X% was used for training (default is 95%). We used a X-based neural network (i.e. X = ResNet-50, ResNet-101, MobileNetV2-0.35, MobileNetV2-0.5, MobileNetV2-0.75, MobileNetV2-1***) with default parameters* for X number of training iterations. We validated with X number of shuffles, and found the test error was: X pixels, train: X pixels (image size was X by X). We then used a p-cutoff of X (i.e. 0.9) to condition the X,Y coordinates for future analysis. This network was then used to analyze videos from similar experimental settings. +For body part tracking we used DeepLabCut (version 2.X.X)\* [Mathis et al, 2018, Nath et al, 2019, Lauer et al. 2022]. Specifically, we labeled X number of frames taken from X videos/animals (then X% was used for training (default is 95%). We used a X-based neural network (i.e. X = ResNet-50, ResNet-101, MobileNetV2-0.35, MobileNetV2-0.5, MobileNetV2-0.75, MobileNetV2-1\*\*\*) with default parameters\* for X number of training iterations. We validated with X number of shuffles, and found the test error was: X pixels, train: X pixels (image size was X by X). We then used a p-cutoff of X (i.e. 0.9) to condition the X,Y coordinates for future analysis. This network was then used to analyze videos from similar experimental settings. > Mathis, A. et al. Deeplabcut: markerless pose estimation > of user-defined body parts with deep learning. Nature @@ -106,16 +109,16 @@ For body part tracking we used DeepLabCut (version 2.X.X)* [Mathis et al, 2018, > estimation across species and behaviors. Nature Protocols > 14, 2152–2176 (2019). -*If any defaults were changed in *`pose_config.yaml`*, mention them here. +\*If any defaults were changed in *`pose_config.yaml`*, mention them here. i.e. common things one might change: -* the loader (options are `default`, `imgaug`, `tensorpack`, `deterministic`). -* the `post_dist_threshold` (default is 17 and determines training resolution). -* optimizer: do you use the default `SGD` or `ADAM`? -*** here, you could add additional citations. -If you use ResNets, consider citing Insafutdinov et al 2016 & He et al 2016. If you use the MobileNetV2s consider citing Mathis et al 2019, and Sandler et al, 2018. +- the loader (options are `default`, `imgaug`, `tensorpack`, `deterministic`). +- the `post_dist_threshold` (default is 17 and determines training resolution). +- optimizer: do you use the default `SGD` or `ADAM`? +\*\*\* here, you could add additional citations. +If you use ResNets, consider citing Insafutdinov et al 2016 & He et al 2016. If you use the MobileNetV2s consider citing Mathis et al 2019, and Sandler et al, 2018. > Mathis, A. et al. Pretraining boosts out-of-domain robustness for pose estimation > arXiv 1909.11229 (2019) diff --git a/docs/convert_maDLC.md b/docs/convert_maDLC.md index 14e697c719..3eff9b10cf 100644 --- a/docs/convert_maDLC.md +++ b/docs/convert_maDLC.md @@ -4,12 +4,13 @@ deeplabcut: last_metadata_updated: '2026-03-06' ignore: false --- + (convert-maDLC)= + # How to convert a pre-2.2 project for use with DeepLabCut 2.2 or later DLC! - If you have a pre-2.2 project (`labeled-data`) with a **single animal** that you want to use with a multianimal project in DLC 2.2 or later, i.e. use your older data to now train the new multi-task deep neural network, here is what you need to do. @@ -23,7 +24,7 @@ need to do.

- After `task, scorer, date, project_path` please add the following (i.e. in the image above, you would start adding -below line 6) Note, the ordering isn't important but useful to keep consistent with the template: + below line 6) Note, the ordering isn't important but useful to keep consistent with the template: ```python multianimalproject: true @@ -32,18 +33,21 @@ uniquebodyparts: [] multianimalbodyparts: identity: false/true ``` + - Now, please name the animal you have a new name under individuals, i.e.: + ```python individuals: - mouse1 ``` - `"uniquebodyparts: []` can stay blank, unless you have other items labeled you want to estimate (consider these as -similar to bodyparts in pre-2.2); i.e. corners of a box, etc. All unique bodyparts should not be connected to the -multianimal bodyparts in the skeleton you will eventually make. See "advanced option" below. + similar to bodyparts in pre-2.2); i.e. corners of a box, etc. All unique bodyparts should not be connected to the + multianimal bodyparts in the skeleton you will eventually make. See "advanced option" below. - Please move your "bodyparts:" to "multianimalbodyparts:" (bodypart names must stay the same!) These are the parts -that will always be interconnected fully! + that will always be interconnected fully! + ```python multianimalbodyparts: - snout @@ -51,9 +55,11 @@ multianimalbodyparts: - rightear - tailbase ``` + then you can set `bodyparts: MULTI!` (3) Save the config.yaml (be sure to double check for spacing or typos first!) and then run: + ```python deeplabcut.convert2_maDLC(path_config_file, userfeedback=True) ``` @@ -64,9 +70,11 @@ saved for you under a new file named `CollectedData_ ...singleanimal.h5` and `.c (4) We strongly recommend to first run check_labels and verify that the conversion was as expected before creating a multianimal training dataset. For instance, you can load this project `config.yaml` in the Project Manager GUI and check labels then create a multi-animal training set with + ```python deeplabcut.create_multianimaltraining_dataset(path_config_file) ``` + to begin training. **Advanced option:** You can also assign former `bodyparts` to either `uniquebodyparts` or `multianimalbodyparts` @@ -77,16 +85,20 @@ Example: Imagine you had a project with the moon and a rocket with two parts lab Now you want to use this former project (labeled-data) and work on a new dataset (videos) with one moon but multiple (3) rockets. Then convert it as follows: + ``` individuals: [rocket1, rocket2, rocket3] uniquebodyparts: [moon] multianimalbodyparts: [rocket_tip,rocket_bottom] skeleton: [[[rocket_tip,rocket_bottom]]] ``` + In the unusual case, that your data also has multiple moons (e.g. is now carried out around Jupiter), but one rocket: + ``` individuals: [Io, Europa, Ganymede, Callisto] uniquebodyparts: [rocket_tip,rocket_bottom] multianimalbodyparts: [moon] ``` + Note you can use the single object tracker for this situation. What if you have multiple moons and rockets? diff --git a/docs/course.md b/docs/course.md index 58c9d153c0..83d23e5848 100644 --- a/docs/course.md +++ b/docs/course.md @@ -3,17 +3,21 @@ deeplabcut: last_content_updated: '2025-06-30' last_metadata_updated: '2026-03-06' ignore: false + visibility: online + status: outdated + recommendation: archive --- + # DeepLabCut Self-paced Course -::::{warning} +```{warning} This course was designed for DLC 2. An updated version for DLC 3 is in the works. -:::: +``` Do you have video of animal behaviors? Step 1: Get Poses ... - DLC LIVE! +DLC LIVE! This document is an outline of resources for a course for those wanting to learn to use `Python` and `DeepLabCut`. We expect it to take *roughly* 1-2 weeks to get through if you do it rigorously. To get the basics, it should take 1-2 days. @@ -24,14 +28,13 @@ We expect it to take *roughly* 1-2 weeks to get through if you do it rigorously.

- -## Installation: +## Installation You need Python and DeepLabCut installed! -- [See these "beginner docs" for help!](beginners-guide) -- **WATCH:** overview of conda: [Python Tutorial: Anaconda - Installation and Using Conda](https://www.youtube.com/watch?v=YJC6ldI3hWk) +- See the {ref}`file:beginners-guide` for help! +- **WATCH:** overview of conda: [Python Tutorial: Anaconda - Installation and Using Conda](https://www.youtube.com/watch?v=YJC6ldI3hWk) ## Outline: @@ -44,93 +47,91 @@ You need Python and DeepLabCut installed! - **Learning:** learning and teaching signal processing, and overview from Prof. Demba Ba [talk at JupyterCon](https://www.youtube.com/watch?v=ywz-LLYwkQQ) - **DEMO:** Can I DEMO DEEPLABCUT (DLC) quickly? - - Yes: [you can click through this DEMO notebook](https://github.com/DeepLabCut/DeepLabCut/blob/main/examples/COLAB/COLAB_DEMO_mouse_openfield.ipynb) - - AND follow along with me: [Video Tutorial!](https://www.youtube.com/watch?v=DRT-Cq2vdWs) + - Yes: [you can click through this DEMO notebook](https://github.com/DeepLabCut/DeepLabCut/blob/main/examples/COLAB/COLAB_DEMO_mouse_openfield.ipynb) + - AND follow along with me: [Video Tutorial!](https://www.youtube.com/watch?v=DRT-Cq2vdWs) - **WATCH:** How do you know DLC is installed properly? (i.e. how to use our test script!) [Video Tutorial!](https://youtu.be/IOWtKn3l33s) - review! - **REVIEW PAPER:** The state of animal pose estimation w/ deep learning i.e. "Deep learning tools for the measurement of animal behavior in neuroscience" [arXiv](https://arxiv.org/abs/1909.13868) & [published version](https://www.sciencedirect.com/science/article/pii/S0959438819301151) - **REVIEW PAPER:** [A Primer on Motion Capture with Deep Learning: Principles, Pitfalls and Perspectives](https://www.sciencedirect.com/science/article/pii/S0896627320307170) - - **WATCH:** There are a lot of docs... where to begin: [Video Tutorial!](https://www.youtube.com/watch?v=A9qZidI7tL8) ### **Module 1: getting started on data** **What you need:** any videos where you can see the animals/objects, etc. You can use our demo videos, grab some from the internet, or use whatever older data you have. Any camera, color/monochrome, etc will work. Find diverse videos, and label what you want to track well :) -- IF YOU ARE PART OF THE COURSE: you will be contributing to the DLC Model Zoo 😊 - - **Slides:** [Overview of starting new projects](https://github.com/DeepLabCut/DeepLabCut-Workshop-Materials/blob/main/part1-labeling.pdf) - - **READ ME PLEASE:** [DeepLabCut, the science](https://rdcu.be/4Rep) - - **READ ME PLEASE:** [DeepLabCut, the user guide](https://rdcu.be/bHpHN) - - **WATCH:** Video tutorial 1: [using the Project Manager GUI](https://www.youtube.com/watch?v=KcXogR-p5Ak) - - Please go from project creation (use >1 video!) to labeling your data, and then check the labels! - - **WATCH:** Video tutorial 2: [using the Project Manager GUI for multi-animal pose estimation](https://www.youtube.com/watch?v=Kp-stcTm77g) - - Please go from project creation (use >1 video!) to labeling your data, and then check the labels! - - **WATCH:** Video tutorial 3: [using ipython/pythonw (more functions!)](https://www.youtube.com/watch?v=7xwOhUcIGio) - - multi-animal DLC: [labeling](https://www.youtube.com/watch?v=Kp-stcTm77g) - - Please go from project creation (use >1 video!) to labeling your data, and then check the labels! +- IF YOU ARE PART OF THE COURSE: you will be contributing to the DLC Model Zoo 😊 + - **Slides:** [Overview of starting new projects](https://github.com/DeepLabCut/DeepLabCut-Workshop-Materials/blob/main/part1-labeling.pdf) + - **READ ME PLEASE:** [DeepLabCut, the science](https://rdcu.be/4Rep) + - **READ ME PLEASE:** [DeepLabCut, the user guide](https://rdcu.be/bHpHN) + - **WATCH:** Video tutorial 1: [using the Project Manager GUI](https://www.youtube.com/watch?v=KcXogR-p5Ak) + - Please go from project creation (use >1 video!) to labeling your data, and then check the labels! + - **WATCH:** Video tutorial 2: [using the Project Manager GUI for multi-animal pose estimation](https://www.youtube.com/watch?v=Kp-stcTm77g) + - Please go from project creation (use >1 video!) to labeling your data, and then check the labels! + - **WATCH:** Video tutorial 3: [using ipython/pythonw (more functions!)](https://www.youtube.com/watch?v=7xwOhUcIGio) + - multi-animal DLC: [labeling](https://www.youtube.com/watch?v=Kp-stcTm77g) + - Please go from project creation (use >1 video!) to labeling your data, and then check the labels! ### **Module 2: Neural Networks** - - **Slides:** [Overview of creating training and test data, and training networks](https://github.com/DeepLabCut/DeepLabCut-Workshop-Materials/blob/main/part2-network.pdf) - - **READ ME PLEASE:** [What are convolutional neural networks?](https://towardsdatascience.com/a-comprehensive-guide-to-convolutional-neural-networks-the-eli5-way-3bd2b1164a53) +- **Slides:** [Overview of creating training and test data, and training networks](https://github.com/DeepLabCut/DeepLabCut-Workshop-Materials/blob/main/part2-network.pdf) + +- **READ ME PLEASE:** [What are convolutional neural networks?](https://towardsdatascience.com/a-comprehensive-guide-to-convolutional-neural-networks-the-eli5-way-3bd2b1164a53) - - **READ ME PLEASE:** Here is a new paper from us describing challenges in robust pose estimation, why PRE-TRAINING really matters - which was our major scientific contribution to low-data input pose-estimation - and it describes new networks that are available to you. [Pretraining boosts out-of-domain robustness for pose estimation](https://paperswithcode.com/paper/pretraining-boosts-out-of-domain-robustness) +- **READ ME PLEASE:** Here is a new paper from us describing challenges in robust pose estimation, why PRE-TRAINING really matters - which was our major scientific contribution to low-data input pose-estimation - and it describes new networks that are available to you. [Pretraining boosts out-of-domain robustness for pose estimation](https://paperswithcode.com/paper/pretraining-boosts-out-of-domain-robustness) - - **MORE DETAILS:** ImageNet: check out the original paper and dataset: http://www.image-net.org/ + - **MORE DETAILS:** ImageNet: check out the original paper and dataset: http://www.image-net.org/ - - **REVIEW PAPER:** [A Primer on Motion Capture with Deep Learning: Principles, Pitfalls and Perspectives](https://www.sciencedirect.com/science/article/pii/S0896627320307170) +- **REVIEW PAPER:** [A Primer on Motion Capture with Deep Learning: Principles, Pitfalls and Perspectives](https://www.sciencedirect.com/science/article/pii/S0896627320307170) +review! - review! +Before you create a training/test set, please read/watch: - Before you create a training/test set, please read/watch: - - **More information:** [Which types neural networks are available, and what should I use?](https://github.com/DeepLabCut/DeepLabCut/wiki/What-neural-network-should-I-use%3F-(Trade-offs,-speed-performance,-and-considerations)) - - **WATCH:** Video tutorial 1: [How to test different networks in a controlled way](https://www.youtube.com/watch?v=WXCVr6xAcCA) - - Now, decide what model(s) you want to test. - - IF you want to train on your CPU, then run the step `create_training_dataset`, in the GUI etc. on your own computer. - - IF you want to use GPUs on google colab, [**(1)** watch this FIRST/follow along here!](https://www.youtube.com/watch?v=qJGs8nxx80A) **(2)** move your whole project folder to Google Drive, and then [**use this notebook**](https://github.com/DeepLabCut/DeepLabCut/blob/main/examples/COLAB/COLAB_YOURDATA_TrainNetwork_VideoAnalysis.ipynb) +- **More information:** [Which types neural networks are available, and what should I use?]() +- **WATCH:** Video tutorial 1: [How to test different networks in a controlled way](https://www.youtube.com/watch?v=WXCVr6xAcCA) + - Now, decide what model(s) you want to test. - **MODULE 2 webinar**: https://youtu.be/ILsuC4icBU0 + - IF you want to train on your CPU, then run the step `create_training_dataset`, in the GUI etc. on your own computer. + - IF you want to use GPUs on google colab, [**(1)** watch this FIRST/follow along here!](https://www.youtube.com/watch?v=qJGs8nxx80A) **(2)** move your whole project folder to Google Drive, and then [**use this notebook**](https://github.com/DeepLabCut/DeepLabCut/blob/main/examples/COLAB/COLAB_YOURDATA_TrainNetwork_VideoAnalysis.ipynb) + **MODULE 2 webinar**: https://youtu.be/ILsuC4icBU0 ### **Module 3: Evaluation of network performance** - - **Slides** [Evaluate your network](https://github.com/DeepLabCut/DeepLabCut-Workshop-Materials/blob/master/part3-analysis.pdf) - - **WATCH:** [Evaluate the network in ipython](https://www.youtube.com/watch?v=bgfnz1wtlpo) - - why evaluation matters; how to benchmark; analyzing a video and using scoremaps, conf. readouts, etc. +- **Slides** [Evaluate your network](https://github.com/DeepLabCut/DeepLabCut-Workshop-Materials/blob/master/part3-analysis.pdf) +- **WATCH:** [Evaluate the network in ipython](https://www.youtube.com/watch?v=bgfnz1wtlpo) + - why evaluation matters; how to benchmark; analyzing a video and using scoremaps, conf. readouts, etc. ### **Module 4: Scaling your analysis to many new videos** Once you have good networks, you can deploy them. You can create "cron jobs" to run a timed analysis script, for example. We run this daily on new videos collected in the lab. Check out a simple script to get started, and read more below: - - [Analyzing videos in batches, over many folders, setting up automated data processing](https://github.com/DeepLabCut/DLCutils/tree/master/SCALE_YOUR_ANALYSIS) +- [Analyzing videos in batches, over many folders, setting up automated data processing](https://github.com/DeepLabCut/DLCutils/tree/master/SCALE_YOUR_ANALYSIS) - - How to automate your analysis in the lab: [datajoint.io](https://datajoint.io), Cron Jobs: [schedule your code runs](https://www.ostechnix.com/a-beginners-guide-to-cron-jobs/) +- How to automate your analysis in the lab: [datajoint.io](https://datajoint.io), Cron Jobs: [schedule your code runs](https://www.ostechnix.com/a-beginners-guide-to-cron-jobs/) ### **Module 5: Got Poses? Now what ...** Pose estimation took away the painful part of digitizing your data, but now what? There is a rich set of tools out there to help you create your own custom analysis, or use others (and edit them to your needs). Check out more below: - - [Helper code and packages for use on DLC outputs](https://github.com/DeepLabCut/DLCutils) +- [Helper code and packages for use on DLC outputs](https://github.com/DeepLabCut/DLCutils) - - Create your own machine learning classifiers: https://scikit-learn.org/stable/ +- Create your own machine learning classifiers: https://scikit-learn.org/stable/ - - **REVIEW PAPER:** [Toward a Science of Computational Ethology](https://www.sciencedirect.com/science/article/pii/S0896627314007934) +- **REVIEW PAPER:** [Toward a Science of Computational Ethology](https://www.sciencedirect.com/science/article/pii/S0896627314007934) - - **REVIEW PAPER:** The state of animal pose estimation w/ deep learning i.e. "Deep learning tools for the measurement of animal behavior in neuroscience" [arXiv](https://arxiv.org/abs/1909.13868) & [published version](https://www.sciencedirect.com/science/article/pii/S0959438819301151) - - - **REVIEW PAPER:** [Big behavior: challenges and opportunities in a new era of deep behavior profiling](https://www.nature.com/articles/s41386-020-0751-7) +- **REVIEW PAPER:** The state of animal pose estimation w/ deep learning i.e. "Deep learning tools for the measurement of animal behavior in neuroscience" [arXiv](https://arxiv.org/abs/1909.13868) & [published version](https://www.sciencedirect.com/science/article/pii/S0959438819301151) - - **READ**: [Automated measurement of mouse social behaviors using depth sensing, video tracking, and machine learning](https://www.pnas.org/content/112/38/E5351) +- **REVIEW PAPER:** [Big behavior: challenges and opportunities in a new era of deep behavior profiling](https://www.nature.com/articles/s41386-020-0751-7) +- **READ**: [Automated measurement of mouse social behaviors using depth sensing, video tracking, and machine learning](https://www.pnas.org/content/112/38/E5351) *compiled and edited by Mackenzie Mathis* diff --git a/docs/dlc-live/deeplabcutlive.md b/docs/dlc-live/deeplabcutlive.md index 1d4b38d9ff..819c4eac98 100644 --- a/docs/dlc-live/deeplabcutlive.md +++ b/docs/dlc-live/deeplabcutlive.md @@ -3,7 +3,9 @@ deeplabcut: last_metadata_updated: '2026-03-17' ignore: false --- + (deeplabcut-live)= + # Running DeepLabCut models in real-time We provide two additional packages that allow you to record and stream camera data and run DeepLabCut models in real-time. diff --git a/docs/dlc-live/dlc-live-gui/index.md b/docs/dlc-live/dlc-live-gui/index.md index 8f82588b2d..27ea5c7d0e 100644 --- a/docs/dlc-live/dlc-live-gui/index.md +++ b/docs/dlc-live/dlc-live-gui/index.md @@ -3,6 +3,7 @@ deeplabcut: last_metadata_updated: '2026-03-17' ignore: false --- + # DeepLabCut-live-GUI A graphical application for **real-time pose estimation with DeepLabCut** using one or more cameras. @@ -20,7 +21,7 @@ This GUI is designed for **scientists and experimenters** who want to preview, r Please be aware of the {ref}`sec:dlclivegui-index-limitations` ``` ---- +______________________________________________________________________ ## Description @@ -34,16 +35,17 @@ Please be aware of the {ref}`sec:dlclivegui-index-limitations` - **Optional processor plugins** to extend behavior (e.g. remote control, triggers) The application is built with **PySide6 (Qt)** and is intended for interactive experimental use rather than offline batch processing. + ### Typical workflow 1. **Install** the application and required camera backends -2. **Configure cameras** (single or multi-camera) -3. **Select a DeepLabCut Live model** -4. **Start preview** and verify frame rate -5. **Run pose inference** on a selected camera -6. **Record video** (optionally with overlays) +1. **Configure cameras** (single or multi-camera) +1. **Select a DeepLabCut Live model** +1. **Start preview** and verify frame rate +1. **Run pose inference** on a selected camera +1. **Record video** (optionally with overlays) - With **organized results** by session and run Each of these steps is covered in the *{doc}`Quickstart `* @@ -55,8 +57,10 @@ and *{doc}`User Guide `* sections of this documentation. - Experimentalists running real-time tracking - Anyone who wants a **GUI-first** workflow for DeepLabCut Live ---- +______________________________________________________________________ + (sec:dlclivegui-index-limitations)= + ## Current limitations Before getting started, be aware of the following constraints: @@ -71,7 +75,7 @@ Before getting started, be aware of the following constraints: - **Performance** depends on camera resolution, frame rate, GPU availability, and codec choice - Expect bottlenecks with heavy models, multiple high-resolution cameras, or CPU-only inference. ---- +______________________________________________________________________ ## Feedback, issues, and contributions diff --git a/docs/dlc-live/dlc-live-gui/quickstart/install.md b/docs/dlc-live/dlc-live-gui/quickstart/install.md index a9f0ccc228..ff9a009ca9 100644 --- a/docs/dlc-live/dlc-live-gui/quickstart/install.md +++ b/docs/dlc-live/dlc-live-gui/quickstart/install.md @@ -3,6 +3,7 @@ deeplabcut: last_metadata_updated: '2026-03-17' ignore: false --- + # Installation This page explains how to install **DeepLabCut-live-GUI** for interactive, real‑time pose estimation. @@ -13,7 +14,7 @@ We support various installation methods, including `uv` and `mamba`/`conda`. If you feel confident you meet the requirements and you just want to get started quickly, see the {ref}`sec:dlclivegui-install-quickstart` section below. ``` ---- +______________________________________________________________________ ## System requirements @@ -25,11 +26,11 @@ If you feel confident you meet the requirements and you just want to get started ### OS support -| OS | PyTorch | TensorFlow | Notes & recommendations | -| -- | ------- | ---------- | ----- | -| Windows | ✅ | ❌ | Limited TensorFlow support due to lack of official Windows builds for Python 3.11+ onwards | -| Linux | ✅ | ✅ | Full support for both backends | -| macOS | ✅ | ⚠️ | PyTorch MPS support is improving but still has limitations; TensorFlow only supports CPU on macOS | +| OS | PyTorch | TensorFlow | Notes & recommendations | +| ------- | ------- | ---------- | ------------------------------------------------------------------------------------------------- | +| Windows | ✅ | ❌ | Limited TensorFlow support due to lack of official Windows builds for Python 3.11+ onwards | +| Linux | ✅ | ✅ | Full support for both backends | +| macOS | ✅ | ⚠️ | PyTorch MPS support is improving but still has limitations; TensorFlow only supports CPU on macOS | ### Hardware requirements @@ -54,8 +55,10 @@ If you use an OpenCV-compatible camera (e.g. USB webcam, OBS virtual camera), yo - **TensorFlow** (for backwards compatibility with existing models) - A working camera backend (see *{ref}`file:dlclivegui-camera-support`*) ---- +______________________________________________________________________ + (sec:dlclivegui-install-quickstart)= + ## Quickstart (recommended defaults) ```bash diff --git a/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/aravis_backend.md b/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/aravis_backend.md index 9cacbe8bb4..e51fb670e8 100644 --- a/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/aravis_backend.md +++ b/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/aravis_backend.md @@ -3,7 +3,9 @@ deeplabcut: last_metadata_updated: '2026-03-17' ignore: false --- + (file:dlclivegui-camera-aravis-backend)= + # Aravis backend The Aravis backend provides support for GenICam-compatible cameras using the @@ -14,7 +16,7 @@ Support for Aravis in the GUI is currently experimental. Please report issues on GitHub to help improve this backend. ``` ---- +______________________________________________________________________ ## Features @@ -61,7 +63,7 @@ dependencies such as `gobject-introspection` and `cairo`. ```` ````` ---- +______________________________________________________________________ ## Basic configuration @@ -79,7 +81,7 @@ Select the Aravis backend either in the GUI or via configuration: } ``` ---- +______________________________________________________________________ ## Camera selection @@ -115,7 +117,7 @@ The backend may automatically populate additional read-only identity fields used internally and set by the GUI. ``` ---- +______________________________________________________________________ ## Full properties and advanced configuration @@ -128,13 +130,13 @@ the settings used by the GUI and configuration files. These values are accessible directly in the GUI and are shared for all backends. ``` -| Property | Type | Description | -|--------|------|-------------| -| `width` | int | Requested image width (optional) | -| `height` | int | Requested image height (optional) | -| `fps` | float | Target acquisition frame rate | -| `exposure` | float | Exposure time in microseconds | -| `gain` | float | Camera gain value | +| Property | Type | Description | +| ---------- | ----- | --------------------------------- | +| `width` | int | Requested image width (optional) | +| `height` | int | Requested image height (optional) | +| `fps` | float | Target acquisition frame rate | +| `exposure` | float | Exposure time in microseconds | +| `gain` | float | Camera gain value | ### Common Aravis properties @@ -142,12 +144,12 @@ These values are accessible directly in the GUI and are shared for all backends. These properties are specific to the Aravis backend and must be set manually in the configuration file. ``` -| Property | Type | Default | Description | -|--------|------|---------|-------------| -| `device_id` | string | — | Explicit Aravis device ID (overrides index) | -| `pixel_format` | string | `Mono8` | Requested pixel format | -| `timeout` | int | `2000000` | Frame timeout in microseconds | -| `n_buffers` | int | `10` | Number of streaming buffers | +| Property | Type | Default | Description | +| -------------- | ------ | --------- | ------------------------------------------- | +| `device_id` | string | — | Explicit Aravis device ID (overrides index) | +| `pixel_format` | string | `Mono8` | Requested pixel format | +| `timeout` | int | `2000000` | Frame timeout in microseconds | +| `n_buffers` | int | `10` | Number of streaming buffers | ### Pixel format @@ -277,7 +279,7 @@ Adjust frame timeout for slower cameras or congested networks: (5 seconds = 5,000,000 microseconds) ---- +______________________________________________________________________ ## Troubleshooting @@ -287,8 +289,8 @@ Adjust frame timeout for slower cameras or congested networks: ```bash arv-tool-0.8 -l ``` -2. Check power, cabling, and network configuration -3. Ensure sufficient permissions for USB or network devices +1. Check power, cabling, and network configuration +1. Ensure sufficient permissions for USB or network devices ### Timeout errors @@ -304,19 +306,19 @@ Adjust frame timeout for slower cameras or congested networks: ``` - Try a simpler format such as `Mono8` ---- +______________________________________________________________________ ## Comparison with GenTL backend -| Feature | Aravis | GenTL | -| ------- | ------ | ----- | -| Best Platform | Linux | Windows | -| Camera Support | GenICam / GigE | Vendor GenTL | -| Installation | System packages | Vendor CTI files | -| Auto-detection | Yes | Yes | -| Performance | Excellent | Excellent | +| Feature | Aravis | GenTL | +| -------------- | --------------- | ---------------- | +| Best Platform | Linux | Windows | +| Camera Support | GenICam / GigE | Vendor GenTL | +| Installation | System packages | Vendor CTI files | +| Auto-detection | Yes | Yes | +| Performance | Excellent | Excellent | ---- +______________________________________________________________________ ## Example configuration @@ -339,7 +341,7 @@ Adjust frame timeout for slower cameras or congested networks: } ``` ---- +______________________________________________________________________ ## Resources diff --git a/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/basler_backend.md b/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/basler_backend.md index 7bd5b7097b..1350ec64f8 100644 --- a/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/basler_backend.md +++ b/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/basler_backend.md @@ -3,7 +3,9 @@ deeplabcut: last_metadata_updated: '2026-03-17' ignore: false --- + (file:dlclivegui-basler-backend)= + # Basler backend The Basler backend provides support for Basler cameras using the official **pylon SDK** through the **pypylon** Python bindings. @@ -14,7 +16,7 @@ Download the official pylon SDK from Basler and install the `pypylon` Python pac This backend requires the optional `pypylon` dependency. If `pypylon` is not installed, the backend will be unavailable. ``` ---- +______________________________________________________________________ ## Features & design @@ -24,7 +26,7 @@ This backend requires the optional `pypylon` dependency. If `pypylon` is not ins - Configurable exposure, gain, frame rate, and resolution. - Frames are converted to **BGR (8-bit)** for consistency with other GUI backends. ---- +______________________________________________________________________ ## Installation @@ -67,7 +69,7 @@ pip install pypylon # OR uv pip install pypylon `pypylon` is the official Python wrapper for the Basler pylon Camera Software Suite ---- +______________________________________________________________________ ## Basic configuration @@ -85,7 +87,7 @@ Select the Basler backend in the GUI or via configuration: } ``` ---- +______________________________________________________________________ ## Camera selection @@ -121,9 +123,9 @@ The backend supports a stable identity field `device_id` (serial number). When p How selection works: 1. If `properties.basler.device_id` is set, the backend selects the device with a matching serial number. -2. Otherwise, the backend uses `index`. +1. Otherwise, the backend uses `index`. ---- +______________________________________________________________________ ## Full properties and advanced configuration @@ -159,7 +161,7 @@ After a successful open, the backend may populate the following read-only conven These fields are managed automatically and are not required to configure the backend. ---- +______________________________________________________________________ ### Exposure and gain @@ -181,7 +183,7 @@ Example: } ``` ---- +______________________________________________________________________ ### Frame rate (FPS) @@ -189,7 +191,7 @@ Example: - The backend attempts to enable `AcquisitionFrameRateEnable` when available, then sets `AcquisitionFrameRate`. - The backend reads back the **actual FPS** (if available) and exposes it via telemetry. ---- +______________________________________________________________________ ### Resolution handling @@ -198,7 +200,7 @@ Resolution is only changed when explicitly requested. Priority order for requesting a resolution: 1. `width` + `height` (GUI fields) -2. `properties.basler.resolution` (namespaced override) +1. `properties.basler.resolution` (namespaced override) If no resolution is provided (or if width/height are `0`), the backend preserves the camera’s default configuration. @@ -208,7 +210,7 @@ Increment and range constraints: - The backend snaps requested values down to the nearest valid increment (best-effort) and clamps to min/max. - A warning is logged if the requested and applied resolutions differ. ---- +______________________________________________________________________ ### Pixel format and color conversion @@ -218,7 +220,7 @@ To provide a consistent frame format across backends, the Basler backend convert Internally, it uses a pypylon `ImageFormatConverter` configured for `PixelType_BGR8packed`. ---- +______________________________________________________________________ ### Device discovery @@ -231,7 +233,7 @@ The backend can enumerate devices without opening them and returns (best-effort) Note that availability and richness of fields depend on camera transport and SDK support. ---- +______________________________________________________________________ ## Troubleshooting @@ -246,7 +248,6 @@ Note that availability and richness of fields depend on camera transport and SDK pip install pypylon ``` - ### No cameras detected - Verify the Basler pylon runtime is installed and your camera is visible in Basler tooling. @@ -256,7 +257,7 @@ Note that availability and richness of fields depend on camera transport and SDK If you request a resolution that violates camera constraints (min/max or increment), the backend will snap/clamp to valid values and log a warning. ---- +______________________________________________________________________ ## Example configuration @@ -279,7 +280,7 @@ If you request a resolution that violates camera constraints (min/max or increme } ``` ---- +______________________________________________________________________ ## Resources diff --git a/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/camera_support.md b/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/camera_support.md index a3b7e1c440..007a8e0ac3 100644 --- a/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/camera_support.md +++ b/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/camera_support.md @@ -3,7 +3,9 @@ deeplabcut: last_metadata_updated: '2026-03-17' ignore: false --- + (file:dlclivegui-camera-support)= + # Camera support DeepLabCut-live-GUI supports multiple camera backends for different platforms and camera types: @@ -12,10 +14,10 @@ DeepLabCut-live-GUI supports multiple camera backends for different platforms an 1. {ref}`OpenCV ` - "Universal" webcam and USB camera support *(all platforms)* - Expect some limitations in camera control and performance -2. {ref}`GenTL ` - Industrial cameras via GenTL producers *(Windows, Linux)* - - Requires vendor-provided CTI files -3. {ref}`Aravis ` - GenICam/GigE Vision cameras *(Linux, experimental on macOS)* -4. {ref}`Basler ` - Basler cameras via pypylon *(all platforms)* +1. {ref}`GenTL ` - Industrial cameras via GenTL producers *(Windows, Linux)* + - Requires vendor-provided CTI files +1. {ref}`Aravis ` - GenICam/GigE Vision cameras *(Linux, experimental on macOS)* +1. {ref}`Basler ` - Basler cameras via pypylon *(all platforms)* ## Backend selection @@ -87,10 +89,10 @@ Install vendor-provided camera drivers and SDK. CTI files are typically in: ## Backend comparison -| Feature | OpenCV | GenTL | Aravis | Basler (pypylon) | -|---------|--------|-------|--------|------------------| -| Exposure control | No | Yes | Yes | Yes | -| Gain control | No | Yes | Yes | Yes | -| Windows | ✅ | ✅ | ❌ | ✅ | -| Linux | ✅ | ✅ | ✅ | ✅ | -| macOS | ✅ | ❌ | ⚠️ | ✅ | +| Feature | OpenCV | GenTL | Aravis | Basler (pypylon) | +| ---------------- | ------ | ----- | ------ | ---------------- | +| Exposure control | No | Yes | Yes | Yes | +| Gain control | No | Yes | Yes | Yes | +| Windows | ✅ | ✅ | ❌ | ✅ | +| Linux | ✅ | ✅ | ✅ | ✅ | +| macOS | ✅ | ❌ | ⚠️ | ✅ | diff --git a/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/gentl_backend.md b/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/gentl_backend.md index 84a53aed1a..721ab4ac30 100644 --- a/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/gentl_backend.md +++ b/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/gentl_backend.md @@ -3,6 +3,7 @@ deeplabcut: last_metadata_updated: '2026-03-17' ignore: false --- + # GenTL backend The GenTL backend provides support for **GenICam / GenTL** compatible cameras using the **Harvesters** Python library (a GenTL consumer). @@ -16,7 +17,7 @@ Support for GenTL in the GUI is currently experimental. Please report issues on GitHub to help improve this backend. ``` ---- +______________________________________________________________________ ## Features & design @@ -38,7 +39,7 @@ Please report issues on GitHub to help improve this backend. - `RGB8` → BGR - Non-8-bit frames → scaled down to 8-bit (per-frame scaling) ---- +______________________________________________________________________ ## Installation @@ -75,7 +76,7 @@ If you have multiple producers installed, separate entries with: Many vendor installers set `GENICAM_GENTL64_PATH` automatically. If your camera is not discovered, explicitly set the variable (or provide `cti_file` / `cti_files` in configuration as described below). ``` ---- +______________________________________________________________________ ## Basic configuration @@ -99,7 +100,7 @@ Select the GenTL backend in the GUI or via configuration: } ``` ---- +______________________________________________________________________ ## CTI / producer configuration @@ -118,20 +119,24 @@ By default, the backend will **discover** and **try to load all available** GenT CTI locations are resolved in this order: 1. **Namespace explicit CTIs** (`properties.gentl`): + - `properties.gentl.cti_files` - `properties.gentl.cti_file` Behavior depends on the persisted source marker `properties.gentl.cti_files_source`: - If `cti_files_source == "user"` (or missing/unknown): + - Treated as a **user override** - **strict**: missing paths cause `open()` to raise - If `cti_files_source == "auto"`: + - Treated as an **auto-discovered cache** - If cached paths are stale/missing, `open()` will **fall back to discovery** automatically -2. **Discovery** (auto): +1. **Discovery** (auto): + - environment: `GENICAM_GENTL64_PATH` / `GENICAM_GENTL32_PATH` - optional: `properties.gentl.cti_search_paths` (glob patterns) - optional: `properties.gentl.cti_dirs` (extra directories; non-recursive) @@ -224,7 +229,7 @@ After `open()` (success or failure), the backend writes: These fields are intended for UI troubleshooting and do not normally need manual edits. ---- +______________________________________________________________________ ## Camera selection and stable identity @@ -264,9 +269,9 @@ Prefer `properties.gentl.device_id`, which is persisted automatically after a su The backend selects a device in this order: 1. Exact match of `device_id` against computed IDs for discovered devices -2. If `device_id` starts with `serial:`, match by exact serial number, then (if needed) substring -3. Legacy serial keys (`serial_number` / `serial`) if present (exact then substring) -4. Fallback to `index` +1. If `device_id` starts with `serial:`, match by exact serial number, then (if needed) substring +1. Legacy serial keys (`serial_number` / `serial`) if present (exact then substring) +1. Fallback to `index` If a serial substring matches **multiple** cameras, an “ambiguous” error is raised. @@ -274,7 +279,7 @@ If a serial substring matches **multiple** cameras, an “ambiguous” error is The backend updates `settings.index` to the selected device’s current index to improve UI stability. ``` ---- +______________________________________________________________________ ### Automated rebind (index changes, reconnects) @@ -288,9 +293,9 @@ When the UI restarts (or devices re-enumerate), the backend can **rebind setting Matching strategy: 1. Exact match on computed `device_id` -2. Fallback: treat stored value as a serial-like substring and match the first serial containing it +1. Fallback: treat stored value as a serial-like substring and match the first serial containing it ---- +______________________________________________________________________ ## Camera settings @@ -302,7 +307,7 @@ These settings are shared across backends and configurable in the GUI: - `exposure` (float): exposure time; `<= 0` means do not set - `gain` (float): gain value; `<= 0` means do not set ---- +______________________________________________________________________ ## Full properties and advanced configuration @@ -341,7 +346,7 @@ Probe / telemetry: - `cti_files_loaded` (list[string]): populated automatically after open - `cti_files_failed` (list[object]): populated automatically after open; each entry has `cti` and `error` ---- +______________________________________________________________________ ### Pixel format @@ -355,7 +360,7 @@ Frames are normalized to **BGR (8-bit)**: - `RGB8` is converted to BGR - Higher bit-depth images are scaled to 8-bit based on the frame’s max value (per frame) ---- +______________________________________________________________________ ### Exposure and gain @@ -370,7 +375,7 @@ Best-effort behavior (depends on producer + camera GenApi implementation): If nodes are missing or read-only, the backend logs a warning and continues. ---- +______________________________________________________________________ ### Frame rate (FPS) @@ -385,7 +390,7 @@ If `fps` is set to a non-zero value: The backend also tries to read back `ResultingFrameRate` for GUI telemetry (`actual_fps`). ---- +______________________________________________________________________ ### Resolution handling @@ -397,7 +402,7 @@ Resolution is applied **only when explicitly requested** (either `width+height`, If no resolution is specified, the device’s current/default configuration is preserved. ---- +______________________________________________________________________ ### Streaming and probe mode @@ -414,7 +419,7 @@ If `properties.gentl.fast_start` is `true`: This is intended for capability probing and faster startup of probe workers. ---- +______________________________________________________________________ ## Troubleshooting @@ -457,7 +462,7 @@ If you pinned CTIs as a user override and paths no longer exist, `open()` will f - Inspect available formats via vendor tools or by checking `PixelFormat.symbolics` - Try a simpler format such as `Mono8` ---- +______________________________________________________________________ ## Example configuration @@ -483,7 +488,7 @@ If you pinned CTIs as a user override and paths no longer exist, `open()` will f } ``` ---- +______________________________________________________________________ ## Resources diff --git a/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/opencv_backend.md b/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/opencv_backend.md index df58205f5b..18b0d7cbf9 100644 --- a/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/opencv_backend.md +++ b/docs/dlc-live/dlc-live-gui/user_guide/cameras_backends/opencv_backend.md @@ -3,7 +3,9 @@ deeplabcut: last_metadata_updated: '2026-03-17' ignore: false --- + (file:dlclivegui-opencv-backend)= + # OpenCV backend The OpenCV backend provides camera support via `cv2.VideoCapture`. @@ -15,7 +17,7 @@ Due to lack of standardization across OpenCV backends, exposure and gain control **Other settings may not always behave as expected due to driver and backend limitations.** ``` ---- +______________________________________________________________________ ## Features & design @@ -32,7 +34,7 @@ Due to lack of standardization across OpenCV backends, exposure and gain control - Mismatch handling is configurable (warn/strict/accept). - Optional MJPG (Windows) and explicit FOURCC requests. ---- +______________________________________________________________________ ## Dependencies information @@ -41,7 +43,7 @@ as part of the core DeepLabCut-Live-GUI package, so **no additional installation `cv2-enumerate-cameras` is also installed by default to provide camera enumeration support for this backend and make device selection more robust. ---- +______________________________________________________________________ ## Basic configuration @@ -64,7 +66,7 @@ Notes: - If `width`/`height` are omitted or set to `0`, the backend keeps the camera’s default mode. - OpenCV may ignore FPS and resolution requests depending on driver/backend. ---- +______________________________________________________________________ ## Camera selection configuration @@ -107,11 +109,11 @@ Example: Selection priority in `open()`: 1. `properties.opencv.device_id` (stable ID) -2. `properties.opencv.device_name` (substring match) -3. `properties.opencv.device_vid` + `device_pid` -4. `index` fallback +1. `properties.opencv.device_name` (substring match) +1. `properties.opencv.device_vid` + `device_pid` +1. `index` fallback ---- +______________________________________________________________________ ## Advanced configuration @@ -165,7 +167,7 @@ Codec policy: - `fourcc` (string | null): explicit FOURCC request, overrides `prefer_mjpg`. - Examples: `MJPG`, `YUY2`, `NV12`, `H264`, `XRGB`, `BGR3` ---- +______________________________________________________________________ ### Resolution and FPS behavior @@ -183,7 +185,7 @@ Codec policy: - If `fps > 0`, the backend attempts to set `CAP_PROP_FPS` best-effort. - Many drivers return `0.0` for FPS even when streaming successfully; this is normal for some OpenCV backends. ---- +______________________________________________________________________ ### Device discovery and rebind @@ -204,7 +206,7 @@ If enumeration is not available, `discover_devices()` returns `None` so the fact If `properties.opencv.device_id` (or VID/PID/name) exists, `rebind_settings()` attempts to map the saved identity to the current index and refresh stored fields. ---- +______________________________________________________________________ ## Troubleshooting @@ -228,7 +230,6 @@ Try: } ``` - ### Slow open on Windows (MSMF) If MSMF is selected and opening is slow, consider setting: @@ -249,6 +250,7 @@ If you request a resolution that the driver cannot apply, you may see warnings. On Windows, MJPG can reduce USB bandwidth and improve FPS for some webcams. - Enable MJPG attempt: + ```json { "camera": { @@ -261,6 +263,7 @@ On Windows, MJPG can reduce USB bandwidth and improve FPS for some webcams. ``` - Or force a specific FOURCC: + ```json { "camera": { @@ -272,7 +275,7 @@ On Windows, MJPG can reduce USB bandwidth and improve FPS for some webcams. } ``` ---- +______________________________________________________________________ ## Example configuration @@ -299,7 +302,7 @@ On Windows, MJPG can reduce USB bandwidth and improve FPS for some webcams. } ``` ---- +______________________________________________________________________ ## Notes and limitations diff --git a/docs/dlc-live/dlc-live-gui/user_guide/misc/misc_landing.md b/docs/dlc-live/dlc-live-gui/user_guide/misc/misc_landing.md index 40f8fa7d00..6ebe0d0a81 100644 --- a/docs/dlc-live/dlc-live-gui/user_guide/misc/misc_landing.md +++ b/docs/dlc-live/dlc-live-gui/user_guide/misc/misc_landing.md @@ -3,6 +3,7 @@ deeplabcut: last_metadata_updated: '2026-03-17' ignore: false --- + # Additional resources In this section, you can find additional resources related to the GUI and DLC-live, including: diff --git a/docs/dlc-live/dlc-live-gui/user_guide/misc/modelzoo_downloads.md b/docs/dlc-live/dlc-live-gui/user_guide/misc/modelzoo_downloads.md index 1cd04b34d9..9a64875687 100644 --- a/docs/dlc-live/dlc-live-gui/user_guide/misc/modelzoo_downloads.md +++ b/docs/dlc-live/dlc-live-gui/user_guide/misc/modelzoo_downloads.md @@ -3,7 +3,9 @@ deeplabcut: last_metadata_updated: '2026-03-17' ignore: false --- + (file:dlclivegui-pretrained-models)= + # Pre-trained models This page explains how to programmatically download and export **pre-trained, GUI-compatible** models from the DeepLabCut Model Zoo using the `dlclive.modelzoo` API, and convert them for use in DLC-live and by extension, the GUI. @@ -25,7 +27,7 @@ The example below is intended for the PyTorch engine. If you are using **TensorFlow models**, you will typically point the GUI to a DLC model *.pb file* instead of a model *.pth/.pt file*. ``` ---- +______________________________________________________________________ ## Quick start @@ -65,10 +67,10 @@ assert TORCH_CONFIG["checkpoint"].exists(), "Export failed" What this does: 1. Creates the destination directory if needed. -2. Downloads the correct model snapshot (weights) for the specified `super_animal` + `model_name`. -3. Writes a **single `.pt` export file** containing the model config and weights. +1. Downloads the correct model snapshot (weights) for the specified `super_animal` + `model_name`. +1. Writes a **single `.pt` export file** containing the model config and weights. ---- +______________________________________________________________________ ## API reference @@ -84,7 +86,7 @@ Behavior: - If `export_path` already exists, the function **skips** exporting (and emits a warning). - If `detector_name` is provided, it downloads and exports a top-down model with the detector weights as well. ---- +______________________________________________________________________ ## What gets saved in the exported `.pt` @@ -117,7 +119,7 @@ export_modelzoo_model( print(f"Exported model zoo checkpoint to: {export_path}") ``` ---- +______________________________________________________________________ ## In the future diff --git a/docs/dlc-live/dlc-live-gui/user_guide/misc/timestamp_format.md b/docs/dlc-live/dlc-live-gui/user_guide/misc/timestamp_format.md index e7aed2612e..edd0ff9c21 100644 --- a/docs/dlc-live/dlc-live-gui/user_guide/misc/timestamp_format.md +++ b/docs/dlc-live/dlc-live-gui/user_guide/misc/timestamp_format.md @@ -3,7 +3,9 @@ deeplabcut: last_metadata_updated: '2026-03-17' ignore: false --- + (file:dlclivegui-timestamp-format)= + # Video timestamp format When recording videos, the application automatically saves frame timestamps to a JSON file alongside the video file. @@ -17,7 +19,6 @@ please refer to the {ref}`sec:dlclivegui-recording-paths-info` section. For a video file named `recording_2025-10-23_143052.mp4`, the timestamp file will be: - ``` recording_2025-10-23_143052.mp4_timestamps.json ``` diff --git a/docs/dlc-live/dlc-live-gui/user_guide/overview.md b/docs/dlc-live/dlc-live-gui/user_guide/overview.md index 521b872dda..e94186a47d 100644 --- a/docs/dlc-live/dlc-live-gui/user_guide/overview.md +++ b/docs/dlc-live/dlc-live-gui/user_guide/overview.md @@ -3,13 +3,14 @@ deeplabcut: last_metadata_updated: '2026-03-17' ignore: false --- + # GUI overview DeepLabCut-live-GUI (`dlclivegui`) is a **PySide6-based desktop application** for running real-time DeepLabCut pose estimation experiments with **one or multiple cameras**, optional **processor plugins**, and **video recording** (with or without overlays). This page gives you a **guided tour of the main window**, explains the **core workflow**, and introduces the key concepts used throughout the user guide. ---- +______________________________________________________________________ ## Main window at a glance @@ -23,15 +24,15 @@ When you first launch the application, you will see the main window with three p - A **Video panel** (right) showing the live preview (single or tiled multi-camera) - A **Stats area** (below the video) summarizing camera, inference, and recorder performance -:::{figure} ../_static/images/main_window_100226.png +:::\{figure} ../\_static/images/main_window_100226.png :alt: Screenshot of the main window :width: 100% :align: center - The main window on startup, showing the Controls panel (left), Video panel (right), and Stats area (below video). +The main window on startup, showing the Controls panel (left), Video panel (right), and Stats area (below video). ::: ---- +______________________________________________________________________ ## Intended workflow @@ -41,23 +42,30 @@ as well as pick a model for pose inference. To start running an experiment, the typical workflow is: 1. **Configure Cameras** + - Use **Configure Cameras…** to select one or more cameras and their parameters. - See {ref}`file:dlclivegui-camera-support` for details on supported camera backends and troubleshooting. -2. **Start Preview** +1. **Start Preview** + - Click **Start Preview** to begin streaming all selected configured cameras. - If multiple cameras are active, the preview becomes a **tiled view**. -3. **Start Pose Inference** *(when ready)* +1. **Start Pose Inference** *(when ready)* + - Choose a **Model file**, optionally a DLC-live **Processor**[^processor-footnote], select the **Inference Camera**, then click **Start pose inference**. + + - Toggle **Display pose predictions** to show or hide pose estimation overlays. -4. **Start Recording** *(when ready)* +1. **Start Recording** *(when ready)* + - Choose an **Output directory**, session/run naming options, and encoding settings, then click **Start recording**. - Recording includes **all active cameras** in multi-camera mode in separate files. -5. **Stop** +1. **Stop** + - Use **Stop Preview**, **Stop pose inference**, and/or **Stop recording** as needed. ```{note} @@ -66,7 +74,7 @@ Pose inference requires the camera preview to be running. If you start pose inference while the preview is stopped, the GUI will automatically start the preview first. ``` ---- +______________________________________________________________________ ## Main control panel @@ -104,7 +112,7 @@ In multi-camera mode, pose inference runs on **one selected camera at a time** ( even though preview and recording may include multiple cameras. ``` ---- +______________________________________________________________________ ### DLCLive settings @@ -129,6 +137,7 @@ Find more information here if needed: {ref}`deeplabcut-live`. - **Start pose inference / Stop pose inference** The button indicates inference state: + - *Initializing DLCLive!* → Model loading - *DLCLive running!* → Inference active @@ -138,7 +147,7 @@ Find more information here if needed: {ref}`deeplabcut-live`. - **Processor Status** Displays processor-specific status information when available. ---- +______________________________________________________________________ ### Recording @@ -150,6 +159,7 @@ See {ref}`file:dlclivegui-timestamp-format` for details. ``` (sec:dlclivegui-recording-paths-info)= + #### Recording output options - **Output directory**: Base directory for all recordings @@ -179,7 +189,7 @@ You can hover over the preview path to see the full path, and click to copy it t - **Record video with overlays** Include pose predictions and/or bounding boxes directly in the recorded video. - :::{danger} + :::\{danger} This **cannot be easily undone** once the recording is saved. Use with caution if you want to preserve **raw footage** intact. @@ -195,7 +205,6 @@ Frame size must remain constant for a recording session. If the recorder is conf - Stop the recorder and start a new recording after fixing the frame size ``` - ```{note} Frames are converted automatically for encoding: @@ -204,7 +213,7 @@ Frames are converted automatically for encoding: - Frames are made contiguous in memory before being passed to the encoder. ``` ---- +______________________________________________________________________ ### Visualization settings @@ -222,7 +231,7 @@ To adjust the bounding box intuitively, hover over a coordinate field (`x0`, `y0 and drag horizontally. ``` ---- +______________________________________________________________________ ## Video Panel and Stats @@ -244,7 +253,7 @@ Three continuously updated sections: Stats text can be selected and copied directly from the GUI ``` ---- +______________________________________________________________________ ## Menu bar actions @@ -287,7 +296,7 @@ Configuration files store camera configurations, model paths, recording options, - **Ctrl+Shift+S**: Save configuration as... - **Ctrl+Q**: Quit application ---- +______________________________________________________________________ ## Configuration and Persistence diff --git a/docs/dlc-utils/XROMM/usage.md b/docs/dlc-utils/XROMM/usage.md new file mode 100644 index 0000000000..f581ce61f8 --- /dev/null +++ b/docs/dlc-utils/XROMM/usage.md @@ -0,0 +1,64 @@ +(file:xamalab-dlc-integration)= + +# XROMM + DeepLabCut local integration + +These notes describe how this repository is used in the local 3-repo XROMM workflow together with `../XROMM_DLCTools` and `../xmalab`. + +> Contributed by [@homfunc](https://github.com/homfunc) + +## 1) Expected local layout + +Recommended sibling checkout layout: + +- `XROMM_DLCTools/` +- `DeepLabCut/` +- `xmalab/` + `XROMM_DLCTools/pyproject.toml` maps its optional `dlc` dependency group to this repository through `tool.uv.sources`. + +## 2) DeepLabCut’s role in the workflow + +Within the integrated workflow, DeepLabCut provides: + +- project creation / dataset generation support +- video analysis / prediction entrypoints +- the local import target used by `XROMM_DLCTools` +- synthetic smoke coverage through the baseline harness + The current local integration suite also uses this repo to verify that the newer workflow service in `XROMM_DLCTools` still interoperates with a sibling DeepLabCut checkout. + +## 3) Local setup for this repo + +Standard developer setup: + +```bash +uv sync --group dev +``` + +When working from `../XROMM_DLCTools`, enable the sibling import path there with: + +```bash +uv sync --group dlc +``` + +## 4) Integration validation from XROMM_DLCTools + +Run these commands from `../XROMM_DLCTools`: + +```bash +uv run python scripts/baseline_harness.py --scenario deeplabcut_repo_smoke --output-dir baseline_artifacts/deeplabcut_smoke --deeplabcut-repo ../DeepLabCut +``` + +Full multi-repo suite: + +```bash +uv run python scripts/baseline_harness.py --scenario all --output-dir baseline_artifacts/integration_all --deeplabcut-repo ../DeepLabCut --xmalab-repo ../xmalab +``` + +True end-to-end local workflow scenario: + +```bash +uv run python scripts/baseline_harness.py --scenario phase3_local_workflow_e2e --output-dir baseline_artifacts/e2e_local_workflow --deeplabcut-repo ../DeepLabCut --xmalab-repo ../xmalab +``` + +## 5) Compatibility notes + +The local workflow integration path expects this repo to remain importable in “lite mode” when GUI dependencies are unavailable, and relies on the public `deeplabcut` import surface plus the synthetic project helpers under `examples/utils.py`. diff --git a/docs/dlc-utils/index.md b/docs/dlc-utils/index.md new file mode 100644 index 0000000000..36cdd16023 --- /dev/null +++ b/docs/dlc-utils/index.md @@ -0,0 +1,190 @@ +--- +deeplabcut: + last_metadata_updated: '2026-05-06' + last_verified: '2026-05-06' + verified_for: 3.0.0rc14 + ignore: false +--- + +# DeepLabCut-Utils - Community contributions + +[![Image.sc forum](https://img.shields.io/badge/dynamic/json.svg?label=forum&url=https%3A%2F%2Fforum.image.sc%2Ftags%2Fdeeplabcut.json&query=%24.topic_list.tags.0.topic_count&colorB=brightgreen&suffix=%20topics&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA4AAAAOCAYAAAAfSC3RAAABPklEQVR42m3SyyqFURTA8Y2BER0TDyExZ+aSPIKUlPIITFzKeQWXwhBlQrmFgUzMMFLKZeguBu5y+//17dP3nc5vuPdee6299gohUYYaDGOyyACq4JmQVoFujOMR77hNfOAGM+hBOQqB9TjHD36xhAa04RCuuXeKOvwHVWIKL9jCK2bRiV284QgL8MwEjAneeo9VNOEaBhzALGtoRy02cIcWhE34jj5YxgW+E5Z4iTPkMYpPLCNY3hdOYEfNbKYdmNngZ1jyEzw7h7AIb3fRTQ95OAZ6yQpGYHMMtOTgouktYwxuXsHgWLLl+4x++Kx1FJrjLTagA77bTPvYgw1rRqY56e+w7GNYsqX6JfPwi7aR+Y5SA+BXtKIRfkfJAYgj14tpOF6+I46c4/cAM3UhM3JxyKsxiOIhH0IO6SH/A1Kb1WBeUjbkAAAAAElFTkSuQmCC)](https://forum.image.sc/tags/deeplabcut) + +```{image} https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1572296495650-Y4ZTJ2XP2Z9XF1AD74VW/ke17ZwdGBToddI8pDm48kMulEJPOrz9Y8HeI7oJuXxR7gQa3H78H3Y0txjaiv_0fDoOvxcdMmMKkDsyUqMSsMWxHk725yiiHCCLfrh8O1z5QPOohDIaIeljMHgDF5CVlOqpeNLcJ80NK65_fV7S1UZiU3J6AN9rgO1lHw9nGbkYQrCLTag1XBHRgOrY8YAdXW07ycm2Trb21kYhaLJjddA/DLC_logo_blk-01.png?format=1000w +--- +alt: DLC Utils +width: 350px +align: right +--- +``` + +The DeepLabCut-Utils repository contains various scripts as well as links to other packages related to [DeepLabCut](https://github.com/DeepLabCut/DeepLabCut). Feel free to contribute your own analysis methods, and perhaps some short notebook of how to use it. Thanks! + +```{admonition} DLC-Utils +--- +class: tip +--- +[Link to repository](https://github.com/DeepLabCut/DLCutils) +``` + +```{caution} +Please direct inquiries to the **contributors/code maintainers of that code**. Note that the software(s) are provided "as is", without warranty of any kind, express or implied. +The DeepLabCut team is not responsible for the maintenance of these packages, and cannot guarantee that they will work with present & future versions of DeepLabCut. +``` + +## Example scripts for scaling up your DLC analysis & training + +These two scripts illustrate how to train, test, and analyze videos for multiple projects automatically (`scale_training_and_evaluation.py`) and how to analyze videos that are organized in subfolders automatically (`scale_analysis_oversubfolders.py`). Feel free to adjust them for your needs! + +- [Code: `scale_analysis_oversubfolders.py`](https://github.com/DeepLabCut/DLCutils/tree/master/SCALE_YOUR_ANALYSIS/scale_analysis_oversubfolders.py) +- [Code: `scale_training_and_evaluation.py`](https://github.com/DeepLabCut/DLCutils/blob/master/SCALE_YOUR_ANALYSIS/scale_training_and_evaluation.py) + +Contributed by [Alexander Mathis](https://github.com/AlexEMG) + +## Using DLC + XROMM_DLCTools + xmalab + +> Contributed by [@homfunc](https://github.com/homfunc) + +The DeepLabCut repository can also be used as a sibling checkout together with: + +- `../XROMM_DLCTools` +- `../xmalab` + +In that local layout, `XROMM_DLCTools` uses the optional `dlc` dependency group to import this checkout directly, and its baseline harness runs both a DeepLabCut smoke scenario and a broader end-to-end local workflow integration scenario. + +See {ref}`file:xamalab-dlc-integration` for the local integration notes and exact commands. + +## Using your DLC outputs, loading, simple ROI analysis, visualization examples + +### Time spent of a body part in a particular region of interest (ROI) + +You can compute time spent in particular ROIs in frames. This demo Jupyter Notebook shows you how to load the outputs of DLC and perform the analysis (plus other plotting functions): + +- [Code: `Demo_loadandanalyzeDLCdata.ipynb`](https://github.com/DeepLabCut/DLCutils/blob/master/Demo_loadandanalyzeDLCdata.ipynb) +- [Code: `time_in_each_roi.py`](https://github.com/DeepLabCut/DLCutils/blob/master/time_in_each_roi.py) + +Contributed by [Federico Claudi](https://github.com/FedeClaudi) and Jupyter Notebook from [Alexander Mathis](https://github.com/AlexEMG) + +### DeepLabCut-Display GUI + +Open and view data to understand pose estimation errors and trends. Filter data by likelihood threshold. + +- [Code: `DeepLabCut-Display`](https://github.com/jakeshirey/DeepLabCut-Display) + +Contributed by [Jacob Shirey](https://github.com/jakeshirey) + +### A GUI-based ROI tool for time spent of a body part in a defined region of interest + +- [Code: `DLC_ROI_tool`](https://github.com/PolarBean/DLC_ROI_tool) + +Contributed by [Harry Carey](https://github.com/PolarBean) + +### Linear transformation and scaling of DLC output data (`transform_and_scale`) + +This package is designed for anyone who wants to know where a tracked marker is within a reference frame (i.e. behavioral context). DeepLabCut outputs coordinates in relation to the field of view of the recorded video. With this tool, these coordinates can be linearly transformed and scaled to the reference frame of the behavioral context, meaning that the output coordinates are distances [cm] to a corner of the behavioral context, instead of distances [px] to a corner of the video field of view. + +- [Code: `transform_and_scale`](https://github.com/DeepLabCut/DLCutils/tree/master/transform_and_scale/) +- [Tutorial: `transform_and_scale_tutorial.ipynb`](https://github.com/DeepLabCut/DLCutils/tree/master/transform_and_scale/transform_and_scale_tutorial.ipynb) + +Contributed by [Michael Schellenberger](https://github.com/MSchellenberger) + +## Clustering tools (using the output of DLC) + +### Identifying Behavioral Structure from Deep Variational Embeddings of Animal Motion + +- [Paper](https://www.biorxiv.org/content/10.1101/2020.05.14.095430) +- [Code: `VAME`](https://github.com/LINCellularNeuroscience/VAME) + +### Behavior clustering with MotionMapper + +- Adapted from [MotionMapper](https://github.com/gordonberman/MotionMapper) +- [Code: `DLC_2_MotionMapper`](https://github.com/DeepLabCut/DLCutils/tree/master/DLC_2_MotionMapper) + +Contributed by [Mackenzie Mathis](https://github.com/MMathisLab) + +### Behavior clustering with B-SOiD + +B-SOiD is an open source unsupervised algorithm for discovery of spontaneous behaviors, and you can use the outputs of DLC to feed directly into B-SOiD in MATLAB. + +- [Paper](https://www.biorxiv.org/content/10.1101/770271v1.abstract) +- [Code: `B-SOiD`](https://github.com/YttriLab/B-SOiD) + +## Machine-learning helper packages (using the output of DLC) + +### Behavior analysis with machine learning in R (`ETH-DLCAnalyzer`) + +Deep learning based behavioral analysis enables high precision rodent tracking and is capable of outperforming commercial solutions. Oliver Sturman, Lukas von Ziegler, Christa Schläppi, Furkan Akyol, Benjamin Grewe, Johannes Bohacek + +- [Paper](https://www.biorxiv.org/content/10.1101/2020.01.21.913624v1) +- [Code: `DLCAnalyzer`](https://github.com/ETHZ-INS/DLCAnalyzer) + +### Behavior analysis with machine learning classifiers (SimBA) + +A pipeline for using pose estimation (i.e. DeepLabCut) then behavioral annotation and generation of supervised machine-learning-based classifiers. \<-- you can use the outputs of DLC to feed directly into SimBA (in Python). + +Code written by: [Simon Nilsson](https://github.com/sronilsson) (please direct use questions to Simon). + +- [Paper](https://www.biorxiv.org/content/10.1101/2020.04.19.049452v2) +- [Code: `simba`](https://github.com/sgoldenlab/simba) + +## 3D DeepLabCut helper packages + +### A wrapper package for DeepLabCut 2.0 for 3D videos (`anipose`) + +- [Code: `anipose`](https://github.com/lambdaloop/anipose) + +Maintainer: [Pierre Karashchuk](https://github.com/lambdaloop) + +### 3D reconstruction with EasyWand/Argus DLT system with DeepLabCut data + +Written by [Brandon Jackson](https://github.com/haliaetus13), post our DLC workshop in Jan 2020: + +A small set of utilities that allow conversion between the data storage formats of DeepLabCut (DLC) and one of the DLT-based 3D tracking systems: either Ty Hedrick's DigitizingTools in MATLAB, or the Python-based Argus. These functions should allow you to use data previously digitized in a DLT system to create the files needed to train a DLC model, and to import DLC-tracked points back into a DLT 3D calibration to reconstruct 3D points. + +- [Code: `DLCconverterDLT`](https://github.com/haliaetus13/DLCconverterDLT) + +### Pupil Tracking + +- From Tom Vaissie - tvaissie@scripps.edu +- Please see the [README.txt file](https://github.com/DeepLabCut/DLCutils/tree/master/pupilTracking) for details; this code makes the video in case study 7 [http://www.mousemotorlab.org/deeplabcut/](http://www.mousemotorlab.org/deeplabcut/). + +### Using DeepLabCut for USB-CGPIO feedback + +- [Paper](https://www.biorxiv.org/content/early/2018/11/28/482349) +- [Code: `DeepCutRealTime`](https://github.com/bf777/DeepCutRealTime) + +Maintainer: [Brandon Forys](https://github.com/bf777) + +## Legacy utility functions (no longer required in DLC 2+) + +```{warning} +These utilities are marked as legacy and are no longer required in DLC 2+. +``` + +### DLC 1 to DLC 2 conversion code + +This code allows you to import the labeled data from DLC 1 to DLC 2 projects. Note, it is not streamlined and should be used with care. + +- [Conversion scripts (`conversion_scripts_LEGACY`)](https://github.com/DeepLabCut/DLCutils/tree/master/conversion_scripts_LEGACY) + +Contributed by [Alexander Mathis](https://github.com/AlexEMG) + +### Running project created on Windows on Colaboratory + +```{note} +**UPDATE:** as of DeepLabCut 2.0.4 onwards you no longer need to use this code! You can simply create the training set on the cloud and it will automatically convert your project for you. +``` + +- This solves a path problem when creating a project and annotating data on Windows (see [issue #172](https://github.com/AlexEMG/DeepLabCut/issues/172)). This functionality will be included in a later version of DLC 2 (DONE!) +- [Conversion scripts (`conversion_scripts_LEGACY`)](https://github.com/DeepLabCut/DLCutils/tree/master/conversion_scripts_LEGACY) + +*Usage:* change in lines 70 and 71 of [`convertWin2Unix.py`](https://github.com/DeepLabCut/DLCutils/tree/master/conversion_scripts_LEGACY/convertWin2Unix.py) + +```python +basepath='/content/drive/My Drive/DeepLabCut/examples/' +projectname='Reaching-Mackenzie-2018-08-30' +``` + +then run this script on Colaboratory after uploading your labeled data to the drive. Thereby it will be converted to Unix format, then create a training set (with DeepLabCut) and proceed as usual... + +Contributed by [Alexander Mathis](https://github.com/AlexEMG) diff --git a/docs/docker.md b/docs/docker.md index dbca607353..c632a75602 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -1,69 +1,107 @@ --- deeplabcut: - last_content_updated: '2025-04-15' - last_metadata_updated: '2026-03-06' + last_content_updated: '2026-05-22' + last_metadata_updated: '2026-05-22' ignore: false + visibility: online + status: viable + last_verified: '2026-05-22' + verified_for: 3.0.0 --- + (docker-containers)= -# DeepLabCut Docker containers - -For DeepLabCut 2.2.0.2 and onwards, we provide container containers on [DockerHub]( -https://hub.docker.com/r/deeplabcut/deeplabcut). Using Docker is an alternative approach -to using DeepLabCut, which only requires the user to install [Docker]( -https://www.docker.com/) on your machine, vs. following the step-by-step installation -guide for a Anaconda setup. All dependencies needed to run DeepLabCut in the terminal or -running Jupyter notebooks with DeepLabCut pre-installed are shipped with the provided -Docker images. - -The [`napari-deeplabcut` labelling GUI]( -https://deeplabcut.github.io/DeepLabCut/docs/gui/napari_GUI.html) can be used to label -your data, but it cannot be run in a Docker container: it should be installed as -documented in the link above: `pip install napari-deeplabcut` (checkout the [workflow]( -https://deeplabcut.github.io/DeepLabCut/docs/gui/napari_GUI.html#workflow) as well!). + +# DeepLabCut in Docker + +From DeepLabCut 2.2.0.2 onward, we provide container images on [DockerHub](https://hub.docker.com/r/deeplabcut/deeplabcut). +Using Docker is an alternative approach to installing DeepLabCut in a local conda or pip environment: the images bundle all dependencies needed to run DeepLabCut in a reproducible, self-contained environment. +In a Docker container, DeepLabCut can be used from the terminal, or with Jupyter notebooks; the DeepLabCut GUI is not supported. +The approach requires a local installation of [Docker / Docker Desktop](https://www.docker.com/), and is meant for users who need strict reproducibility, an isolated environment, or server-based automation. + +```{important} +The napari-deeplabcut plugin **cannot be run in a Docker container**. +To label your data, please {ref}`install napari-deeplabcut ` in a local, non-dockerized environment, e.g. using pip: `pip install napari-deeplabcut` . +``` Advanced users can directly head to [DockerHub](https://hub.docker.com/r/deeplabcut/deeplabcut) and use the provided images there. To get started with using the images, we however also provide a helper tool, `deeplabcut-docker`, which makes the transition to docker images particularly convenient; to install the tool, run -``` bash +```bash $ pip install deeplabcut-docker ``` -on your machine (potentially in a virtual environment, or an existing Anaconda environment). -Note that this will *not* disprupt or install Tensorflow, or any other DeepLabCut dependencies on your computer---the Docker containers are completely isolated from your existing software installation! +on your machine (in any environment). `deeplabcut-docker` is just a lightweight package for setting up the Docker environment and it will *not* disrupt your existing software installation. The Docker container itself is completely isolated from your local environment! + +## Available images + +The following images are published to [DockerHub](https://hub.docker.com/r/deeplabcut/deeplabcut). All images come with Python 3.11 and CUDA pre-installed. + +| Tag | Description | +| ---------------------------------------------------- | -------------------------------------- | +| `deeplabcut/deeplabcut:latest` | Default runtime image for terminal use | +| `deeplabcut/deeplabcut:latest-jupyter` | Jupyter Notebook server | +| `deeplabcut/deeplabcut:-core-cuda` | Versioned runtime image | +| `deeplabcut/deeplabcut:-jupyter-cuda` | Versioned Jupyter image | + +By default `deeplabcut-docker` pulls the `latest` / `latest-jupyter` tag. To select a specific DeepLabCut or CUDA version, set the `DLC_VERSION` and `CUDA_VERSION` environment variables: + +```bash +DLC_VERSION=3.0.0 CUDA_VERSION=12.4 deeplabcut-docker bash --gpus all +``` + +To use a completely custom image instead of the default tags, pass `--image repo:tag`. Make sure the image supports Jupyter notebooks when using `deeplabcut-docker notebook`. ## Usage modes -With `deeplabcut-docker`, you can use the images in two modes. +With `deeplabcut-docker`, you can use the images in two modes: terminal mode and Jupyter Notebook mode. + + -- *Note 1: When running any of the following commands first, it can take some time to complete (a few minutes, depending on your internet connection), since it downloads the Docker image in the background. If you do not see any errors in your terminal, assume that everything is working fine! Subsequent runs of the command will be faster.* -- *Note 2: The labelling GUI cannot be used through the Docker images. However, you can install [`napari-deeplabcut`](https://github.com/DeepLabCut/napari-deeplabcut/tree/main?tab=readme-ov-file#napari-deeplabcut-keypoint-annotation-for-pose-estimation) in a conda environment to do the labelling!* -- *Note 3: For any mode below, you might want to set which directory is the base, namely, so you can have read/write (or read-only access). Here is how to do so: -If you want to mount the whole directory could e.g., pass* +```{note} +When running any of the following commands first, it can take some time to complete (a few minutes, depending on your internet connection), since it downloads the Docker image in the background. If you do not see any errors in your terminal, assume that everything is working fine! Subsequent runs of the command will be faster. +``` -`deeplabcut-docker bash -v /home/mackenzie/DEEPLABCUT:/home/mackenzie/DEEPLABCUT` +````{admonition} Choosing which directory to mount in the container +--- +class: tip dropdown +--- +For any mode below, you might want to set which directory is the base, so you can +have read/write or read-only access. -(which will mount the full directory into the container in read/write mode) +If you want to mount the whole directory, you could e.g. pass: -If read-only access is enough, `deeplabcut-docker bash -v /home/mackenzie/DEEPLABCUT:/home/mackenzie/DEEPLABCUT:ro` +```bash +deeplabcut-docker bash -v /home/mackenzie/DEEPLABCUT:/home/mackenzie/DEEPLABCUT +``` +This will mount the full directory into the container in read/write mode. + +If read-only access is enough: + +```bash +deeplabcut-docker bash -v /home/mackenzie/DEEPLABCUT:/home/mackenzie/DEEPLABCUT:ro +``` +```` ### Terminal mode You can run the light version of DeepLabCut and open a terminal by running -``` bash +```bash $ deeplabcut-docker bash ``` -**Important:** if have GPUs on your machine and want to use them to train models, you +````{important} +If you have GPUs on your machine and want to use them to train models, you need to pass the `--gpus all` argument to `deeplabcut-docker`: -``` bash +```bash $ deeplabcut-docker bash --gpus all ``` +```` Inside the terminal, you can confirm that DeepLabCut is correctly installed by running and noting which version installs. -``` bash +```bash $ ipython >>> import deeplabcut ``` @@ -72,49 +110,90 @@ $ ipython You can run DeepLabCut by starting a jupyter notebook server. The corresponding image can be pulled and started by running -``` bash +```bash $ deeplabcut-docker notebook ``` which will start a Jupyter notebook server. Follow the terminal instructions to open the notebook, by entering `http://127.0.0.1:8888` in your favorite browser. When prompted for a password, use `deeplabcut`, which is the pre-set option in the container. -The DeepLabCut version in this container is equivalent to the one you install with `pip install deeplabcut[gui]`. This means that you can start the DeepLabCut GUI with the appropriate commands in your notebook! +The container comes with `deeplabcut[modelzoo,wandb]` pre-installed. Note that the DeepLabCut GUI is not available inside the container. + +```{danger} +The Jupyter image uses a fixed default access token (`deeplabcut`) that is publicly known. + +**Anyone who can reach port 8888 on your machine can execute arbitrary code in the container.** + +Do not expose port 8888 to the internet (e.g. via a cloud VM's firewall or a public `0.0.0.0` +binding without a reverse proxy). +For local use, bind the port to localhost only (e.g. `-p 127.0.0.1:8888:8888`) and use SSH +port forwarding to access the server remotely (see below). +To use a custom token, pass `-e NOTEBOOK_TOKEN=` to `docker run`. You can pass an empty string to disable token-authentication: `-e NOTEBOOK_TOKEN=`. +``` + +#### Jupyter Notebooks on remote servers + +Sometimes you want to run Jupyter Notebooks on a remote server and connect from your local +browser. This requires SSH port forwarding. For general guidance see +[this StackOverflow post](https://stackoverflow.com/a/69244262) or the +[Jupyter Notebook docs](https://jupyter-notebook.readthedocs.io/en/4.x/public_server.html). + +With `deeplabcut-docker` and `DLC_NOTEBOOK_PORT`, this is straightforward: + +```bash +# Example: remote port XXXX=8889, local port YYYY=8890 + +# 1. Connect to your server with port forwarding +ssh -L localhost:8890:localhost:8889 you@your-server + +# 2. On the remote server, launch the container +DLC_NOTEBOOK_PORT=8889 deeplabcut-docker notebook --gpus all + +# 3. Open http://127.0.0.1:8890 in your local browser +``` ### Advanced usage -Advanced users and developers can visit the [`/docker` subdirectory](https://github.com/DeepLabCut/DeepLabCut/tree/main/docker) in the DeepLabCut codebase on Github. We provide Dockerfiles for all images, along with build instructions there. +Advanced users and developers can visit the [`/docker` subdirectory](https://github.com/DeepLabCut/DeepLabCut/tree/main/docker) in the DeepLabCut codebase on GitHub. +It contains a single multi-stage Dockerfile covering all images, along with build instructions. ## Prerequisites (if you don't have Docker installed already) **(1)** Install Docker. See https://docs.docker.com/install/ & for Ubuntu: https://docs.docker.com/install/linux/docker-ce/ubuntu/ Test docker: - $ sudo docker run hello-world - - The output should be: ``Hello from Docker! This message shows that your installation appears to be working correctly.`` +``` +$ sudo docker run hello-world +``` -*if you get the error ``docker: Error response from daemon: Unknown runtime specified nvidia.`` just simply restart docker: +The output should be: `Hello from Docker! This message shows that your installation appears to be working correctly.` - $ sudo systemctl daemon-reload - $ sudo systemctl restart docker +\*if you get the error `docker: Error response from daemon: Unknown runtime specified nvidia.` just simply restart docker: +``` + $ sudo systemctl daemon-reload + $ sudo systemctl restart docker +``` **(2)** Add your user to the docker group (https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user) -Quick guide to create the docker group and add your user: +Quick guide to create the docker group and add your user: Create the docker group. - $ sudo groupadd docker +``` +$ sudo groupadd docker +``` + Add your user to the docker group. - $ sudo usermod -aG docker $USER +``` +$ sudo usermod -aG docker $USER +``` (perhaps restart your computer (best) or (at min) open a new terminal to make sure that you are added from now on) - ## Notes and troubleshooting We dropped GUI support in 2.3.5+ due to too many numerous issues supporting them. Also please note these are tested on unix systems. When running containers on Linux, in some systems it might be necessary to run `host +local:docker` before starting the image via `deeplabcut-docker`. -If you encounter errors while using the images, please open an issue in the DeepLabCut repo---especially the `deeplabcut-docker` is still in its alpha version, and we appreciate user feedback to make the tool robust to use across many operating systems! +If you encounter errors while using the images, please open an issue in the DeepLabCut repo. We appreciate user feedback to make the tool robust across many operating systems! diff --git a/docs/gui/PROJECT_GUI.md b/docs/gui/PROJECT_GUI.md index 479b6c01d9..ed53b4794d 100644 --- a/docs/gui/PROJECT_GUI.md +++ b/docs/gui/PROJECT_GUI.md @@ -3,59 +3,72 @@ deeplabcut: last_content_updated: '2025-02-28' last_metadata_updated: '2026-03-06' ignore: false + visibility: online + status: review_needed + recommendation: update + notes: 'While the content is generally accurate, repeating installation instructions is not ideal. I would suggest linking to the installation guide instead of re-suggesting commands but then still saying to read the install page... Also, the GUI is likely used by the majority of users, so I would even consider making this a full section in the TOC, and maybe even having one file per GUI tab, which would make tracking code/docs sync easier. Addendum: it seems the beginner guide section is more of a GUI step-by-step, as mentioned earlier in this comment. I would suggest merging/moving and adding links in the present doc, which would make it less of a video list and more of a proper GUI guide.' --- -(project-manager-gui)= -# Interactive Project Manager GUI - -As some users may be more comfortable working with an interactive interface, we wanted to provide an easy-entry point to the software. All the main functionality is available in an easy-to-deploy GUI interface. Thus, while the many advanced features are not fully available in this Project GUI, we hope this gets more users up-and-running quickly. -**Release notes:** As of DeepLabCut 2.1+ now provide a full front-end user experience for DeepLabCut, and as of 2.3+ we changed the GUI from wxPython to PySide6 with napari support. - -## Get Started: +(project-manager-gui)= -(1) Install DeepLabCut using the simple-install with Anaconda found [here!](how-to-install)*. -Now you have DeepLabCut installed, but if you want to update it, either follow the prompt in the GUI which will ask you to upgrade when a new version is available, or just go into your env (activate DEEPLABCUT) then run: +# Project Manager GUI -` pip install 'deeplabcut[gui,modelzoo]'` *but please see [full install guide](how-to-install)! +As some users may be more comfortable working with an interactive interface, we wanted to provide an easy entry point to the software. All the main functionality is available in an easy-to-use GUI interface. +While several advanced features are not fully available in this Project GUI, we hope this gets more users up-and-running quickly. +## Getting started -(2) Open the terminal and run: `python -m deeplabcut` +1. Install DeepLabCut following the instructions in the {ref}`installation page`. +1. Open the terminal and run: `python -m deeplabcut` +```{important} +If you use Windows, please always open the terminal with administrator privileges! Right click, and "run as administrator". +```

-Start at the Project Management Tab and work your way through the tabs to built your customized model and deploy it on new data. +Start at the Project Management Tab and work your way through the tabs to build your customized model and deploy it on new data. We recommend to keep the terminal visible (as well as the GUI) so you can see the ongoing processes as you step through your project, or any errors that might arise. -- For specific napari-based labeling features, see the ["napari gui" docs](file:napari-gui-landing). +- For specific napari-based labeling features, see the {ref}`napari gui` section. - To change from dark to light mode, set appearance at the top: +

- +

-## Video Demos: How to launch and run the Project Manager GUI: +## User guide + +```{important} +See the dedicated {ref}`file:beginners-guide` section for a step-by-step walkthrough of the GUI. +``` + +## Video demos + +### How to launch and run the Project Manager GUI +```{tip} **Click on the images!** +``` Note that currently the video demo is the wxPython version, but the logic is the same! [![Watch the video](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1572824438905-QY9XQKZ8LAJZG6BLPWOQ/ke17ZwdGBToddI8pDm48kIIa76w436aRzIF_cdFnEbEUqsxRUqqbr1mOJYKfIPR7LoDQ9mXPOjoJoqy81S2I8N_N4V1vUb5AoIIIbLZhVYxCRW4BPu10St3TBAUQYVKcLthF_aOEGVRewCT7qiippiAuU5PSJ9SSYal26FEts0MmqyMIhpMOn8vJAUvOV4MI/guilaunch.jpg?format=1000w)](https://youtu.be/KcXogR-p5Ak) -### Using the Project Manager GUI with the latest DLC code (single animals, plus objects): ⬇️ +### Using the Project Manager GUI with the latest DLC code (single animals, plus objects) [![Watch the video](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1589046800303-OV1CCNZINWDMF1PZWCWE/ke17ZwdGBToddI8pDm48kB4PVlRPKDmSlQNbUD3wvXgUqsxRUqqbr1mOJYKfIPR7LoDQ9mXPOjoJoqy81S2I8N_N4V1vUb5AoIIIbLZhVYxCRW4BPu10St3TBAUQYVKcaja1QZ1SznGf7WzFOi-J6zLusnaF2VdeZcKivwxvFiDfGDqVYuwbAlftad9hfoui/dlc_gui_22.png?format=1000w)](https://www.youtube.com/watch?v=JDsa8R5J0nQ) -[Read more here](important-info-regd-usage) +{ref}`Read more here ` -### Using the Project Manager GUI with the latest DLC code (multiple identical-looking animals, plus objects): +### Using the Project Manager GUI with the latest DLC code (multiple identical-looking animals, plus objects) [![Watch the video](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1589047147498-G1KTFA5BXR4PVHOOR7OG/ke17ZwdGBToddI8pDm48kJDij24pM2COisBTLIGjR1pZw-zPPgdn4jUwVcJE1ZvWQUxwkmyExglNqGp0IvTJZamWLI2zvYWH8K3-s_4yszcp2ryTI0HqTOaaUohrI8PIel60EThn7SDFlTiSprUhmjQQHn9bhdY9dnQSKs8bCCo/Untitled.png?format=1000w)](https://www.youtube.com/watch?v=Kp-stcTm77g) -[Read more here](important-info-regd-usage) +{ref}`Read more here ` -## VIDEO DEMO: How to benchmark your data with the new networks and data augmentation pipelines: +### How to benchmark your data with the new networks and data augmentation pipelines [Watch the video](https://youtu.be/WXCVr6xAcCA) diff --git a/docs/gui/index.md b/docs/gui/index.md new file mode 100644 index 0000000000..7219999f91 --- /dev/null +++ b/docs/gui/index.md @@ -0,0 +1,10 @@ +# GUIde + +```{toctree} +--- +maxdepth: 2 +caption: GUI Workflows & Beginner Guides +--- + docs/gui/PROJECT_GUI + docs/gui/napari_GUI +``` diff --git a/docs/gui/napari/advanced_usage.md b/docs/gui/napari/advanced_usage.md index 20aaafd254..6e380c3257 100644 --- a/docs/gui/napari/advanced_usage.md +++ b/docs/gui/napari/advanced_usage.md @@ -9,7 +9,7 @@ deeplabcut: (file:napari-dlc-advanced-features)= -# napari-DLC - Advanced features +# Advanced features napari-DLC provides several additional features to enhance the annotation experience. @@ -25,11 +25,11 @@ This is the folder where annotations will be saved when using **File -> Save Sel ### Labeling progress -When a labeled data folder is loaded, the widget shows a percentage of labeled frames, based on the theoretical maximum number of keypoints (i.e. number of body parts x number of individuals x number of frames) that could be labeled. +When a labeled data folder is loaded, the widget shows a percentage of labeled frames, based on the theoretical maximum number of keypoints (i.e. number of bodyparts x number of individuals x number of frames) that could be labeled. ```{note} This can be a useful reference to track labeling progress.
-Since visibility cannot be accounted for, it should be considered an estimate of relative labeling progress rather than an absolute measure of completeness. (as not all videos would need 100% labeling, i.e. every body part on every individual in every frame). +Since visibility cannot be accounted for, it should be considered only a rough estimate of relative labeling progress rather than an absolute measure of completeness: hidden/occluded keypoints are not counted, therefore projects with occlusions will not have every body part on every individual in every frame. ``` ### Point size slider @@ -46,7 +46,7 @@ To copy-paste keypoints from one frame to another: ## Color scheme display features -The plugin shows a list of body parts and their corresponding colors in the dock widget. You can toggle the visibility of this color scheme using the **Show color scheme** button. +The plugin shows a list of bodyparts and their corresponding colors in the dock widget. You can toggle the visibility of this color scheme using the **Show color scheme** button. ```{tip} The display only shows keypoints that are currently visible in the viewer.
@@ -63,7 +63,11 @@ In individual coloring mode, the color scheme also shows the individuals list, a ### Jump to body part in viewer -If showing all body parts in the color scheme from the config, clicking on a keypoint in the list that is not currently visible in the viewer will jump to the first instance of that body part in the viewer and select it, if applicable. +To locate a bodypart label that is currently not visible in the viewer, enable "Show all bodyparts" in the color scheme list. +Then, click on a bodypart entry in the color scheme list. +The viewer will jump to the first instance of that body part and select it (when it exists). +If the bodypart is already visible in the viewer, clicking on it in the color scheme will simply select all keypoints of that bodypart, as described above. + This helps quickly find a specific body part in the viewer. ## Trajectory plot diff --git a/docs/gui/napari/basic_usage.md b/docs/gui/napari/basic_usage.md index b6f2cb6f3f..1bddf910cf 100644 --- a/docs/gui/napari/basic_usage.md +++ b/docs/gui/napari/basic_usage.md @@ -8,7 +8,8 @@ deeplabcut: --- (file:napari-dlc-basic-usage)= -# napari-DLC - Basic usage + +# Basic usage `napari-deeplabcut` is a napari plugin for keypoint annotation and label refinement. It can be used either as part of the DeepLabCut GUI or as a standalone annotation tool. @@ -56,6 +57,14 @@ You can load files either by: If you drag and drop a compatible labeled-data folder, the widget opens automatically. ``` +## Using napari + +```{important} +To familiarize yourself with napari, we recommend checking out the [official napari documentation and tutorials](https://napari.org/stable/usage.html). +``` + +(sec:napari-dlc-basic-workflow)= + ## Recommended basic labeling workflow The simplest way to **start labeling** is: @@ -164,7 +173,7 @@ Keeping data inside the project directory is recommended for best compatibility. - napari-deeplabcut specific: - `M`: cycle through annotation modes - `E`: toggle edge coloring - - `F`: toggle between individual and body-part coloring modes + - `F`: toggle between individual and bodypart coloring modes - `V`: toggle visibility of the selected layer - `Backspace`: delete selected point(s) - `Ctrl+C` / `Ctrl+V`: copy and paste selected points @@ -175,7 +184,7 @@ Use the **View shortcuts** button in the dock widget for a quick reference of na ### More quality-of-life features -See the {ref}`Advanced features ` for useful features such as copy-pasting annotations, quick body part selection, and more. +See the {ref}`Advanced features ` for useful features such as copy-pasting annotations, quick bodypart selection, and more. ## Labeling workflows @@ -203,9 +212,9 @@ Use this when the folder already contains a `CollectedData_.h5` file - Open (or drag and drop) the folder in napari. Existing annotations and keypoint metadata will be loaded automatically from the H5 file. -In this case, loading `config.yaml` is usually **not needed** unless : +In this case, loading `config.yaml` is usually **not needed** unless: -- The project's body parts have changed or +- The project's bodyparts have changed or - You want to refresh the configured color scheme ### Refining machine labels @@ -270,8 +279,9 @@ This helps keep saving behavior unambiguous. A short demo video is available here: +[Link to video](https://youtu.be/hsA9IB5r73E) + ```{warning} This demo may be outdated, but the general annotation workflow remains the same. If you would like an updated video tutorial, please open a feature request issue on GitHub, and we will update it. -[Link to video](https://youtu.be/hsA9IB5r73E) ``` diff --git a/docs/gui/napari/tracking/basic_usage.md b/docs/gui/napari/tracking/basic_usage.md new file mode 100644 index 0000000000..6580f1942c --- /dev/null +++ b/docs/gui/napari/tracking/basic_usage.md @@ -0,0 +1,353 @@ +--- +deeplabcut: + last_metadata_updated: '2026-05-08' + last_verified: '2026-05-08' + verified_for: 3.0.0rc14 + ignore: false + last_content_updated: '2026-05-08' +--- + +# Automated annotation with point tracking + +```{seealso} +For basic usage of the annotation plugin, see {ref}`file:napari-dlc-basic-usage` for the recommended workflow. +``` + +```{note} +The plugin relies on third-party open-source tracking models. +Please see {ref}`sec:napari-tracking-models-attribution` at the end of this page for information about the tracking models used in the plugin and their citation information. +``` + +## Overview + +The **Tracking Controls** widget is designed to help automate DeepLabCut annotation workflows: + +1. Manually annotate a small set of keypoints on a *reference frame*. +1. Use a point tracking model to propagate those keypoints forward and/or backward in time. +1. Inspect, refine, delete, and merge tracked results before exporting them back to DeepLabCut. + +> **Tracking is intended to accelerate annotation, and cannot replace manual review.** + +## Requirements + +```{tip} +We recommend **having a GPU available for tracking**, as it can be computationally intensive and slow on CPU. +Expect longer processing times on CPU, especially for longer videos or larger tracking ranges. +``` + +### In napari + +```{important} +Before using tracking, you must: + +- Load a **video** or **extracted frames** as an `Image` layer with time as the first dimension. + - For DLC-integrated workflows, the **easiest starting point is often to drag-and-drop one of the `labeled-data` folders from your DLC project**. + - See {ref}`sec:napari-dlc-basic-workflow` for more details on how to prepare your data and annotations before tracking. +- Ensure you have a **Points** layer containing DeepLabCut-style keypoints. + - If annotating from scratch, drag-and-drop the `config.yaml` file from your DLC project to create a new Points layer with the correct metadata. + - If loading a folder which already contains a `CollectedData_*.h5` file, the plugin will automatically create a Points layer with the existing annotations. +- Annotate at least one frame with valid keypoints. +- Tracking is most useful on temporally continuous image sequences or videos. + +See the workflow guides below for more details of the tracking process. +``` + +### In your Python environment + +**Skip this if you have already installed PyTorch or DeepLabCut** + +```{important} +**By default, installing the `[tracking]` extra alone will not enable GPU support.** +Check the [official PyTorch installation guide](https://pytorch.org/get-started/locally/) for GPU support and installation instructions for your system. +``` + +If you do not have PyTorch installed, or if you are using the plugin without the DeepLabCut package installed, install with: + +```bash +pip install napari-deeplabcut[tracking] +``` + +## User interface + + + +```{figure} ../../../images/napari/tracking/controls.png +--- +name: tracking-controls +caption: Tracking Controls widget with annotated keypoints and tracking results. +--- +Tracking Controls widget with annotated keypoints and tracking results. +``` + +### Showing the widget + +Use: + +> Plugins -> napari-deeplabcut -> Tracking controls + +### 1. Model selection + +| Control | Description | +| --------------- | ------------------------------------------------------- | +| **Tracker** | Selects the tracking backend from `AVAILABLE_TRACKERS`. | +| **Info button** | Hover to see tracker-specific details. | + + + +```{note} +Available models may depend on your installation and optional dependencies. +``` + +### 2. Layer selection + +| Control | Description | +| ------------- | -------------------------------------------------------- | +| **Keypoints** | Points layer containing manually annotated DLC keypoints | +| **Video** | Image layer containing the video to track | + +The widget automatically updates based on layer changes. + +### 3. Reference frame selection + +- The **Current** spinbox always reflects the viewer's current time index. +- This frame is used as the **query frame** for tracking. + - The model generates tracking predictions from the keypoints present on this frame and uses them as seeds to track forward and/or backward in time. + +```{note} +Only keypoints present on the selected reference frame are used to initialize a tracking run. +Neighboring frames or frames later in the video are never considered for initialization, even if they contain keypoints. +``` + +### 4. Frame range controls + +Tracking range can be specified **relative** or **absolute** to the reference frame. + +#### Backward (left) + +- Slider: relative negative offset +- `<< Abs`: absolute frame index +- `<< Rel`: relative frame offset + +#### Forward (right) + +- Slider: relative positive offset +- `Abs >>`: absolute frame index +- `Rel >>`: relative frame offset + +Changing the current frame updates the valid forward/backward range automatically. + +### 5. Tracking actions + +| Button | Action | +| ------ | ----------------------------- | +| ◀ | Track backward | +| ◀◀ | Track backward to first frame | +| ▶ | Track forward | +| ▶▶ | Track forward to last frame | +| ⟳ | Track both directions | +| ■ | Stop tracking | + +```{note} +Tracking runs in the background. You can continue navigating the viewer and editing layers while it runs; results will appear as a new layer once tracking is complete. +``` + +## Keyboard shortcuts + +Most tracking functions have keyboard shortcuts for easier usage. + +```{tip} +You can see shortcuts and their status using: +> Help -> Show napari-dlc shortcuts + +This is only available if the Keypoint controls widget has been opened at least once. +``` + +## Tracking results + +```{tip} +**Being able to tell which results originate from which layer is very important for effectively using the plugin.** +- Layers can be toggled (visible/invisible) with `V` by default or by clicking the eye icon next to the layer name in the layer list. +- Grid mode (toggled with `Ctrl+G` by default) can also help visually separate different layers and their results. +``` + +Each tracking run creates a **new Points layer**: + +- Named automatically (`[Tracking v] Ref. layer name - t - Tracker name`) + - `XX` refers to the iteration number (if multiple tracking runs are performed from the same reference layer and model) + - `T` refers to the reference frame index used to generate the tracking result +- **Visually distinct from manual annotations**: + - Cross symbol + - Slight transparency + - Green border + +```{note} +The original annotation layer is never modified by tracking. +To incorporate tracking results into your annotation data, use the merge workflow described below. +``` + +```{important} +If you run into accessibility issues with the default visualization style, please [open an issue](https://github.com/DeepLabCut/napari-deeplabcut/issues). +We would be happy to expand settings and provide more customization options if requested. +``` + +## Refinement and saving tools + +```{danger} +There is **currently no undo option**. Any **deletion or merging action you perform on layers is irreversible**, so we recommend keeping track of your layers and using visibility toggles to compare before and after merge results. + +Overwrite warnings will be shown where relevant. +``` + +### Deleting tracked points in future frames + +**Tracking results are often satisfactory for a certain number of frames, then start to drift or produce errors.** +For example, a tracked point may start following the background instead of the intended body part, or jump to a different body part or individual. +Because of this sometimes unavoidable drift, we provide a way to delete future tracked points while keeping the current frame intact. + +1. Select a tracking result Points layer. + - This action is always disabled for the original annotation layer. +1. Select one or more points on the **current frame**. +1. Click **Delete selected points in future frames**. + +Only *exact identity matches* in future frames are removed. + +```{important} +Points on the current frame are preserved so you can correct them and re-run tracking. +``` + +This allows you to run tracking, and iteratively progress through the frames, correcting keypoints as you go, and merging the final results back into the original annotation layer when you are satisfied, see below for more details on merging. + +### Merging tracked points + +The **Merge tracked points** workflow allows you to: + +- Combine multiple tracking passes +- Decide how to handle overlaps or conflicts +- Produce a clean final annotation layer + +This is especially useful when tracking was run from multiple reference frames. +There are several merge options available to help you achieve the desired result: + +- **Fill missing only**: Existing keypoints are always preserved. Missing keypoints in frames are filled with tracked results. + - Intended for merging final tracking results into the original annotation layer. +- **Overwrite existing target points**: Tracked keypoints overwrite existing ones in the target layer. + - Intended for replacing poor tracking results with a new, updated tracking pass. + +```{important} +Tracking result layers are intermediate working layers. +To save results back into the DeepLabCut project, first merge tracked points into a standard DLC annotation layer, then save that final annotation layer. +Tracking layers will be saved as CSVs, which are not compatible with DLC project annotations and will not be written back to the `CollectedData_*.h5` workflow. +``` + +## Workflow example + +### Loading and annotating from scratch + +1. Create a DeepLabCut project and add the videos to label. +1. Extract frames from the videos. + - Currently implemented trackers prefer continuous video frames. We recommend avoiding large gaps in frames ("jumpy" video), which can make tracking more difficult. + - For this reason, you may want to run tracking on the original video, then extract frames with tracking/refined annotations directly. +1. Go to the `labeled-data` folder, then drag-and-drop a folder with extracted frames into napari. + - This creates an Image layer with the frames. +1. Drag-and-drop the `config.yaml` file from your DLC project into napari. + - This creates an empty Points layer with the correct DLC metadata, ready for annotation. +1. Annotate keypoints on a reference frame. + +> Go to {ref}`sec:tracking-workflow-guides`. + +### Loading and annotating from existing DLC annotations + +1. Go to the `labeled-data` folder, then drag-and-drop a subfolder with extracted frames into napari. + - This creates an Image layer with the frames. + - Existing annotations from the `CollectedData_*.h5` file are loaded as a Points layer. +1. Inspect existing annotations, select a reference frame, and refine keypoints if needed. + +> Go to {ref}`sec:tracking-workflow-guides`. + +(sec:tracking-workflow-guides)= + +### Tracking + +1. Open the Tracking Controls widget (`Plugins -> napari-deeplabcut -> Tracking controls`). +1. Go to the desired reference frame, with annotated keypoints visible. +1. Select the forward/backward tracking range using the sliders and track forward/backward, or track to the beginning/end of the video using the seek buttons. +1. Inspect the tracking results. + - You can use **Show trajectories** in the Keypoint Controls dock widget to visualize the trajectories of tracked points across frames, which can help identify where tracking starts to drift. + - The plot is filtered by selected keypoints, so you can select a subset of points to inspect their trajectories more closely. +1. If there are problematic points: + 1. On the frame where tracking starts to drift, select the problematic point(s) and click **Delete selected points in future frames** to remove incorrect tracking results while preserving the tracked point(s) on the current frame. + 1. Refine the keypoint(s) on the current frame by correcting their position. + 1. Re-run tracking from that frame to propagate the correction forward or backward in time. +1. Merge the new tracking result back into the previous tracking layer when appropriate (for example, using **Overwrite existing target points**). +1. Repeat until satisfied with the tracking result, then merge into the original annotation layer using **Fill missing only** to preserve your original annotations and only add tracked keypoints in frames where you do not yet have manual annotations. +1. **Save the final DLC annotation layer** (usually the original annotation layer after merging). + - Tracking result layers are intermediate working layers and are not written back directly as DLC project annotations. + - **Saving the final merged annotation layer is the step that writes back to the DLC project folder and updates the `CollectedData_*.h5` workflow.** + +```{note} +The **Show trails** feature is currently not available for tracking result layers. Please [open an issue](https://github.com/DeepLabCut/napari-deeplabcut/issues) if this is something you would like to see in the future. +``` + +## Troubleshooting + +### No keypoints found on reference frame + +Ensure that: + +- **The correct Points layer is selected in the tracking controls dropdown menu.** +- You are on the intended frame. +- Points exist exactly on that frame index. + +### Tracking buttons do nothing + +Check that: + +- A video layer is selected. +- A keypoint layer is selected. +- Tracking is not already running. + +(sec:napari-tracking-models-attribution)= + +## Models information and citation info + +### CoTracker3 + +> CoTracker is a fast transformer-based model that can track any point in a video. It brings to tracking some of the benefits of OpticalFlow. + +- [Link to GitHub repository](https://github.com/facebookresearch/co-tracker) +- [Citation information](https://github.com/facebookresearch/co-tracker#citing-cotracker) + +```{admonition} Empirical observations +--- +class: tip +--- +This information is based on our own testing and experience with the model. +Please share any feedback or insights you have with us! + +- **Strengths:** fast on GPU, can output 10-100 frames of satisfactory tracking results, depending on difficulty. +- **Limitations:** strong preference for continuous video frames; struggles with large gaps in frame indices (for example, automated DLC frame extraction via clustering, or uniform extraction with a large step size). + Consider running tracking on the original video, then extracting frames with tracking/refined annotations directly. +``` + +## Limitations and future directions + +### Important considerations + +- As correcting labels can be time-consuming, annotating by hand may sometimes be faster than running tracking and heavily correcting its results. + - The benefits are mostly for long, continuous videos with many frames to annotate, where tracking can save time by propagating annotations across many frames at once. + - In very high-variability or very challenging videos, annotating by hand may still be more efficient than running tracking and correcting its results, especially if you only have a few frames to annotate. +- Manual curation is still essential for good tracking results, and the tracking models do not fully replace the need for manual annotation. +- In practice, a mix of hand-labeled hard frames and tracked easy frames should often works best. +- Be mindful of training set imbalance: if you flood your training set with easy frames that are well tracked, and only have a few hand-picked frames with rare or difficult poses, your model may not learn to generalize well to those challenging poses. + +#### Future features + +- We currently only provide CoTracker3 as a model. It is, however, relatively easy to add new models to the plugin via the registry; feel free to ask if you would like to contribute a model or see a specific model added. +- Generic napari saves or exports of tracking result layers are not part of the recommended DeepLabCut workflow. Tracking result layers are intermediate working layers; to preserve results in a DLC project-compatible way, merge them into a standard annotation layer and save that layer. +- If there is demand, we may add support for saving and loading tracking layers as separate files in the DLC project folder. +- If you have ideas for specific refinement tools, shortcuts, or other features that would be useful to add to the plugin, please share them with us. + +## Getting help and providing feedback + +- [GitHub issues](https://github.com/DeepLabCut/napari-deeplabcut/issues): for bug reports, feature requests, or general questions. We welcome your feedback and contributions. +- [Discussion forum](https://forum.image.sc/tag/deeplabcut): for general discussion, questions, and sharing your work with the community. We also provide troubleshooting help and guidance here, but may open an issue for actual bugs or feature requests directly on GitHub, as well as request more information there. diff --git a/docs/gui/napari_GUI.md b/docs/gui/napari_GUI.md index 0bafe5c84f..2fefc50e82 100644 --- a/docs/gui/napari_GUI.md +++ b/docs/gui/napari_GUI.md @@ -3,10 +3,16 @@ deeplabcut: last_content_updated: '2026-02-10' last_metadata_updated: '2026-04-09' ignore: false + visibility: online + status: outdated + recommendation: archive + notes: Being updated in a separate PR (#3280) last_verified: '2026-04-09' verified_for: 3.0.0rc14 --- + (file:napari-gui-landing)= + # napari GUI Welcome to the documentation for napari-DLC, the napari plugin for keypoint annotation and label refinement. This plugin can be used either as part of the DeepLabCut GUI or as a standalone annotation tool. diff --git a/docs/images/box1-multi-rec.png b/docs/images/box1-multi-rec.png new file mode 100644 index 0000000000..6b2cc2cb0c Binary files /dev/null and b/docs/images/box1-multi-rec.png differ diff --git a/docs/images/box2-single.png b/docs/images/box2-single.png new file mode 100644 index 0000000000..e680e64b7e Binary files /dev/null and b/docs/images/box2-single.png differ diff --git a/docs/images/dlc-workflow.png b/docs/images/dlc-workflow.png new file mode 100644 index 0000000000..a1f8091cd9 Binary files /dev/null and b/docs/images/dlc-workflow.png differ diff --git a/docs/images/napari/tracking/controls.png b/docs/images/napari/tracking/controls.png new file mode 100644 index 0000000000..682f8e59e6 Binary files /dev/null and b/docs/images/napari/tracking/controls.png differ diff --git a/docs/installation.md b/docs/installation.md index 333a1e72bc..4628990add 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,140 +1,215 @@ --- deeplabcut: last_content_updated: '2026-02-23' - last_metadata_updated: '2026-03-06' + last_metadata_updated: '2026-04-21' ignore: false + visibility: online + status: viable + recommendation: move + notes: Could be moved to a core/installation folder for clarity. + last_verified: '2026-04-21' + verified_for: 3.0.0rc14 --- -(how-to-install)= -# How To Install DeepLabCut -- **DeepLabCut can be run on Windows, Linux, or MacOS as long as you have Python 3.10 installed** - - (see also [technical considerations](tech-considerations-during-install) and if you run into issues also check out the [Installation Tips](https://deeplabcut.github.io/DeepLabCut/docs/recipes/installTips.html) page). -- 🚧 Please note, there are several modes of installation: - - please decide to either use a [**conda environment**](https://deeplabcut.github.io/DeepLabCut/docs/installation.html#conda-the-installation-process-is-as-easy-as-this-figure) based installation (**recommended**), - - or the supplied [**Docker container**](docker-containers) (recommended for Ubuntu advanced users). -- 🚀 Please note, you will get the best performance with using a **GPU**! - - Please see the section on [GPU support](https://deeplabcut.github.io/DeepLabCut/docs/installation.html#gpu-support) to install your GPU driver and CUDA. +(file:how-to-install)= -```{Hint} Familiar with python packages and conda? Quick Install Guide: +# Installing DeepLabCut + +- **DeepLabCut can be run on Windows, Linux, or MacOS as long as you have Python 3.10-3.12 installed** + - See also {ref}`technical considerations `. + + + + + +- 🚧 Please note, there are several possibilities for installation: + - **Recommended for most users**: Install in a {ref}`conda environment ` + - Install with **{ref}`uv `** (recommended for developers) + - In the supplied **{ref}`Docker container `** (recommended for Ubuntu advanced users and reproducibility). +- 🚀 You will get the best performance when using a **GPU**! + - Please see the section on {ref}`GPU support ` to install your GPU driver and CUDA. + +````{hint} +Familiar with python packages and conda? This assumes you have `conda`/`mamba` installed and this will install DeepLabCut in a fresh -environment. If you have an NVIDIA GPU, install PyTorch according to [their instructions -](https://pytorch.org/get-started/locally/) (with your desired CUDA version) - you just -need your GPU drivers installed. +environment. +If you have an NVIDIA GPU, install PyTorch according to [their instructions](https://pytorch.org/get-started/locally/) (with your desired CUDA version) - you just need your GPU drivers installed. ```bash conda create -n DEEPLABCUT python=3.12 conda activate DEEPLABCUT -# install PyTorch with your desired CUDA version (or for CPU only) - check [their -](https://pytorch.org/get-started/locally/) website: -# GPU version of pytorch for CUDA 11.3 -conda install pytorch cudatoolkit=11.3 -c pytorch - +# Install PyTorch with your desired CUDA version (or CPU only) +# Example: install GPU-enabled pytorch for CUDA 12.6 +pip install torch torchvision --index-url https://download.pytorch.org/whl/cu126 # install the latest version of DeepLabCut -pip install --pre deeplabcut +pip install deeplabcut # add --pre for pre-release versions! # or if you want to use the GUI -pip install --pre deeplabcut[gui] +pip install deeplabcut[gui] # ONLY IF YOU HAVE A CUDA GPU - check that PyTorch can access your GPU; this # should print `True` python -c "import torch; print(torch.cuda.is_available())" ``` +```` -- If you're familiar with the command line and want TensorFlow support, look [below]( -deeplabcut-with-tf-install) for a fresh installation that has worked for us (on Linux) -and makes it possible to use the GPU with both PyTorch and TensorFlow. +- If you're familiar with the command line and want TensorFlow support, look {ref}`below `. +(sec:installation-using-conda)= -## CONDA: The installation process is as easy as this figure! --> +## Using Conda - DLC +DLC -### 🚨 Before you start with our conda file, do you have a GPU? -````{admonition} 🚨 Click here for more information! -:class: dropdown -- We recommend having a GPU if possible! -- You **need to decide if you want to use a CPU or GPU for your models**: (Note, you can also use the CPU-only for project management and labeling the data! Then, for example, use Google Colaboratory GPUs for free (read more [here](https://github.com/DeepLabCut/DeepLabCut/tree/master/examples#demo-4-deeplabcut-training-and-analysis-on-google-colaboratory-with-googles-gpus) and there are a lot of helper videos on [our YouTube channel!](https://www.youtube.com/playlist?list=PLjpMSEOb9vRFwwgIkLLN1NmJxFprkO_zi)). +**The installation process is as easy as the figure on the right!↘️** - - **CPU?** Great, jump to the next section below! +### 🚨 Before you start... - - **NVIDIA GPU?** If you want to use your own GPU (i.e., a GPU is in your workstation), then you need to be sure you have a CUDA compatible GPU, CUDA, and cuDNN installed. Please note, which CUDA you install depends on what version of PyTorch you want to use. So, please check "GPU Support" below carefully. **Note, DeepLabCut is up to date with the latest CUDA and PyTorch!** +Do you have a GPU? If yes, see the {ref}`GPU support section ` below for installation instructions. - - **Apple M-chip GPU?** Be sure to install miniconda3, and your GPU will be used by default. -```` +If not, you can still install DeepLabCut and use it on your CPU, but it will be much slower for training and evaluation (but not for labeling or project management). -### Step 1: Install Python via Anaconda +`````{admonition} 🚨 Hardware information! +--- +class: dropdown +--- +- We recommend having a GPU if possible! +- You **need to decide if you want to use a CPU or GPU for your models** + + ````{tab-set} + ```{tab-item} CPU + Great, jump to the next section below! + ``` + ```{tab-item} NVIDIA GPU + If you want to use your own GPU (i.e., a GPU is in your workstation), then you need to be sure you have a CUDA compatible GPU, CUDA, and cuDNN installed. + Please note, which CUDA you install depends on what version of PyTorch you want to use. So, please check {ref}`sec:install-gpu-support` below carefully. **Note, DeepLabCut is up to date with the latest CUDA and PyTorch!** + ``` + ```{tab-item} Apple M-chip GPU + Install miniconda and use the standard `DEEPLABCUT.yaml` conda environment — PyTorch will use your Apple GPU via Metal automatically. For TensorFlow, add the `tf` extra after install (see {ref}`TensorFlow Support `). More tips are on the {ref}`installation tips ` page. + ``` + ```` + +- Note, you can also use the CPU-only install for project management and labeling the data! + Then, for example, use Google Colaboratory GPUs for free (read more [here](https://github.com/DeepLabCut/DeepLabCut/tree/master/examples#demo-4-deeplabcut-training-and-analysis-on-google-colaboratory-with-googles-gpus) and there are a lot of helper videos on [our YouTube channel!](https://www.youtube.com/playlist?list=PLjpMSEOb9vRFwwgIkLLN1NmJxFprkO_zi)). +````` + +### Step 1: Install miniconda + +```{important} +Download [miniconda](https://www.anaconda.com/docs/getting-started/miniconda/main) for your operating system +``` -### Install [anaconda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html#), or use miniconda3 for MacOS users (see below) +- miniconda is an easy way to install Python and additional packages across various operating systems +- With miniconda, you can install all the dependencies in an [environment](https://conda.io/docs/user-guide/tasks/manage-environments.html) on your machine +- Miniconda is a lightweight version of Anaconda that includes only conda and its dependencies. -- Anaconda is an easy way to install Python and additional packages across various operating systems. With Anaconda you create all the dependencies in an [environment](https://conda.io/docs/user-guide/tasks/manage-environments.html) on your machine. +```{admonition} Wait, why are we mixing Anaconda, miniconda and conda? +--- +class: dropdown tip +--- +`conda` is the terminal-based environment management system that is included in both Anaconda and Miniconda. This is the actual workhorse that allows you to create and manage environments, and install packages. + +**Anaconda** is a full-featured distribution that includes conda, Python, and a large number of scientific packages and their dependencies, plus some graphical user interfaces (GUIs) for managing environments and packages. It is a larger download and takes up more disk space. -```{Hint} -Download anaconda for your operating system: [anaconda.com/download/ -](https://www.anaconda.com/download/) +**Miniconda** is a minimal distribution that includes only conda and its dependencies, along with Python. It does not include any additional packages or GUIs. We recommend it as most GUIs and base packages provided by the full Anaconda distribution are not necessary for DeepLabCut. ``` -- IF you use a M1 or M2 chip in your MacBook with v12.5+ (typically 2020 or newer machines), we recommend **miniconda3,** which operates with the same principles as anaconda. This is straight forward and explained in detail here: https://docs.conda.io/projects/conda/en/latest/user-guide/install/macos.html. But in short, open the program "terminal" and copy/paste and run the code that is supplied below. +(sec:conda-build-env)= -### 💡 miniconda for Mac -````{admonition} Click the button to see code for miniconda for Mac -:class: dropdown -wget https://repo.anaconda.com/miniconda/Miniconda3-py310_4.12.0-MacOSX-arm64.sh -O ~/miniconda.sh -bash ~/miniconda.sh -b -p $HOME/miniconda -source ~/miniconda/bin/activate -conda init zsh -```` +### Step 2: Build a conda environment -### Step 2: Build an Env using our Conda file! +Use the `DEEPLABCUT.yaml` file to build a conda environment with all the dependencies for DeepLabCut. -You simply need to have this `.yaml` file anywhere locally on your computer. So, let's download it! +You simply need to have this `.yaml` file locally on your computer. -```{Hint} -Windows users: Be sure you have `git` installed along with anaconda: https://gitforwindows.org/ +```{warning} +On **Windows**, make sure you have `git` installed: [Git for Windows](https://gitforwindows.org/) ``` -- TO DIRECTLY DOWNLOAD THE CONDA FILE conda: +- Follow the link ➡️ for the [conda file](https://github.com/DeepLabCut/DeepLabCut/blob/main/conda-environments/DEEPLABCUT.yaml#:~:text=Raw%20file%20content-,Download,-%E2%8C%98) and then click "..." and select Download - - click ➡️ for [CONDA FILE](https://github.com/DeepLabCut/DeepLabCut/blob/main/conda-environments/DEEPLABCUT.yaml#:~:text=Raw%20file%20content-,Download,-%E2%8C%98) and then click the "..." and select Download - Screen Shot 2023-09-13 at 10 33 32 PM + Screen Shot 2023-09-13 at 10 33 32 PM -- **Now, in Terminal (or Anaconda Command Prompt for Windows users)**, if you clicked to download, go to your downloads folder. +- **Now, in Terminal (or Anaconda Command Prompt for Windows users)**: -```{Hint} -Windows users: Be sure to open the program terminal/cmd/anaconda prompt with a RIGHT-click, "open as admin" -``` + - If you clicked to download, go to your downloads folder. -```{Hint} -:class: dropdown -If you cloned the repo onto your Desktop, the command may look like: -``cd C:\Users\YourUserName\Desktop\DeepLabCut\conda-environments`` -You can (on Windows) hold SHIFT and right-click > Copy as path, or (on Mac) right-click and while in the menu press the OPTION key to reveal Copy as Pathname. -``` -Be sure you are in the folder that has the `.yaml` file, then run: + - Be sure you are in the folder that has the `.yaml` file, then run: -``conda env create -f DEEPLABCUT.yaml`` + `conda env create -f DEEPLABCUT.yaml` +- You can now use this environment from anywhere on your computer. + Just activate your environment by running: `conda activate DEEPLABCUT` -- You can now use this environment from anywhere on your computer (i.e., no need to go back into the conda- folder). Just enter your environment by running: - - Ubuntu/MacOS: ``source/conda activate nameoftheenv`` (i.e. on your Mac: ``conda activate DEEPLABCUT``) - - Windows: ``activate nameoftheenv`` (i.e. ``activate DEEPLABCUT``) +Now you should see (`DEEPLABCUT`) on the left of your terminal screen: -Now you should see (`nameofenv`) on the left of your terminal screen, i.e. ``(DEEPLABCUT) YourName-MacBook...`` -NOTE: no need to run pip install deeplabcut, as it is already installed!!! :) +``` +(DEEPLABCUT) YourName-MacBook... +``` + +```{note} +No need to run `pip install deeplabcut`, it's already in the conda file! +``` + +(sec:deeplabcut-with-tf-install)= -(deeplabcut-with-tf-install)= -### 💡 Notice: PyTorch and TensorFlow Support within DeepLabCut +#### TensorFlow support ````{admonition} DeepLabCut TensorFlow Support -:class: dropdown -As of June 2024 we have a PyTorch Engine backend and we will be depreciating the -TensorFlow backend by the end of 2024. Currently, if you want to use TensorFlow, you +--- +class: dropdown +--- +💡 **PyTorch and TensorFlow Support within DeepLabCut** + +As of June 2024 we have a PyTorch Engine backend and we will be deprecating the +TensorFlow backend by version 3.2 latest (TBD). +Currently, if you want to use TensorFlow, you need to run `pip install deeplabcut[tf]` in order to install the correct version of TensorFlow in your conda env. Please note, we will be providing bug fixes, but we will -not be supporting new TensorFlow versions beyond 2.10 (Windows), and 2.12 for other OS. +not be supporting new TensorFlow versions beyond version 2.18. + +Installing TensorFlow manually and getting it to have access to the GPU can be a bit tricky. +However, we try to simplify the installation procedure via optional dependencies. + +A specific note for **Windows users**: TensorFlow’s own docs state that **native Windows GPU** support +ended after **2.10**. We recommend Windows users to install [The Windows Subsystem for Linux (WSL)](https://learn.microsoft.com/en-us/windows/wsl/install) +if they want GPU support. + +**Installation via the `tf` optional dependencies** +We recommend installing DeepLabCut with TensorFlow by specifying one of the 'extra's': `tf`, `tf-cu11` or `tf-cu12`. E.g, + +``` +pip install deeplabcut[tf] +``` + +This table provides a more detailed summary on the available extras: + +| Extra | Version | Python | GPU backend | Role (summary) | +|--------------|-----------------------------|-------------|--------------------------|-----------------------------------------------------------------------------| +| tf | 2.12–2.18 (Python-dependent)| 3.10-3.12 | CUDA (Linux); Metal (macOS) | Default TensorFlow stack for most users. | +| tf-cu11 | 2.14 | 3.10 / 3.11 | CUDA 11.8 | Pinned TF for CUDA 11.x-era stack | +| tf-cu12 | 2.18 | 3.10-3.12 | CUDA 12.5 | Pinned TF for CUDA 12.x-era stack | +| tf-latest | 2.18+ | 3.10-3.12 | CUDA 12.5+ | (Not recommended!) Newest TensorFlow ≥ 2.18 | +| apple_mchips | 2.12 - 2.18 | 3.10-3.12 | macOS Metal | (Not recommended!) Legacy extra; installs `tensorflow` + `tensorflow-metal`. Prefer `tf` instead. | + + +Note that TensorFlow and PyTorch may try to install competing CUDA-toolkit dependencies. +This is addressed in the listed extras by capping the PyTorch version to match the CUDA requirements. +In case you experience problems with the above installation, you can try to let TensorFlow install their own CUDA-toolkit libraries. +Please run the following installation command (in Linux), replacing with your TensorFlow version (see table above). +Note that this may break PyTorch functionality. +``` +pip install "tensorflow[and-cuda]==" +``` + + +**Advanced manual setup (Linux):** +if you do **not** use `deeplabcut[tf]`, you must align the following dependencies yourself: +`tensorflow`, `tensorpack`, `tf-keras` / Keras, `tf-slim`, CUDA, the NVIDIA **driver**, +and **PyTorch** yourself. -Installing TensorFlow and getting it to have access to the GPU can be a bit tricky. Check TensorFlow's [compatibility matrix](https://www.tensorflow.org/install/source#gpu) to know which version of CUDA and cuDNN you should install. @@ -164,135 +239,178 @@ pip install --pre deeplabcut ``` ```` -**Great, that's it! DeepLabCut is installed!** 🎉💜 +### Step 3: Let's run DeepLabCut! +**DeepLabCut is installed!** 🎉💜 -### Step 3: Really, that's it! Let's run DeepLabCut +Launch the DeepLabCut GUI in your new conda env by running `python -m deeplabcut` Head over to the [User Guide Overview](https://deeplabcut.github.io/DeepLabCut/docs/UseOverviewGuide.html) for information. -🎉 Launch DeepLabCut in your new env by running `python -m deeplabcut` +```{warning} +On **Windows**: Open the terminal/cmd/anaconda prompt as **Administrator** (right click and select "Run as administrator") to avoid permission issues when downloading models, and for symlink support when videos are not copied into the project folder. +``` -## Other ways to install DeepLabCut and additional tips +### Conda environment management tips -### Alternatively, you can git clone this repo and install from source! -i.e., if the download did not work or you just want to have the source code handy! +Here are some conda environment management tips: [kapeli.com: Conda Cheat Sheet](https://kapeli.com/cheat_sheets/Conda.docset/Contents/Resources/Documents/index) -- **Windows/Linux/MacBooks:** git clone this repo (in the terminal/cmd program, while **in a folder** you wish to place DeepLabCut -To git clone type: ``git clone https://github.com/DeepLabCut/DeepLabCut.git``). Note, this can be anywhere, even downloads is fine.) -- Then follow the same steps as in Step 2 above, adjusting for the file now being in the downloaded folder. + -### PIP: + -- Everything you need to build custom models within DeepLabCut (i.e., use our source code and our dependencies) can be installed with `pip install 'deeplabcut[gui]'` (for GUI support w/PyTorch) or without the gui: `pip install 'deeplabcut'`. -- If you want to use the SuperAnimal models, then please use `pip install 'deeplabcut[gui,modelzoo]'`. +Please see how to test your installation by following [this video](https://www.youtube.com/watch?v=IOWtKn3l33s). -## DOCKER: + -- We also have docker containers. Docker is the most reproducible way to use and deploy code. Please see our dedicated docker package and page [here](https://deeplabcut.github.io/DeepLabCut/docs/docker.html). +## Other ways to install DeepLabCut -## Pro Tips: +### git clone -More [installation ProTips](installation-tips) are also available. +Recommended for users who want to modify the code, or want to be up-to-date with the latest code on GitHub. -If you ever want to update your DLC, just run `pip install --upgrade deeplabcut` once -you are inside your env. If you want to use a specific release, then you need to specify -the version you want, such as `pip install deeplabcut==3.0`. Once installed, you can -check the version by running `import deeplabcut` `deeplabcut.__version__`. Don't be -afraid to update, DLC is backwards compatible with your 2.0+ projects and performance -continues to get better and new features are added nearly monthly. +- To clone the repository run: `git clone https://github.com/DeepLabCut/DeepLabCut.git` +- Then follow the same steps as in Step 2 above, adjusting for the `DEEPLABCUT.yaml` env file now being in the folder where you git cloned the repo. +- Or use pip/uv to install from the cloned repo (see below). -**All of the data you labelled in version 2.X is also compatible with version 3+ and the -PyTorch engine**! There is no change in the workflow or the way labels are handled: the -big changes happen under-the-hood! If you've been working with DeepLabCut 2.X and want -to learn more about moving to the PyTorch engine, checkout our docs on [moving from -TensorFlow to PyTorch](dlc3-user-guide) +(sec:uv-install)= + +### `uv` (recommended for developers) + +- Clone the [repository](https://github.com/DeepLabCut/DeepLabCut) +- Install `uv` following [instructions here](https://docs.astral.sh/uv/getting-started/installation/) +- Run in the cloned repo: + +```bash +uv venv -p 3.12 +uv pip install -e '.[gui]' # Change optional installs as needed +source .venv/bin/activate # or & .venv\Scripts\activate.ps1 on Windows +``` -Here are some conda environment management tips: [kapeli.com: Conda Cheat Sheet]( -https://kapeli.com/cheat_sheets/Conda.docset/Contents/Resources/Documents/index) +- Add **`modelzoo`** for SuperAnimal models: `uv pip install -e '.[gui,modelzoo]'`. +- Add **`tf`** (or `tf-cu11` / `tf-cu12` as appropriate) for the TensorFlow training engine — see {ref}`TensorFlow Support `. -**Pro Tip:** If you want to modify code and then test it, you can use our provided -testscripts. This would mean you need to be up-to-date with the latest GitHub-based code -though! Please see [here](installation-tips) on how to get the latest GitHub code, and -how to test your installation by following this video: -https://www.youtube.com/watch?v=IOWtKn3l33s. +### `pip` -## Creating your own customized conda env (recommended route for Linux: Ubuntu, CentOS, Mint, etc.) +If you already have a local environment, everything you need to use the project manager GUI, train and/or build custom models within DeepLabCut (i.e., use our source code and our dependencies) can be installed with `pip install 'deeplabcut[gui]'` (for GUI support w/PyTorch) or without the gui: `pip install 'deeplabcut'`. + +- If you **cloned the repo** and want to make edits to the code locally, navigate to the cloned repo folder and run `pip install -e .[gui]` to install the package in "editable" mode, which allows you to make changes to the code and have those changes reflected when you import the package. +- If you want to use the SuperAnimal models, then please use `pip install 'deeplabcut[gui,modelzoo]'`. +- If you need the **TensorFlow** training engine, add the **`tf`** extra (or `tf-cu11` / `tf-cu12` as appropriate): `pip install 'deeplabcut[tf]'` — see {ref}`TensorFlow Support `. + +### Docker + +- We also have docker containers. Docker is the most reproducible way to use and deploy code. Please see our dedicated docker package and page [here](https://deeplabcut.github.io/DeepLabCut/docs/docker.html). -*Note in a fresh ubuntu install, you will often have to run: ``sudo apt-get install gcc python3-dev`` to install the GNU Compiler Collection and the python developing environment. +### Creating your own conda environment -Some users might want to create their own customize env. - Here is an example. + -In the terminal type: + + +```{tip} +In a fresh ubuntu install, you will often have to run: `sudo apt-get install gcc python3-dev` to install the GNU Compiler Collection and the python developing environment. +``` + +Create a new conda environment with Python 3.10 (or 3.11, 3.12) by running: `conda create -n DLC python=3.10` **Current version:** The only thing you then need to add to the env is deeplabcut ( -`pip install deeplabcut`) or `pip install 'deeplabcut[gui]'` which has a napari based -GUI. +`pip install deeplabcut`) or `pip install 'deeplabcut[gui]'` if you are using the GUI, which includes the napari based labeling +interface. +## Updating your installation -## **GPU Support:** +If you ever want to update your DLC, just run `pip install --upgrade deeplabcut` (alongside optional needed requirements, e.g. `[gui]`) using your environment. -The ONLY thing you need to do **first** if you have an NVIDIA GPU and the matching NVIDIA CUDA+driver installed. -- CUDA: https://developer.nvidia.com/cuda-downloads (just follow the prompts here!) -- DRIVERS: https://www.nvidia.com/Download/index.aspx +If you would like to use a specific release, then specify the version you want, such as `pip install deeplabcut==3.0` and optional requirements. + +Once installed, you can +check the version by running: + +```python +import deeplabcut +deeplabcut.__version__ +``` + +Don't be afraid to update, DLC is backwards compatible with your 2.0+ projects and performance continues to get better and new features are added often. + +### Data compatibility + +**All of the data you labelled in version 2.X is also compatible with version 3+ and the +PyTorch engine**! +There is no change in the workflow or the way labels are handled: the +big changes happen under-the-hood! If you've been working with DeepLabCut 2.X and want +to learn more about moving to the PyTorch engine, check out our docs on [moving from +TensorFlow to PyTorch](dlc3-user-guide) + +(sec:install-gpu-support)= + +## GPU Support + +### General GPU support + +Please ensure you have an NVIDIA GPU and the matching NVIDIA driver installed. + +```{warning} +If you have a GPU, you should first **install an appropriate driver for +your specific GPU**, then you can use the supplied conda file. +``` -### The most common "new user" hurdle is installing and using your GPU, so don't get discouraged! +- Drivers: see [NVIDIA Drivers](https://www.nvidia.com/Download/index.aspx) +- CUDA: download [here](https://developer.nvidia.com/cuda-downloads) if needed. Installing the drivers usually allows you to skip installing CUDA; instead obtaining via the PyTorch installation process. -**CRITICAL:** If you have a GPU, you should FIRST **install an appropriate driver for -your specific GPU**, then you can use the supplied conda file. You'll need an NVIDIA GPU -which is compatible with CUDA. To see a list of CUDA-enabled NVIDIA GPUs, please [see -their website](https://developer.nvidia.com/cuda-gpus). +### Installing CUDA and cuDNN for TensorFlow GPU support -- Here we provide notes on how to install and check your GPU use with TensorFlow (which -is used by DeepLabCut and already installed with the Anaconda files above). Thus, you do -not need to independently install tensorflow. +You will need an NVIDIA GPU that is compatible with CUDA. -**FIRST**, install a driver for your GPU. Find DRIVER HERE: -https://www.nvidia.com/download/index.aspx +To see a list of CUDA-enabled NVIDIA GPUs, please [see their website](https://developer.nvidia.com/cuda-gpus). -- Check which driver is installed by typing this into the terminal: ``nvidia-smi``. +Here we provide notes on how to install and check your GPU use with TensorFlow, which is used by DeepLabCut. -**SECOND**, install CUDA: https://developer.nvidia.com/ (Note that cuDNN, https://developer.nvidia.com/cudnn, is supplied inside the anaconda environment files, so you don't need to install it again). +1. Install a driver for your GPU, using the NVIDIA Drivers link above. + - Check which driver is installed by typing this into the terminal: `nvidia-smi`. +1. Install [CUDA](https://developer.nvidia.com/). Note that [cuDNN](https://developer.nvidia.com/cudnn) is supplied inside the anaconda environment files, so you don't need to install it again. +1. Follow the steps above to get the `DEEPLABCUT` conda file and install it! -**THIRD:** Follow the steps above to get the `DEEPLABCUT` conda file and install it! +### Notes -### Notes: +- **As of version 3.0+ the default engine is PyTorch.** TensorFlow remains optional via + `pip install "deeplabcut[tf]"` and related extras; **version ranges are defined in + `pyproject.toml`** (typically TensorFlow **2.12+** on supported Python versions). Upstream, **native + Windows GPU** for TensorFlow stopped after **2.10**. We advise Windows users to install [WSL](https://learn.microsoft.com/en-us/windows/wsl/install). We do not guarantee every future TensorFlow release for all platforms. -- **As of version 3.0+ we moved to PyTorch. The Last supported version of TensorFlow is -2.10 (window users) and 2.12 for others (we have not tested beyond this).** - Please be mindful different versions of TensorFlow require different CUDA versions. + - As the combination of TensorFlow and CUDA matters, we strongly encourage you to -**check your driver/cuDNN/CUDA/TensorFlow versions** [on this StackOverflow post]( -https://stackoverflow.com/questions/30820513/what-is-version-of-cuda-for-nvidia-304-125/30820690#30820690 -). + **check your driver/cuDNN/CUDA/TensorFlow versions** [on this StackOverflow post](https://stackoverflow.com/questions/30820513/what-is-version-of-cuda-for-nvidia-304-125/30820690#30820690). + - To check your GPU is working, in the terminal, run: -`nvcc -V` to check your installed version(s). + `nvcc -V` to check your installed version(s). - The best practice is to then run the supplied `testscript_pytorch_single_animal.py` -(or `testscript_tensorflow_single_animal.py` for the TensorFlow engine); this is inside the examples folder you -acquired when you git cloned the repo. Here is more information/a short -[video on running the testscript](https://www.youtube.com/watch?v=IOWtKn3l33s). -- Additionally, if you want to use the bleeding edge, with your git clone you also get -the latest code. While inside the main DeepLabCut folder, you can run `./reinstall.sh` -to be sure it's installed (more [here](installation-tips)) -- You can test that your GPU is being properly engaged with these additional [tips]( -https://www.tensorflow.org/programmers_guide/using_gpu). -- Ubuntu users might find this [installation guide]( -https://deeplabcut.github.io/DeepLabCut/docs/recipes/installTips.html#installation-on-ubuntu-20-04-lts -) for a fresh ubuntu install useful as well. - -## Troubleshooting: - -TensorFlow: + (or `testscript_tensorflow_single_animal.py` for the TensorFlow engine); this is inside the examples folder you + acquired when you git cloned the repo. Here is more information/a short + [video on running the test scripts](https://www.youtube.com/watch?v=IOWtKn3l33s). + +- You can test that your GPU is being properly used with these additional [tips](https://www.tensorflow.org/programmers_guide/using_gpu). + +- Ubuntu users might find this [installation guide](https://deeplabcut.github.io/DeepLabCut/docs/recipes/installTips.html#installation-on-ubuntu-20-04-lts) for a fresh DLC install on Ubuntu useful as well. + +## Troubleshooting + +### TensorFlow + Here are some additional resources users have found helpful (posted without endorsement): - https://stackoverflow.com/questions/30820513/what-is-the-correct-version-of-cuda-for-my-nvidia-driver/30820690

- +

- https://www.tensorflow.org/install/source#gpu @@ -301,38 +419,66 @@ Here are some additional resources users have found helpful (posted without endo - https://developer.nvidia.com/cuda-toolkit-archive - -FFMPEG: +### FFMPEG - A few Windows users report needing to install re-install ffmpeg (after windows updates) as described here: https://video.stackexchange.com/questions/20495/how-do-i-set-up-and-use-ffmpeg-in-windows (A potential error could occur when making new videos). On Ubuntu, the command is: `sudo apt install ffmpeg` -DEEPLABCUT: +### DeepLabCut + +- If you git clone or download this folder, and are inside of it then `import deeplabcut` will import the package from the local folder rather than from the latest on PyPi! + +(sec:system-wide-considerations-during-install)= + +## System-wide installation considerations + +```{note} +**What is a system-wide installation?** + +A system-wide installation, or a base environment installation, is when you install using the default Python environment/interpreter on your computer, instead of a compartmentalized, separate environment (e.g., a conda environment). + +This is often a source of conflicts between packages, user confusion and progressive "dependency hell" (where you have to keep installing and uninstalling packages to get the right versions for different applications). + +To avoid this, we recommend using a virtual environment (e.g., conda or uv managed environments) to keep your DeepLabCut installation separate from other Python packages and applications on your system. +``` + +If you perform a system-wide/base environment installation, and the computer has other Python packages or TensorFlow versions installed that conflict, this will overwrite them. + +If you have a dedicated machine for DeepLabCut, this may be *temporarily* fine, but will degrade over time as you try to install or update other packages. + +Indeed, if there are other applications that require different versions of libraries, then installing/updating anything would potentially break those applications. + +One way to manage virtual environments is to use conda environments (for which you need Anaconda/miniconda installed). +An environment is a self-contained directory that contains a Python installation for a particular version of Python, plus additional packages, without any cross-talk with other environments (NVIDIA drivers being a notable exception, as they are system-wide by nature). + +(sec:hardware-considerations-during-install)= + +## Hardware considerations + +- **Computer**: + + - For reference, we use e.g. Dell workstations (79xx series) with **Ubuntu 16.04 LTS, 18.04 LTS, 20.04 LTS, 22.04 LTS** and for versions prior to 2.2, we run a Docker container that has TensorFlow, etc. installed (https://github.com/DeepLabCut/Docker4DeepLabCut2.0). Now we use the new Docker containers supplied on this repo (linux support only), also available through [DockerHub](https://hub.docker.com/r/deeplabcut/deeplabcut) or the [`deeplabcut-docker`](https://pypi.org/project/deeplabcut-docker/) helper script. + +- **Computing Hardware**: -- if you git clone or download this folder, and are inside of it then ``import deeplabcut`` will import the package from there rather than from the latest on PyPi! + - An NVIDIA GPU with *at least* 8GB VRAM (memory) is ideal. + - A GPU is not strictly necessary, but on a CPU the (training and evaluation) code is considerably slower (10x) for ResNets, but MobileNets are faster. You might also consider using cloud computing services like [Google cloud/amazon web services](https://github.com/DeepLabCut/DeepLabCut/issues/47) or Google Colaboratory. -(system-wide-considerations-during-install)= -## System-wide considerations: +- **Camera Hardware**: -If you perform the system-wide installation, and the computer has other Python packages or TensorFlow versions installed that conflict, this will overwrite them. If you have a dedicated machine for DeepLabCut, this is fine. If there are other applications that require different versions of libraries, then one would potentially break those applications. The solution to this problem is to create a virtual environment, a self-contained directory that contains a Python installation for a particular version of Python, plus additional packages. One way to manage virtual environments is to use conda environments (for which you need Anaconda installed). + - The software is very robust to variations stemming from various cameras (cell phone cameras, grayscale, color; captured under infrared light, different manufacturers, etc.). See demos on our [website](https://www.mousemotorlab.org/deeplabcut/). + - Note that a model trained on certain data/camera may not generalize to data from a different camera however, so we recommend using the same camera for training and inference. -(tech-considerations-during-install)= -## Technical Considerations: +- **Software**: -- Computer: + - Operating System: Linux (Ubuntu), MacOS[^1] (Mojave), or Windows 10. However, we the authors strongly recommend Ubuntu! + - DeepLabCut is written in Python 3 (https://www.python.org/) and not compatible with Python 2. - - For reference, we use e.g. Dell workstations (79xx series) with **Ubuntu 16.04 LTS, 18.04 LTS, 20.04 LTS, 22.04 LTS** and for versions prior to 2.2, we run a Docker container that has TensorFlow, etc. installed (https://github.com/DeepLabCut/Docker4DeepLabCut2.0). Now we use the new Docker containers supplied on this repo (linux support only), also available through [DockerHub](https://hub.docker.com/r/deeplabcut/deeplabcut) or the [`deeplabcut-docker`](https://pypi.org/project/deeplabcut-docker/) helper script. + -- Computer Hardware: - - Ideally, you will use a strong NVIDIA GPU with *at least* 8GB memory. A GPU is not necessary, but on a CPU the (training and evaluation) code is considerably slower (10x) for ResNets, but MobileNets are faster (see WIKI). You might also consider using cloud computing services like [Google cloud/amazon web services](https://github.com/DeepLabCut/DeepLabCut/issues/47) or Google Colaboratory. + -- Software: - - Operating System: Linux (Ubuntu), MacOS* (Mojave), or Windows 10. However, the authors strongly recommend Ubuntu! *MacOS does not support NVIDIA GPUs (easily), so we only suggest this option for CPU use or a case where the user wants to label data, refine data, etc and then push the project to a cloud resource for GPU computing steps, or use MobileNets. - - Anaconda/Python3: Anaconda: a free and open source distribution of the Python programming language (download from https://www.anaconda.com/). DeepLabCut is written in Python 3 (https://www.python.org/) and not compatible with Python 2. - - `pip install deeplabcut` - - TensorFlow - - If you want to use a pre3.0 version, you will need [TensorFlow](https://www.tensorflow.org/) (we used version 1.0 in the Nature Neuroscience paper, later versions also work with the provided code (we tested **TensorFlow versions 1.0 to 1.15, and 2.0 to 2.10**; we recommend TF2.10 now) for Python 3.8, 3.9, 3.10 with GPU support. - - To note, is it possible to run DeepLabCut on your CPU, but it will be VERY slow (see: [Mathis & Warren](https://www.biorxiv.org/content/early/2018/10/30/457242)). However, this is the preferred path if you want to test DeepLabCut on your own computer/data before purchasing a GPU, with the added benefit of a straightforward installation! Otherwise, use our COLAB notebooks for GPU access for testing. - - Docker: We highly recommend advanced users use the supplied [Docker container](docker-containers) +[^1]: MacOS does not support NVIDIA GPUs (easily), so we only suggest this option for CPU use or a case where the user wants to label data, refine data, etc, and then push the project to a cloud resource for GPU computing steps, or use MobileNets diff --git a/docs/intro.md b/docs/intro.md index cab6f4b183..c731cba264 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -4,4 +4,5 @@ deeplabcut: last_metadata_updated: '2026-03-06' ignore: false --- + Please see the main [READ ME!](https://deeplabcut.github.io/DeepLabCut/README.html) diff --git a/docs/maDLC_UserGuide.md b/docs/maDLC_UserGuide.md index 1c4dcc5731..0ce799554f 100644 --- a/docs/maDLC_UserGuide.md +++ b/docs/maDLC_UserGuide.md @@ -3,26 +3,40 @@ deeplabcut: last_content_updated: '2026-02-10' last_metadata_updated: '2026-03-06' ignore: false + visibility: online + status: review_needed + recommendation: verify + notes: Could use a small formatting pass. Contents are 4-5y old in some places, recommend to review for accuracy. --- + (multi-animal-userguide)= -# DeepLabCut for Multi-Animal Projects + +# Multi-animal projects + +```{contents} +--- +local: +depth: 3 +--- +``` This document should serve as the user guide for maDLC, and it is here to support the scientific advances presented in [Lauer et al. 2022](https://doi.org/10.1038/s41592-022-01443-0). Note, we strongly encourage you to use the [Project Manager GUI](project-manager-gui) when you first start using multi-animal mode. Each tab is customized for multi-animal when you create or load a multi-animal project. As long as you follow the recommendations within the GUI, you should be good to go! -````{versionadded} 3.0.0 +```{versionadded} 3.0.0 PyTorch is now available as a deep learning engine for pose estimation models, along with new model architectures! For more information about moving from TensorFlow to PyTorch (if you're already familiar with DeepLabCut & the TensorFlow engine), check out [the PyTorch user guide](dlc3-user-guide). If you're just starting out with DeepLabCut, we suggest you use the PyTorch backend. -```` +``` ## How to think about using maDLC: You should think of maDLC being **four** parts. + - (1) Curate annotation data that allows you to learn a model to track the objects/animals of interest. - (2) Create a high-quality pose estimation model. - (3) Track in space and time, i.e., assemble bodyparts to detected objects/animals and link across time. This step performs assembly and tracking (comprising first local tracking and then tracklet stitching by global reasoning). @@ -30,37 +44,65 @@ You should think of maDLC being **four** parts. Thus, you should always label, train, and evaluate the pose estimation performance first. If and when that performance is high, then you should go advance to the tracking step (and video analysis). There is a natural break point for this, as you will see below. -maDLC +```{figure} https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1596370260800-SP2GWKDPJCOIR7LJ31VM/ke17ZwdGBToddI8pDm48kB4fL2ovSQh5dRlH2jCMtpoUqsxRUqqbr1mOJYKfIPR7LoDQ9mXPOjoJoqy81S2I8N_N4V1vUb5AoIIIbLZhVYxCRW4BPu10St3TBAUQYVKcSV94BuD0XUinmig_1P1RJNYVU597j3jgswapL4c_w92BJE9r6UgUperYhWQ2ubQ_/workflow.png?format=2500w +--- +name: fig-madlc-workflow +alt: Overview of the four-part multi-animal DeepLabCut workflow +width: 550px +align: center +--- +Overview of the multi-animal DeepLabCut workflow. +``` -## Install: +## Getting started -**Quick start:** If you are using DeepLabCut on the cloud, or otherwise cannot use the GUIs and you should install with: `pip install 'deeplabcut'`; if you need GUI support, please use: `pip install 'deeplabcut[gui]'`. Check the [installation page](how-to-install) for more information, including GPU support. +DeepLabCut offers two equivalent interfaces: a **GUI** for those who prefer a visual +workflow (no Python knowledge required), and a **Python API** for users who want +scripting flexibility or to integrate DeepLabCut into a larger pipeline. All workflow +steps are available in both. -IF you want to use the bleeding edge version to make edits to the code, see [here on how to install it and test it](https://deeplabcut.github.io/DeepLabCut/docs/recipes/installTips.html#how-to-use-the-latest-updates-directly-from-github). +We assume you have DeepLabCut installed (if not, see {ref}`file:how-to-install`). +Open a terminal and activate your conda environment: -## Get started in the terminal or Project GUI: +```bash +conda activate DEEPLABCUT +``` -**GUI:** simply launch your conda env, and type `python -m deeplabcut` in the terminal. -Then follow the tabs! It might be useful to read the following, however, so you understand what each command does. +```{important} +On Windows, always open the terminal with administrator privileges: right-click and +select "Run as administrator". +``` + +Choose your interface below to launch DeepLabCut: -**TERMINAL:** To begin, 🚨 (windows) navigate to anaconda prompt and right-click to "open as admin", or (unix/MacOS) simply launch "terminal" on your computer. We assume you have DeepLabCut installed (if not, [see installation instructions](how-to-install)). Next, launch your conda env (i.e., for example `conda activate DEEPLABCUT`). +### GUI (recommended for beginners) -```{Hint} -🚨 If you use Windows, please always open the terminal with administrator privileges! Right click, and "run as administrator". +```bash +python -m deeplabcut ``` - Please read more [here](https://deeplabcut.github.io/DeepLabCut/docs/docker.html), and in our Nature Protocols paper [here](https://www.nature.com/articles/s41596-019-0176-0). And, see our [troubleshooting wiki](https://github.com/DeepLabCut/DeepLabCut/wiki/Troubleshooting-Tips). -Open an ``ipython`` session and import the package by typing in the terminal: +Each workflow step has a corresponding tab in the Project Manager. It is useful to +read the sections below so you understand what each step does. + +### Python API + +In an interactive Python session (e.g. `ipython`), import DeepLabCut: + ```python -ipython import deeplabcut ``` -```{TIP} -for every function there is a associated help document that can be viewed by adding a **?** after the function name; i.e. ``deeplabcut.create_new_project?``. To exit this help screen, type ``:q``. -``` +## Workflow -### (A) Create a New Project +DeepLabCut's full multi-animal workflow is described in steps (A)–(L) below. +Every step can be completed either via the **GUI** or the **Python API** — both are +fully equivalent. Code examples throughout this page use the Python API; if you are +using the GUI, the same steps are available in the corresponding tabs of the Project +Manager. + +### Phase 1 — Project setup + +#### (A) Create a New Project ```python deeplabcut.create_new_project( @@ -72,17 +114,19 @@ deeplabcut.create_new_project( ) ``` -Tip: if you want to place the project folder somewhere specific, please also pass : ``working_directory = "FullPathOftheworkingDirectory"`` +Tip: if you want to place the project folder somewhere specific, please also pass : `working_directory = "FullPathOftheworkingDirectory"` + +- Note, if you are a linux/macOS user the path should look like: `["/home/username/yourFolder/video1.mp4"]`; if you are a Windows user, it should look like: `[r"C:\username\yourFolder\video1.mp4"]` +- Note, you can also put `config_path = ` in front of the above line to create the path to the config.yaml that is used in the next step, i.e. `config_path=deeplabcut.create_project(...)`) + - If you do not, we recommend setting a variable so this can be easily used! Once you run this step, the config_path is printed for you once you run this line, so set a variable for ease of use, i.e. something like: -- Note, if you are a linux/macOS user the path should look like: ``["/home/username/yourFolder/video1.mp4"]``; if you are a Windows user, it should look like: ``[r"C:\username\yourFolder\video1.mp4"]`` -- Note, you can also put ``config_path = `` in front of the above line to create the path to the config.yaml that is used in the next step, i.e. ``config_path=deeplabcut.create_project(...)``) - - If you do not, we recommend setting a variable so this can be easily used! Once you run this step, the config_path is printed for you once you run this line, so set a variable for ease of use, i.e. something like: ```python config_path = '/thefulloutputpath/config.yaml' ``` - - just be mindful of the formatting for Windows vs. Unix, see above. -This set of arguments will create a project directory with the name **Name of the project+name of the experimenter+date of creation of the project** in the **Working directory** and creates the symbolic links to videos in the **videos** directory. The project directory will have subdirectories: **dlc-models**, **dlc-models-pytorch**, **labeled-data**, **training-datasets**, and **videos**. All the outputs generated during the course of a project will be stored in one of these subdirectories, thus allowing each project to be curated in separation from other projects. The purpose of the subdirectories is as follows: +- just be mindful of the formatting for Windows vs. Unix, see above. + +This set of arguments will create a project directory with the name **Name of the project+name of the experimenter+date of creation of the project** in the **Working directory** and creates the symbolic links to videos in the **videos** directory. The project directory will have subdirectories: **dlc-models**, **dlc-models-pytorch**, **labeled-data**, **training-datasets**, and **videos**. All the outputs generated during the course of a project will be stored in one of these subdirectories, thus allowing each project to be curated in separation from other projects. The purpose of the subdirectories is as follows: **dlc-models** and **dlc-models-pytorch** have a similar structure: the first contains files for the TensorFlow engine while the second contains files for the PyTorch engine. @@ -99,9 +143,9 @@ saved checkpoint, in case the training was interrupted. **labeled-data:** This directory will store the frames used to create the training dataset. Frames from different videos are stored in separate subdirectories. Each frame has a filename related to the temporal index within the corresponding video, which allows the user to trace every frame back to its origin. -**training-datasets:** This directory will contain the training dataset used to train the network and metadata, which contains information about how the training dataset was created. +**training-datasets:** This directory will contain the training dataset used to train the network and metadata, which contains information about how the training dataset was created. -**videos:** Directory of video links or videos. When **copy\_videos** is set to ``False``, this directory contains symbolic links to the videos. If it is set to ``True`` then the videos will be copied to this directory. The default is ``False``. Additionally, if the user wants to add new videos to the project at any stage, the function **add\_new\_videos** can be used. This will update the list of videos in the project's configuration file. Note: you neither need to use this folder for videos, nor is it required for analyzing videos (they can be anywhere). +**videos:** Directory of video links or videos. When **copy_videos** is set to `False`, this directory contains symbolic links to the videos. If it is set to `True` then the videos will be copied to this directory. The default is `False`. Additionally, if the user wants to add new videos to the project at any stage, the function **add_new_videos** can be used. This will update the list of videos in the project's configuration file. Note: you neither need to use this folder for videos, nor is it required for analyzing videos (they can be anywhere). ```python deeplabcut.add_new_videos( @@ -111,24 +155,33 @@ deeplabcut.add_new_videos( ) ``` -*Please note, *Full path of the project configuration file* will be referenced as ``config_path`` throughout this protocol. +\*Please note, *Full path of the project configuration file* will be referenced as `config_path` throughout this protocol. You can also use annotated data from single-animal projects, by converting those files. There are docs for this: [convert single to multianimal annotation data](convert-maDLC) -![Box 1 - Multi Animal Project Configuration File Glossary](images/box1-multi.png) +```{figure} images/box1-multi.png +--- +name: pose-cfg-box1-multi +alt: Box 1 — multi-animal project configuration file parameter glossary +--- +**Box 1.** Multi-animal project `config.yaml` parameter glossary. +``` + +##### API Docs -### API Docs ````{admonition} Click the button to see API Docs -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.create_new_project.rst ``` ```` -### (B) Configure the Project +#### (B) Configure the Project -Next, open the **config.yaml** file, which was created during **create\_new\_project**. +Next, open the **config.yaml** file, which was created during **create_new_project**. You can edit this file in any text editor. Familiarize yourself with the meaning of the parameters (Box 1). You can edit various parameters, in particular you **must add the list of *individuals* and *bodyparts* (or points of interest)**. @@ -136,7 +189,7 @@ parameters (Box 1). You can edit various parameters, in particular you **must ad You can also set the *colormap* here that is used for all downstream steps (can also be edited at anytime), like labeling GUIs, videos, etc. Here any [matplotlib colormaps](https://matplotlib.org/tutorials/colors/colormaps.html) will do! -An easy way to programmatically edit the config file at any time is to use the function **edit\_config**, which takes the full path of the config file to edit and a dictionary of key–value pairs to overwrite. +An easy way to programmatically edit the config file at any time is to use the function **edit_config**, which takes the full path of the config file to edit and a dictionary of key–value pairs to overwrite. ```python import deeplabcut @@ -177,15 +230,29 @@ identity: True/False **Individuals:** are names of "individuals" in the annotation dataset. These should/can be generic (e.g. mouse1, mouse2, etc.). These individuals are comprised of the same bodyparts defined by `multianimalbodyparts`. For annotation in the GUI and training, it is important that all individuals in each frame are labeled. Thus, keep in mind that you need to set individuals to the maximum number in your labeled-data set, .i.e., if there is (even just one frame) with 17 animals then the list should be `- indv1` to `- indv17`. Note, once trained if you have a video with more or less animals, that is fine - you can have more or less animals during video analysis! -**Identity:** If you can tell the animals apart, i.e., one might have a collar, or a black marker on the tail of a mouse, then you should label these individuals consistently (i.e., always label the mouse with the black marker as "indv1", etc). If you have this scenario, please set `identity: True` in your `config.yaml` file. If you have 4 black mice, and you truly cannot tell them apart, then leave this as `false`. +**Identity:** If you can tell the animals apart, i.e., one might have a collar, or a black marker on the tail of a mouse, then you should label these individuals consistently (i.e., always label the mouse with the black marker as "indv1", etc). If you have this scenario, please set `identity: True` in your `config.yaml` file. If you have 4 black mice, and you truly cannot tell them apart, then leave this as `false`. **Multianimalbodyparts:** are the bodyparts of each individual (in the above list). **Uniquebodyparts:** are points that you want to track, but that appear only once within each frame, i.e. they are "unique". Typically these are things like unique objects, landmarks, tools, etc. They can also be animals, e.g. in the case where one German shepherd is attending to many sheep the sheep bodyparts would be multianimalbodyparts, the shepherd parts would be uniquebodyparts and the individuals would be the list of sheep (e.g. Polly, Molly, Dolly, ...). -### (C) Select Frames to Label - -**CRITICAL:** A good training dataset should consist of a sufficient number of frames that capture the breadth of the behavior. This ideally implies to select the frames from different (behavioral) sessions, different lighting and different animals, if those vary substantially (to train an invariant, robust feature detector). Thus for creating a robust network that you can reuse in the laboratory, a good training dataset should reflect the diversity of the behavior with respect to postures, luminance conditions, background conditions, animal identities, etc. of the data that will be analyzed. For the simple lab behaviors comprising mouse reaching, open-field behavior and fly behavior, 100−200 frames gave good results [Mathis et al, 2018](https://www.nature.com/articles/s41593-018-0209-y). However, depending on the required accuracy, the nature of behavior, the video quality (e.g. motion blur, bad lighting) and the context, more or less frames might be necessary to create a good network. Ultimately, in order to scale up the analysis to large collections of videos with perhaps unexpected conditions, one can also refine the data set in an adaptive way (see refinement below). **For maDLC, be sure you have labeled frames with closely interacting animals!** +### Phase 2 — Data preparation + +#### (C) Select Frames to Label + +```{important} +A good training dataset should consist of a sufficient number of frames that capture +the breadth of the behavior. Select frames from different behavioral sessions, different +lighting conditions, and different animals if those vary substantially (to train an +invariant, robust feature detector). The dataset should reflect the diversity of +postures, luminance conditions, background conditions, and animal identities in the data +to be analyzed. For simple lab behaviors such as mouse reaching, open-field behavior, +and fly behavior, 100–200 frames gave good results +([Mathis et al., 2018](https://www.nature.com/articles/s41593-018-0209-y)). However, +more or fewer frames may be needed depending on accuracy requirements, behavior +complexity, and video quality (e.g. motion blur, poor lighting). **For maDLC, make sure +you include labeled frames with closely interacting animals.** +``` The function `extract_frames` extracts frames from all the videos in the project configuration file in order to create a training dataset. The extracted frames from all the videos are stored in a separate subdirectory named after the video file’s name under the ‘labeled-data’. This function also has various parameters that might be useful based on the user’s need. @@ -199,9 +266,11 @@ deeplabcut.extract_frames( ) ``` -**CRITICAL POINT:** It is advisable to keep the frame size small, as large frames increase the training and -inference time, or you might not have a large enough GPU for this. -When running the function `extract_frames`, if the parameter crop=True, then you will be asked to draw a box within the GUI (and this is written to the config.yaml file). +```{important} +Keep frame sizes small — large frames increase training and inference time and may +exceed GPU memory. When running `extract_frames` with `crop=True`, you will be asked to +draw a bounding box in the GUI (this is saved to `config.yaml`). +``` `userfeedback` allows the user to check which videos they wish to extract frames from. In this way, if you added more videos to the config.yaml file it does not, by default, extract frames (again) from every video. If you wish to disable this question, set `userfeedback = True`. @@ -215,13 +284,15 @@ video and clusters the frames using k-means, where each frame is treated as a ve are then selected. This procedure makes sure that the frames look different. However, on large and long videos, this code is slow due to computational complexity. -**CRITICAL POINT:** It is advisable to extract frames from a period of the video that contains interesting -behaviors, and not extract the frames across the whole video. This can be achieved by using the start and stop -parameters in the config.yaml file. Also, the user can change the number of frames to extract from each video using -the numframes2extract in the config.yaml file. +```{important} +Extract frames from video segments that contain the behaviors of interest rather than +from the whole video. Use the `start` and `stop` parameters in `config.yaml` to limit +the extraction window, and `numframes2extract` to control the number of frames per +video. +``` -```{TIP} -For maDLC, **be sure you have labeled frames with closely interacting animals**! +```{tip} +For maDLC, **be sure you have labeled frames with closely interacting animals**! Therefore, manually selecting some frames is a good idea if interactions are not highly frequent in the video. ``` @@ -236,21 +307,23 @@ provided along with the toolbox. This can be launched by using: deeplabcut.extract_frames(config_path, 'manual') ``` -// FIXME(niels) - add a napari frame extractor description. + ````{admonition} Click the button to see API Docs -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.extract_frames.rst ``` ```` -### (D) Label Frames +#### (D) Label Frames ```python deeplabcut.label_frames(config_path) @@ -272,38 +345,47 @@ Keyboard arrows: advance frames. Delete key: delete label. ``` -![hot keys](https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/192345a5-e411-4d56-b718-ef52f91e195e/Qwerty.png?format=2500w) +```{figure} https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/192345a5-e411-4d56-b718-ef52f91e195e/Qwerty.png?format=2500w +--- +name: fig-labeling-hotkeys +alt: Keyboard shortcut reference for the DeepLabCut labeling GUI +--- +Keyboard shortcuts for the labeling GUI. +``` -**CRITICAL POINT:** It is advisable to **consistently label similar spots** (e.g., on a -wrist that is very large, try to label the same location). In general, invisible or -occluded points should not be labeled by the user, unless you want to teach the network -to "guess" - this is possible, but could affect accuracy. If you don't want/or don't see -a bodypart, they can simply be skipped by not applying the label anywhere on the frame. +```{important} +**Label similar spots consistently** (e.g., on a large wrist, always click the same +sub-location). Invisible or occluded points should generally not be labeled unless you +intentionally want to teach the network to predict occluded locations — this is +possible, but may reduce accuracy. Body parts that are not visible can simply be skipped +by leaving them unlabeled. +``` -OPTIONAL: In the event of adding more labels to the existing labeled dataset, the user -needs to append the new labels to the bodyparts in the config.yaml file. Thereafter, the -user can call the function **label_frames**. A box will pop up and ask the user if they -wish to display all parts, or only add in the new labels. Saving the labels after all -the images are labelled will append the new labels to the existing labeled dataset. +```{note} +To add new labels to an existing dataset, first append the new body parts to +`bodyparts` in `config.yaml`, then call `label_frames` again. A dialog will ask whether +to display all parts or only the new ones. Saving will append the new labels to the +existing dataset. +``` -**maDeepLabCut CRITICAL POINT:** For multi-animal labeling, unless you can tell apart -the animals, you do not need to worry about the "ID" of each animal. For example: if you -have a white and black mouse label the white mouse as animal 1, and black as animal 2 -across all frames. If two black mice, then the ID label 1 or 2 can switch between -frames - no need for you to try to identify them (but always label consistently within a -frame). If you have 2 black mice but one always has an optical fiber (for example), then -DO label them consistently as animal1 and animal_fiber (for example). The point of -multi-animal DLC is to train models that can first group the correct bodyparts to -individuals, then associate those points in a given video to a specific individual, -which then also uses temporal information to link across the video frames. +```{important} +**Multi-animal labeling and identity:** Unless you can visually distinguish the animals, +you do not need to maintain a consistent ID across frames. For example, with a white and +a black mouse, always label white as animal 1 and black as animal 2. With two +indistinguishable black mice the ID assignment (1 or 2) may switch between frames — +just be consistent *within* each frame. If one animal always has a distinguishing +feature (e.g., an optical fiber), then label them consistently across all frames. The +goal of maDLC is to train a model that groups body parts to individuals and then links +those individuals across video frames using temporal information. +``` Note, we also highly recommend that you use more bodyparts that you might otherwise have (see the example below). -For more information, checkout the [napari-deeplabcut docs](file:napari-gui-landing) for +For more information, checkout the {ref}`napari-deeplabcut docs ` for more information about the labelling workflow. -### (E) Check Annotated Frames +#### (E) Check Annotated Frames Checking if the labels were created and stored correctly is beneficial for training, since labeling is one of the most critical parts for creating the training dataset. The DeepLabCut toolbox provides a function @@ -311,29 +393,38 @@ is one of the most critical parts for creating the training dataset. The DeepLab ```python deeplabcut.check_labels(config_path, visualizeindividuals=True/False) - ``` +``` **maDeepLabCut:** you can check and plot colors per individual or per body part, just set the flag `visualizeindividuals=True/False`. Note, you can run this twice in both states to see both images. -

- -

+```{figure} https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1586203062876-D9ZL5Q7NZ464FUQN95NA/ke17ZwdGBToddI8pDm48kKmw982fUOZVIQXHUCR1F55Zw-zPPgdn4jUwVcJE1ZvWQUxwkmyExglNqGp0IvTJZUJFbgE-7XRK3dMEBRBhUpx7krGdD6VO1HGZR3BdeCbrijc_yIxzfnirMo-szZRSL5-VIQGAVcQr6HuuQP1evvE/img1068_individuals.png?format=750w +--- +name: fig-check-labels-individuals +alt: Example check_labels output showing annotated individuals per frame +width: 50% +align: center +--- +Example `check_labels` output with annotations shown per individual. +``` For each video directory in labeled-data this function creates a subdirectory with **labeled** as a suffix. Those directories contain the frames plotted with the annotated body parts. The user can double check if the body parts are labeled correctly. If they are not correct, the user can reload the frames (i.e. `deeplabcut.label_frames`), move them around, and click save again. ````{admonition} Click the button to see API Docs -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.check_labels.rst ``` ```` -### (F) Create Training Dataset +### Phase 3 — Training and evaluation + +#### (F) Create Training Dataset At this point, you'll need to select your neural network type. -For the **PyTorch engine**, please see [the PyTorch Model Architectures]( -dlc3-architectures) for options. +For the **PyTorch engine**, please see [the PyTorch Model Architectures](dlc3-architectures) for options. For the **TensorFlow engine**, please see Lauer et al. 2021 for options. Multi-animal models will use `imgaug`, ADAM optimization, our new DLCRNet, and batch training. We @@ -341,8 +432,7 @@ suggest keeping these defaults at this time. At this step, the ImageNet pre-trai networks (i.e. ResNet-50) weights will be downloaded. If they do not download (you will see this downloading in the terminal, then you may not have permission to do so ( something we have seen with some Windows users - see the **[ -WIKI troubleshooting for more help!]( -https://github.com/DeepLabCut/DeepLabCut/wiki/Troubleshooting-Tips)**). +WIKI troubleshooting for more help!](https://github.com/DeepLabCut/DeepLabCut/wiki/Troubleshooting-Tips)**). Then run: @@ -351,53 +441,52 @@ deeplabcut.create_training_dataset(config_path) ``` - The set of arguments in the function will shuffle the combined labeled dataset and split it to create train and test -sets. The subdirectory with suffix ``iteration#`` under the directory **training-datasets** stores the dataset and meta -information, where the ``#`` is the value of ``iteration`` variable stored in the project’s configuration file (this number -keeps track of how often the dataset was refined). + sets. The subdirectory with suffix `iteration#` under the directory **training-datasets** stores the dataset and meta + information, where the `#` is the value of `iteration` variable stored in the project’s configuration file (this number + keeps track of how often the dataset was refined). -- OPTIONAL: If the user wishes to benchmark the performance of the DeepLabCut, they can create multiple -training datasets by specifying an integer value to the `num_shuffles`; see the docstring for more details. +- To benchmark performance across multiple train/test splits, pass an integer to + `num_shuffles`; see the docstring for details. - Each iteration of the creation of a training dataset will create several files, which -is used by the feature detectors, and a ``.pickle`` file that contains the meta -information about the training dataset. This also creates two subdirectories within -**dlc-models-pytorch** (**dlc-models** for the TensorFlow engine) called ``test`` and -``train``, and these each have a configuration file called pose_cfg.yaml. Specifically, -the user can edit the **pytorch_config.yaml** (**pose_cfg.yaml** for TensorFlow engine) -within the **train** subdirectory before starting the training. These configuration -files contain meta information with regard to the parameters of the feature detectors. -Key parameters are listed in Box 2. + is used by the feature detectors, and a `.pickle` file that contains the meta + information about the training dataset. This also creates two subdirectories within + **dlc-models-pytorch** (**dlc-models** for the TensorFlow engine) called `test` and + `train`, and these each have a configuration file called pose_cfg.yaml. Specifically, + the user can edit the **pytorch_config.yaml** (**pose_cfg.yaml** for TensorFlow engine) + within the **train** subdirectory before starting the training. These configuration + files contain meta information with regard to the parameters of the feature detectors. + Key parameters are listed in Box 2. **DATA AUGMENTATION:** At this stage you can also decide what type of augmentation to use. Once you've called `create_training_dataset`, you can edit the [**pytorch_config.yaml**](dlc3-pytorch-config) file that was created (or for the -TensorFlow engine, the [**pose_cfg.yaml**]( -https://github.com/DeepLabCut/DeepLabCut/blob/master/deeplabcut/pose_cfg.yaml) file). +TensorFlow engine, the [**pose_cfg.yaml**](https://github.com/DeepLabCut/DeepLabCut/blob/master/deeplabcut/pose_cfg.yaml) file). - PyTorch Engine: [Albumentations](https://albumentations.ai/docs/) is used for data -augmentation. Look at the [**pytorch_config.yaml**](dlc3-pytorch-config) for more -information about image augmentation options. + augmentation. Look at the [**pytorch_config.yaml**](dlc3-pytorch-config) for more + information about image augmentation options. - TensorFlow Engine: The default augmentation works well for most tasks (as shown on -www.deeplabcut.org), but there are many options, more data augmentation, intermediate -supervision, etc. Only `imgaug` augmentation is available for multi-animal projects. + www.deeplabcut.org), but there are many options, more data augmentation, intermediate + supervision, etc. Only `imgaug` augmentation is available for multi-animal projects. -[A Primer on Motion Capture with Deep Learning: Principles, Pitfalls, and Perspectives]( -https://www.cell.com/neuron/pdf/S0896-6273(20)30717-0.pdf), details the advantage of +[A Primer on Motion Capture with Deep Learning: Principles, Pitfalls, and Perspectives](), details the advantage of augmentation for a worked example (see Fig 8). TL;DR: use imgaug and use the symmetries of your data! Importantly, image cropping as previously done with `deeplabcut.cropimagesandlabels` in multi-animal projects -is now part of the augmentation pipeline. In other words, image crops are no longer stored in labeled-data/..._cropped +is now part of the augmentation pipeline. In other words, image crops are no longer stored in labeled-data/...\_cropped folders. Crop size still defaults to (400, 400); if your images are very large (e.g. 2k, 4k pixels), consider increasing the crop size, but be aware unless you have a strong GPU (24 GB memory or more), you will hit memory errors. You can lower the batch size, but this may affect performance. In addition, one can specify a crop sampling strategy: crop centers can either be taken at random over the image (`uniform`) or the annotated keypoints (`keypoints`); with a focus on regions of the scene with high body part density (`density`); last, combining `uniform` and `density` for a `hybrid` balanced strategy (this is the default strategy). Note that both parameters can be easily edited prior to training in the **pose_cfg.yaml** configuration file. As a reminder, cropping images into smaller patches is a form of data augmentation that simultaneously allows the use of batch processing even on small GPUs that could not otherwise accommodate larger images + larger batchsizes (this usually increases performance and decreasing training time). -**MODEL COMPARISON**: You can also test several models by creating the same train/test -split for different networks. -You can easily do this in the Project Manager GUI (by selecting the "Use an existing -data split" option), which also lets you compare PyTorch and TensorFlow models. +```{tip} +To compare multiple model architectures on the same train/test split, select "Use an +existing data split" in the Project Manager GUI. This also lets you compare PyTorch and +TensorFlow models side by side. +``` ````{versionadded} 3.0.0 You can now create new shuffles using the same train/test split as @@ -420,27 +509,33 @@ deeplabcut.create_training_dataset_from_existing_split( ```` ````{admonition} Click the button to see API Docs for deeplabcut.create_training_dataset -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.create_training_dataset.rst ``` ```` ````{admonition} Click the button to see API Docs for deeplabcut.create_training_model_comparison -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.create_training_model_comparison.rst ``` ```` ````{admonition} Click the button to see API Docs for deeplabcut.create_training_dataset_from_existing_split -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.create_training_dataset_from_existing_split.rst ``` ```` -### (G) Train The Network +#### (G) Train The Network ```python deeplabcut.train_network(config_path, shuffle=1) @@ -455,8 +550,9 @@ At user specified iterations during training checkpoints are stored in the subdi *train* under the respective iteration & shuffle directory. ````{admonition} Tips on training models with the PyTorch Engine -:class: dropdown - +--- +class: dropdown +--- Example parameters that one can call: ```python @@ -488,9 +584,10 @@ full path of the checkpoint to the variable ``resume_training_from`` in the [ dlc3-pytorch-config) file (checkout the "Restarting Training at a Specific Checkpoint" section of the docs) under the *train* subdirectory. -**CRITICAL POINT:** It is recommended to train the networks **until the loss plateaus** -(depending on the dataset, model architecture and training hyper-parameters this happens -after 100 to 250 epochs of training). +```{important} +Train the network **until the loss plateaus** — depending on the dataset, model +architecture, and hyper-parameters this typically occurs after 100–250 epochs. +``` The variables ``display_iters`` and ``save_epochs`` in the [**pytorch_config.yaml**]( dlc3-pytorch-config) file allows the user to alter how often the loss is displayed @@ -498,8 +595,9 @@ and how often the weights are stored. We suggest saving every 5 to 25 epochs. ```` ````{admonition} Tips on training models with the TensorFlow Engine -:class: dropdown - +--- +class: dropdown +--- Example parameters that one can call: ```python @@ -528,9 +626,10 @@ If the user wishes to restart the training at a specific checkpoint they can spe full path of the checkpoint to the variable ``init_weights`` in the **pose_cfg.yaml** file under the *train* subdirectory (see Box 2). -**CRITICAL POINT:** It is recommended to train the networks for thousands of iterations -until the loss plateaus (typically around **500,000**) if you use batch size 1, and -**50-100K** if you use batchsize 8 (the default). +```{important} +Train until the loss plateaus — typically around **500,000** iterations with batch +size 1, or **50–100K** iterations with batch size 8 (the default). +``` If you use **maDeepLabCut** the recommended training iterations is **20K-100K** (it automatically stops at 200K!), as we use Adam and batchsize 8; if you have to reduce @@ -539,43 +638,47 @@ If you use **maDeepLabCut** the recommended training iterations is **20K-100K** The variables ``display_iters`` and ``save_iters`` in the **pose_cfg.yaml** file allows the user to alter how often the loss is displayed and how often the weights are stored. -**maDeepLabCut CRITICAL POINT:** For multi-animal projects we are using not only -different and new output layers, but also new data augmentation, optimization, learning -rates, and batch training defaults. Thus, please use a lower ``save_iters`` and -``maxiters``. I.e. we suggest saving every 10K-15K iterations, and only training until -50K-100K iterations. We recommend you look closely at the loss to not overfit on your -data. The bonus, training time is much less!!! +```{important} +Multi-animal projects use different output layers, data augmentation, optimizers, +learning rates, and batch defaults compared to single-animal projects. Use a lower +`save_iters` and `maxiters`: save every 10K–15K iterations and stop training at +50K–100K iterations. Monitor the loss curve carefully to avoid overfitting. Training +time is correspondingly shorter. +``` ```` ````{admonition} Click the button to see API Docs for train_network -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.train_network.rst ``` ```` -### (H) Evaluate the Trained Network +#### (H) Evaluate the Trained Network It is important to evaluate the performance of the trained network. This performance is measured by computing two metrics: - **Average root mean square error** (RMSE) between the manual labels and the ones -predicted by your trained DeepLabCut model. The RMSE is proportional to the mean average -Euclidean error (MAE) between the manual labels and the ones predicted by DeepLabCut. -The MAE is displayed for all pairs and only likely pairs (>p-cutoff). This helps to -exclude, for example, occluded body parts. One of the strengths of DeepLabCut is that -due to the probabilistic output of the scoremap, it can, if sufficiently trained, also -reliably report if a body part is visible in a given frame. (see discussions of finger -tips in reaching and the Drosophila legs during 3D behavior in [Mathis et al, 2018]). + predicted by your trained DeepLabCut model. The RMSE is proportional to the mean average + Euclidean error (MAE) between the manual labels and the ones predicted by DeepLabCut. + The MAE is displayed for all pairs and only likely pairs (>p-cutoff). This helps to + exclude, for example, occluded body parts. One of the strengths of DeepLabCut is that + due to the probabilistic output of the scoremap, it can, if sufficiently trained, also + reliably report if a body part is visible in a given frame. (see discussions of finger + tips in reaching and the Drosophila legs during 3D behavior in [Mathis et al, 2018]). - **Mean Average Precision** (mAP) and **Mean Average Recall** (mAR) for the individuals -predicted by your trained DeepLabCut model. This metric describes the precision of your -model, based on a considered definition of what a correct detection of an individual is. -It isn't as useful for single-animal models, as RMSE does a great job of evaluating your -model in that case. + predicted by your trained DeepLabCut model. This metric describes the precision of your + model, based on a considered definition of what a correct detection of an individual is. + It isn't as useful for single-animal models, as RMSE does a great job of evaluating your + model in that case. ```{admonition} A more detailed description of mAP and mAR -:class: dropdown - +--- +class: dropdown +--- For multi-animal pose estimation, multiple predictions can be made for each image. We want to get some idea of the proportion of correct predictions among all predictions that are made. @@ -606,7 +709,7 @@ deeplabcut.evaluate_network(config_path, Shuffles=[1], plotting=True) 🎥 [VIDEO TUTORIAL AVAILABLE!](https://www.youtube.com/watch?v=bgfnz1wtlpo) -Setting ``plotting`` to True plots all the testing and training frames with the manual and predicted labels; these will +Setting `plotting` to True plots all the testing and training frames with the manual and predicted labels; these will be colored by body part type by default. They can alternatively be colored by individual by passing `plotting="individual"`. The user should visually check the labeled test (and training) images that are created in the ‘evaluation-results’ directory. Ideally, DeepLabCut labeled unseen (test images) according to the user’s required accuracy, and the average train @@ -617,34 +720,35 @@ also be larger than the training error due to human variability (in labeling, se **Optional parameters:** - `Shuffles: list, optional` - List of integers specifying the shuffle indices of the training dataset. -The default is [1] + The default is [1] - `plotting: bool | str, optional` - Plots the predictions on the train and test images. The default is `False`; -if provided it must be either `True`, `False`, `"bodypart"`, or `"individual"`. + if provided it must be either `True`, `False`, `"bodypart"`, or `"individual"`. - `show_errors: bool, optional` - Display train and test errors. The default is `True` - `comparisonbodyparts: list of bodyparts, Default is all` - The average error will be computed for those body parts -only (Has to be a subset of the body parts). + only (Has to be a subset of the body parts). - `gputouse: int, optional` - Natural number indicating the number of your GPU (see number in nvidia-smi). If you do not -have a GPU, put None. See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries + have a GPU, put None. See: https://nvidia.custhelp.com/app/answers/detail/a_id/3751/~/useful-nvidia-smi-queries - `pcutoff: float | list[float] | dict[str, float], optional` -(Only applicable when using the PyTorch engine. For TensorFlow, set `pcutoff` in the `config.yaml` file.) -Specifies the cutoff value(s) used to compute evaluation metrics. + (Only applicable when using the PyTorch engine. For TensorFlow, set `pcutoff` in the `config.yaml` file.) + Specifies the cutoff value(s) used to compute evaluation metrics. + - If `None` (default), the cutoff will be loaded from the project configuration. - To apply a single cutoff value to all bodyparts, provide a `float`. - To specify different cutoffs per bodypart, provide either: - A `list[float]`: one value per bodypart, with an additional value for each unique bodypart if applicable. - A `dict[str, float]`: where keys are bodypart names and values are the corresponding cutoff values. -If a bodypart is not included in the provided dictionary, a default `pcutoff` of `0.6` will be used for that bodypart. + If a bodypart is not included in the provided dictionary, a default `pcutoff` of `0.6` will be used for that bodypart. The plots can be customized by editing the **config.yaml** file (i.e., the colormap, scale, marker size (dotsize), and transparency of labels (alpha-value) can be modified). By default each body part is plotted in a different color (governed by the colormap) and the plot labels indicate their source. Note that by default the human labels are plotted as plus (‘+’), DeepLabCut’s predictions either as ‘.’ (for confident predictions with likelihood > `pcutoff`) and -’x’ for (likelihood <= `pcutoff`). +’x’ for (likelihood \<= `pcutoff`). The evaluation results for each shuffle of the training dataset are stored in a unique subdirectory in a newly created directory ‘evaluation-results-pytorch’ (or @@ -662,7 +766,9 @@ and the points of interest are labeled accurately • consider labeling additional images and make another iteration of the training data set ````{admonition} Click the button to see API Docs for evaluate_network -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.evaluate_network.rst ``` @@ -683,38 +789,44 @@ deeplabcut.extract_save_all_maps(config_path, shuffle=shuffle, Indices=[0, 5]) You can drop "Indices" to run this on all training/testing images (this is very slow!) -### (I) Analyze new Videos +### Phase 4 — Video analysis and tracking -````{versionadded} 3.0.0 -With the addition of conditional top-down models in DeepLabCut 3.0, it's now possible to -track individuals directly **during video analysis**. If you choose to train any model -with a name that starts with `ctd_`, you'll be able to call `deeplabcut.analyze_videos` -with `ctd_tracking=True`. To learn more about tracking with CTD, see the [ -`COLAB_BUCTD_and_CTD_tracking`]( -https://github.com/DeepLabCut/DeepLabCut/blob/main/examples/COLAB/COLAB_BUCTD_and_CTD_tracking.ipynb) -COLAB notebook. -```` +#### (I) Analyze new Videos -**-------------------- DECISION POINT -------------------** +```{important} +Before moving on, make a deliberate decision about whether the pose estimation quality is sufficient. If you do not have good pose estimation evaluation metrics at this point, please revisit the original labels, add more training data and refine the model rather than proceeding with the current results. +``` -**ATTENTION!** -**Pose estimation and tracking should be thought of as separate steps.** If you do not -have good pose estimation evaluation metrics at this point, stop, check original labels, -add more data, etc --> don't move forward with this model. If you think you have a good -model, please test the "raw" pose estimation performance on a video to validate -performance: +```{note} +In prior versions of DeepLabCut, pose estimation and tracking were separate procedures. From version 3.0 onward, `deeplabcut.analyze_videos` runs the **full pose estimation + tracking pipeline** by default (`auto_track=True`), producing an .h5 file ready for downstream use. To inspect raw detections before any of this is applied, pass `auto_track=False` explicitly. +``` + +##### Pose estimation quality check -Please run: +To validate raw pose estimation performance on a video before committing to the tracking +results, run: ```python videos_to_analyze = ['/fullpath/project/videos/testVideo.mp4'] -scorername = deeplabcut.analyze_videos(config_path, videos_to_analyze, videotype='.mp4') +deeplabcut.analyze_videos(config_path, videos_to_analyze, videotype='.mp4', auto_track=False) deeplabcut.create_video_with_all_detections(config_path, videos_to_analyze, videotype='.mp4') ``` -Please note that you do **not** get the .h5/csv file you might be used to getting (this -comes after tracking). You will get a `pickle` file that is used in -`create_video_with_all_detections`. +With `auto_track=False`, no `.h5` file is produced — only a `*_full.pickle` file +containing the raw detections, which is what `create_video_with_all_detections` uses to +render all detections before any individual is assigned. + +```{versionadded} 3.0.0 +For conditional top-down (CTD) models, tracking can be performed **inside the model +during inference**, using temporal context from previous frames to condition predictions +on the current frame. This is a distinct mechanism from `auto_track`: pass +`ctd_tracking=True` to `deeplabcut.analyze_videos` when using any model whose name +starts with `ctd_`. When `ctd_tracking=True`, post-processing tracking (`auto_track`) is +skipped automatically. To learn more, see the [ +`COLAB_BUCTD_and_CTD_tracking`]( +https://github.com/DeepLabCut/DeepLabCut/blob/main/examples/COLAB/COLAB_BUCTD_and_CTD_tracking.ipynb) +COLAB notebook. +``` For models predicting part-affinity fields, another sanity check may be to examine the distributions of edge affinity costs using `deeplabcut.utils.plot_edge_affinity_distributions`. Easily separable distributions @@ -733,9 +845,10 @@ are far apart for most edges), then go forward!!! If this does not look good, we recommend extracting and labeling more frames (even from more videos). Try to label close interactions of animals for best performance. Once you label more, you can create a new training set and train. You can either: + 1. extract more frames manually from existing or new videos and label as when initially building the training data set, or -2. let DeepLabCut find frames where keypoints were poorly detected and automatically extract those for you. All you need is -to run: +1. let DeepLabCut find frames where keypoints were poorly detected and automatically extract those for you. All you need is + to run: ```python deeplabcut.find_outliers_in_raw_data(config_path, pickle_file, video_file) @@ -744,28 +857,28 @@ deeplabcut.find_outliers_in_raw_data(config_path, pickle_file, video_file) where pickle_file is the `_full.pickle` one obtains after video analysis. Flagged frames will be added to your collection of images in the corresponding labeled-data folders for you to label. - -### Animal Assembly and Tracking across frames +##### Animal Assembly and Tracking across frames After pose estimation, now you perform assembly and tracking. -````{versionadded} v2.2.0 +```{versionadded} v2.2.0 *NEW* in 2.2 is a novel data-driven way to set the optimal skeleton and assembly metrics, so this no longer requires user input. The metrics, in case you do want to edit them, can be found in the `inference_cfg.yaml` file. -```` +``` + +##### Optimized Animal Assembly + Video Analysis: -### Optimized Animal Assembly + Video Analysis: Please note that **novel videos DO NOT need to be added to the config.yaml file**. You can simply have a folder elsewhere on your computer and pass the video folder (then it -will analyze all videos of the specified type (i.e. ``videotype='.mp4'``), or pass the +will analyze all videos of the specified type (i.e. `videotype='.mp4'`), or pass the path to the **folder** or exact video(s) you wish to analyze: ```python deeplabcut.analyze_videos(config_path, ['/fullpath/project/videos/'], videotype='.mp4', auto_track=True) ``` -### IF auto_track = True: +##### IF auto_track = True: ```{versionadded} v2.2.0.3 A new argument `auto_track=True`, was added to `deeplabcut.analyze_videos` chaining pose @@ -776,7 +889,7 @@ DLC. If `auto_track=False`, one must run `convert_detections2tracklets` and the workflow (ideal for advanced users). ``` -### IF auto_track = False: +##### IF auto_track = False: You can validate the tracking parameters. Namely, you can iteratively change the parameters, run `convert_detections2tracklets` then load them in the GUI @@ -796,29 +909,29 @@ max_age: 100 min_hits: 3 ``` - - **IMPORTANT POINT FOR SUPERVISED IDENTITY TRACKING** - - If the network has been trained to learn the animals' identities (i.e., you set `identity=True` in config.yaml before training) this information can be leveraged both during: (i) animal assembly, where body parts are grouped based on the animal they are predicted to belong to (affinity between pairs of keypoints is no longer considered in that case); and (ii) animal tracking, where identity only can be utilized in place of motion trackers to form tracklets. +If the network was trained with identity supervision (i.e., `identity=True` in +`config.yaml` before training), this information can be leveraged during: (i) animal +assembly, where body parts are grouped by predicted identity rather than keypoint +affinity; and (ii) tracking, where identity alone can be used in place of motion +trackers to form tracklets. To use this ID information, simply pass: + ```python deeplabcut.convert_detections2tracklets(..., identity_only=True) ``` - **Note:** If only one individual is to be assembled and tracked, assembly and tracking are skipped, and detections are treated as in single-animal projects; i.e., it is the keypoints with highest confidence that are kept and accumulated over frames to form a single, long tracklet. No action is required from users, this is done automatically. - **Animal assembly and tracking quality** can be assessed via `deeplabcut.utils.make_labeled_video.create_video_from_pickled_tracks`. This function provides an additional diagnostic tool before moving on to refining tracklets. - If animal assemblies do not look pretty, an alternative to the outlier search described above is to pass the `_assemblies.pickle` to `find_outliers_in_raw_data` in place of the `_full.pickle`. This will focus the outlier search on unusual assemblies (i.e., animal skeletons that were oddly reconstructed). This may be a bit more sensitive with crowded scenes or frames where animals interact closely. Note though that at that stage it is likely preferable anyway to carry on with the remaining steps, and extract outliers from the final h5 file as was customary in single animal projects. - -**Next, tracklets are stitched to form complete tracks with: +\*\*Next, tracklets are stitched to form complete tracks with: ```python deeplabcut.stitch_tracklets( @@ -841,30 +954,36 @@ deeplabcut.stitch_tracklets(..., n_tracks=n) In such cases, file columns will default to dummy animal names (ind1, ind2, ..., up to indn). -### API Docs +##### API Docs ````{admonition} Click the button to see API Docs for analyze_videos -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.analyze_videos.rst ``` ```` ````{admonition} Click the button to see API Docs for convert_detections2tracklets -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.convert_detections2tracklets.rst ``` ```` ````{admonition} Click the button to see API Docs for stitch_tracklets -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.stitch_tracklets.rst ``` ```` -### Using Unsupervised Identity Tracking: +##### Using Unsupervised Identity Tracking: In Lauer et al. 2022 we introduced a new method to do unsupervised reID of animals. Here, you can use the tracklets to learn the identity of animals to enhance your @@ -876,7 +995,7 @@ deeplabcut.transformer_reID(config, videos_to_analyze, n_tracks=None, videotype= Note you should pass the n_tracks (number of animals) you expect to see in the video. -### Refine Tracklets: +##### Refine Tracklets: You can also optionally **refine the tracklets**. You can fix both "major" ID swaps, i.e. perhaps when animals cross, and you can micro-refine the individual body points. You will load the `...trackertype.pickle` or `.h5'` file that was created above, and then you can launch a GUI to interactively refine the data. This also has several options, so please check out the docstring. Upon saving the refined tracks you get an `.h5` file (akin to what you might be used to from standard DLC. You can also load (1) filter this to take care of small jitters, and (2) load this `.h5` this to refine (again) in case you find another issue, etc! @@ -886,121 +1005,311 @@ deeplabcut.refine_tracklets(config_path, pickle_or_h5_file, videofile_path, max_ If you use the GUI (or otherwise), here are some settings to consider: -maDLC +```{figure} https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1619628014395-BQ09VLLTKCLQQGRB5T9A/ke17ZwdGBToddI8pDm48kLMj_XrWI9gi4tVeBdgcB8p7gQa3H78H3Y0txjaiv_0fDoOvxcdMmMKkDsyUqMSsMWxHk725yiiHCCLfrh8O1z4YTzHvnKhyp6Da-NYroOW3ZGjoBKy3azqku80C789l0lt53wR20brczws2A6XSGt3kSTbW7uM0ncVKHWPvgHR4kN5Ka1TcK96ljy4ji9jPkQ/TrackletGUI.png?format=1000w +--- +name: fig-tracklet-gui +alt: Tracklet refinement GUI showing key settings +width: 950px +align: center +--- +Tracklet refinement GUI. Key settings to configure are described in the text. +``` -*note, setting `max_gap=0` can be used to fill in all frames across the video; otherwise, 1-n is the # of frames you want to fill in, i.e. maybe you want to fill in short gaps of 5 frames, but 15 frames indicates another issue, etc. You can test this in the GUI very easy by editing the value and then re-launch pop-up GUI. +\*note, setting `max_gap=0` can be used to fill in all frames across the video; otherwise, 1-n is the # of frames you want to fill in, i.e. maybe you want to fill in short gaps of 5 frames, but 15 frames indicates another issue, etc. You can test this in the GUI very easy by editing the value and then re-launch pop-up GUI. If you fill in gaps, they will be associated to an ultra low probability, 0.01, so you are aware this is not the networks best estimate, this is the human-override! Thus, if you create a video, you need to set your pcutoff to 0 if you want to see these filled in frames. [Read more here!](functionDetails.md#madeeplabcut-critical-point---assemble--refine-tracklets) Short demo: -

- -

-### (J) Filter Pose Data +```{figure} https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1588690928000-90ZMRIM8SN6QE20ZOMNX/ke17ZwdGBToddI8pDm48kJ1oJoOIxBAgRD2ClXVCmKFZw-zPPgdn4jUwVcJE1ZvWQUxwkmyExglNqGp0IvTJZUJFbgE-7XRK3dMEBRBhUpxBw7VlGKDQO2xTcc51Yv6DahHgScLwHgvMZoEtbzk_9vMJY_JknNFgVzVQ2g0FD_s/refineDEMO.gif?format=750w +--- +name: fig-refine-tracklets-demo +alt: Animated demonstration of the tracklet refinement workflow +width: 70% +align: center +--- +Short demo of the tracklet refinement workflow. +``` + +#### (J) Filter Pose Data Firstly, Here are some tips for scaling up your video analysis, including looping over many folders for batch processing: https://github.com/DeepLabCut/DeepLabCut/wiki/Batch-Processing-your-Analysis You can also filter the predicted bodyparts by: + ```python deeplabcut.filterpredictions(config_path,['/fullpath/project/videos/reachingvideo1.avi']) ``` -Note, this creates a file with the ending filtered.h5 that you can use for further analysis. This filtering step has many parameters, so please see the full docstring by typing: ``deeplabcut.filterpredictions?`` + +Note, this creates a file with the ending filtered.h5 that you can use for further analysis. This filtering step has many parameters, so please see the full docstring by typing: `deeplabcut.filterpredictions?` ````{admonition} Click the button to see API Docs -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.filterpredictions.rst ``` ```` -### (K) Plot Trajectories , (L) Create Labeled Videos +#### (K) Plot Trajectories -- **NOTE :bulb::mega::** Before you create a video, you should set what threshold to use for plotting. This is set in the `config.yaml` file as `pcutoff` - if you have a well trained network, this should be high, i.e. set it to `0.8` or higher! IF YOU FILLED IN GAPS, you need to set this to `0` to "see" the filled in parts. +Before creating labeled videos, set the `pcutoff` threshold in `config.yaml`. For a +well-trained network this should be high, e.g. `0.8` or higher. If you filled in gaps, +set it to `0` to make those interpolated points visible. +You can determine a good `pcutoff` value by inspecting the likelihood plot produced by +`plot_trajectories`: -- You can also determine a good `pcutoff` value by looking at the likelihood plot created during `plot_trajectories`: - -Plot the outputs: -```python - deeplabcut.plot_trajectories(config_path,['/fullpath/project/videos/reachingvideo1.avi'],filtered = True) -``` - -Create videos: ```python - deeplabcut.create_labeled_video(config_path, [videos], videotype='avi', shuffle=1, trainingsetindex=0, filtered=False, fastmode=True, save_frames=False, keypoints_only=False, Frames2plot=None, displayedbodyparts='all', displayedindividuals='all', codec='mp4v', outputframerate=None, destfolder=None, draw_skeleton=False, trailpoints=0, displaycropped=False, color_by='bodypart', track_method='') +deeplabcut.plot_trajectories(config_path, ['/fullpath/project/videos/reachingvideo1.avi'], filtered=True) ``` -- **NOTE :bulb::mega::** You have a lot of options in terms of video plotting (quality, display type, etc). We recommend checking the docstring! - -(more details [here](functionDetails.md#i-video-analysis-and-plotting-results)) ````{admonition} Click the button to see API Docs for plot_trajectories -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.plot_trajectories.rst ``` ```` +#### (L) Create Labeled Videos + +There are many options for controlling video quality and display style — check the +docstring for the full list. More details are also available +[here](functionDetails.md#i-video-analysis-and-plotting-results). + +```python +deeplabcut.create_labeled_video( + config_path, [videos], videotype='avi', shuffle=1, trainingsetindex=0, + filtered=False, fastmode=True, save_frames=False, keypoints_only=False, + Frames2plot=None, displayedbodyparts='all', displayedindividuals='all', + codec='mp4v', outputframerate=None, destfolder=None, draw_skeleton=False, + trailpoints=0, displaycropped=False, color_by='bodypart', track_method='', +) +``` + ````{admonition} Click the button to see API Docs for create_labeled_video -:class: dropdown +--- +class: dropdown +--- ```{eval-rst} .. include:: ./api/deeplabcut.create_labeled_video.rst ``` ```` -### HELP: +### Phase 5 — Refinement (optional) + +#### (M) Optional Active Learning - Network Refinement: Extract Outlier Frames + +##### Overview -In ipython/Jupyter notebook: +While DeepLabCut typically generalizes well across datasets, one might want to optimize its performance in various, +perhaps unexpected, situations. For generalization to large datasets, images with insufficient labeling performance +can be extracted, manually corrected by adjusting the labels to increase the training set and iteratively improve the +feature detectors. Such an active learning framework can be used to achieve a predefined level of confidence for all +images with minimal labeling cost (discussed in Mathis et al 2018). Then, due to the large capacity of the neural network that underlies the feature detectors, one can continue training the network with these additional examples. One does not +necessarily need to correct all errors as common errors could be eliminated by relabeling a few examples and then +re-training. A priori, given that there is no ground truth data for analyzed videos, it is challenging to find putative +“outlier frames”. However, one can use heuristics such as the continuity of body part trajectories, to identify images +where the decoder might make large errors. +All this can be done for a specific video by typing (see other optional inputs below): + +##### Code example + +```python +deeplabcut.extract_outlier_frames(config_path, ["videofile_path"]) ``` -deeplabcut.nameofthefunction? + +##### Frame-selection methods + +We provide various frame-selection methods for this purpose. In particular +the user can set: + +```text +outlieralgorithm: "fitting", "jump", or "uncertain" ``` -In python or pythonw: +- `outlieralgorithm="uncertain"`: select frames if the likelihood of a particular or all body parts lies below `p_bound` + (note this could also be due to occlusions rather than errors). + +- `outlieralgorithm="jump"`: select frames where a particular body part or all body parts jumped more than `epsilon` + pixels from the last frame. + +- `outlieralgorithm="fitting"`: select frames if the predicted body part location deviates from a state-space model fit + to the time series of individual body parts. Specifically, this method fits an Auto Regressive Integrated Moving Average + (ARIMA) model to the time series for each body part. Thereby each body part detection with a likelihood smaller than + `p_bound` is treated as missing data. Putative outlier frames are then identified as time points, where the average + body part estimates are at least `epsilon` pixels away from the fits. The parameters of this method are `epsilon`, + `p_bound`, the ARIMA parameters as well as the list of body parts to average over (can also be `all`). + +- `outlieralgorithm="manual"`: manually select outlier frames based on visual inspection from the user. +As an example: + +```python +deeplabcut.extract_outlier_frames(config_path, ["videofile_path"], outlieralgorithm="manual") ``` -help(deeplabcut.nameofthefunction) + +##### Selection after detection + +In general, depending on the parameters, these methods might return many more frames than the user wants to +extract (`numframes2pick`). Thus, this list is then used to select outlier frames either by randomly sampling from +this list (`extractionalgorithm="uniform"`), by performing `extractionalgorithm="kmeans"` clustering on the +corresponding frames. + +In the automatic configuration, before the frame selection happens, the user is informed about the amount of frames +satisfying the criteria and asked if the selection should proceed. This step allows the user to perhaps change the +parameters of the frame-selection heuristics first (i.e. to make sure that not too many frames are qualified). The user +can run the `extract_outlier_frames` method iteratively, and (even) extract additional frames from the same video. +Once enough outlier frames are extracted the refinement GUI can be used to adjust the labels based on user feedback +(see below). + +##### API Docs + +````{admonition} Click the button to see API Docs +--- +class: dropdown +--- +```{eval-rst} +.. include:: ./api/deeplabcut.extract_outlier_frames.rst ``` +```` + +______________________________________________________________________ + +#### (N) Refine Labels: Augmentation of the Training Dataset + +##### Overview + +Based on the performance of DeepLabCut, four scenarios are possible: + +- (A) Visible body part with accurate DeepLabCut prediction. These labels do not need any modifications. + +- (B) Visible body part but wrong DeepLabCut prediction. Move the label’s location to the actual position of the + body part. -## Tips for "daily" use: +- (C) Invisible, occluded body part. Remove the predicted label by DeepLabCut with a middle click. Every predicted + label is shown, even when DeepLabCut is uncertain. This is necessary, so that the user can potentially move + the predicted label. However, to help the user to remove all invisible body parts the low-likelihood predictions + are shown as open circles (rather than disks). -

- -

+- (D) Invalid images: In the unlikely event that there are any invalid images, the user should remove such an image + and their corresponding predictions, if any. Here, the GUI will prompt the user to remove an image identified + as invalid. -You can always exit an conda environment and easily jump back into a project by simply: +The labels for extracted putative outlier frames can be refined by opening the GUI: -Linux/MacOS formatting example: +##### Code example + +```python +deeplabcut.refine_labels(config_path) ``` -source activate yourdeeplabcutEnvName -ipython or pythonw -import deeplabcut -config_path ='/home/yourprojectfolder/config.yaml' + +This will launch a GUI where the user can refine the labels. + +Please refer to the {ref}`napari-deeplabcut docs ` for more information about the labelling workflow. + +##### Merge datasets + +After correcting the labels for all the frames in each of the subdirectories, the users should merge the dataset to +create a new dataset. In this step the iteration parameter in the config.yaml file is automatically updated. + +```python +deeplabcut.merge_datasets(config_path) ``` -Windows formatting example: + +Once the dataset is merged, the user can test if the merging process was successful by plotting all the labels (Step E). +Next, with this expanded training set the user can now create a new training set and train the network as described +in Steps F and G. The training dataset will be stored in the same place as before but under a different `iteration-#` +subdirectory, where the `#` is the new value of `iteration` variable stored in the project’s configuration file +(this is automatically done). + +Now you can run `create_training_dataset`, then `train_network`, etc. If your original labels were adjusted at all, +start from fresh weights (which is generally recommended), otherwise consider using your already trained network +weights (see {ref}`Box 2 `). + +If after training the network generalizes well to the data, proceed to analyze new videos. Otherwise, consider labeling +more data. + +##### API Docs for deeplabcut.refine_labels + +````{admonition} Click the button to see API Docs +--- +class: dropdown +--- +```{eval-rst} +.. include:: ./api/deeplabcut.refine_labels.rst ``` -activate yourdeeplabcutEnvName -ipython -import deeplabcut -config_path = r'C:\home\yourprojectfolder\config.yaml' +```` + +##### API Docs for deeplabcut.merge_datasets + +````{admonition} Click the button to see API Docs +--- +class: dropdown +--- +```{eval-rst} +.. include:: ./api/deeplabcut.merge_datasets.rst ``` +```` + +## Resources and further reading + +### Getting function help + +In an interactive Python session or Jupyter notebook, append `?` to any function name: + +```python +deeplabcut.nameofthefunction? +``` + +Or use the built-in `help()`: -Now, you can run any of the functions described in this documentation. +```python +help(deeplabcut.nameofthefunction) +``` -# Getting help with maDLC: +### Tips for daily use -- If you have a detailed question about how to use the code, or you hit errors that are not "bugs" but you want code assistance, please post on the [![Image.sc forum](https://img.shields.io/badge/dynamic/json.svg?label=forum&url=https%3A%2F%2Fforum.image.sc%2Ftags%2Fdeeplabcut.json&query=%24.topic_list.tags.0.topic_count&colorB=brightgreen&&suffix=%20topics&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA4AAAAOCAYAAAAfSC3RAAABPklEQVR42m3SyyqFURTA8Y2BER0TDyExZ+aSPIKUlPIITFzKeQWXwhBlQrmFgUzMMFLKZeguBu5y+//17dP3nc5vuPdee6299gohUYYaDGOyyACq4JmQVoFujOMR77hNfOAGM+hBOQqB9TjHD36xhAa04RCuuXeKOvwHVWIKL9jCK2bRiV284QgL8MwEjAneeo9VNOEaBhzALGtoRy02cIcWhE34jj5YxgW+E5Z4iTPkMYpPLCNY3hdOYEfNbKYdmNngZ1jyEzw7h7AIb3fRTQ95OAZ6yQpGYHMMtOTgouktYwxuXsHgWLLl+4x++Kx1FJrjLTagA77bTPvYgw1rRqY56e+w7GNYsqX6JfPwi7aR+Y5SA+BXtKIRfkfJAYgj14tpOF6+I46c4/cAM3UhM3JxyKsxiOIhH0IO6SH/A1Kb1WBeUjbkAAAAAElFTkSuQmCC)](https://forum.image.sc/tags/deeplabcut) +You can always exit a conda environment and pick up where you left off: -- If you have a quick, short question that fits a "chat" format: -[![Gitter](https://badges.gitter.im/DeepLabCut/community.svg)](https://gitter.im/DeepLabCut/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +Linux/macOS: -- If you want to share some results, or see others: -[![Twitter Follow](https://img.shields.io/twitter/follow/DeepLabCut.svg?label=DeepLabCut&style=social)](https://x.com/DeepLabCut) +```bash +source activate yourdeeplabcutEnvName +ipython +import deeplabcut +config_path = '/home/yourprojectfolder/config.yaml' +``` -- If you have a code bug report, please create an issue and show the minimal code to reproduce the error: https://github.com/DeepLabCut/DeepLabCut/issues +Windows: -- if you are looking for resources to increase your understanding of the software and general guidelines, we have an open source, free course: https://deeplabcut.github.io/DeepLabCut/docs/course.html. +```bash +activate yourdeeplabcutEnvName +ipython +import deeplabcut +config_path = r'C:\home\yourprojectfolder\config.yaml' +``` -**Please note:** what we cannot do is provided support or help designing your experiments and data analysis. The number of requests for this is too great to sustain in our inbox. We are happy to answer such questions in the forum as a community, in a scalable way. We hope and believe we have given enough tools and resources to get started and to accelerate your research program, and this is backed by the >700 citations using DLC, 2 clinical trials by others, and countless applications. Thus, we believe this code works, is accessible, and with limited programming knowledge can be used. Please read our [Missions & Values statement](mission-and-values) to learn more about what we DO hope to provide you. +### Getting help and support + +- **Forum** — for detailed usage questions or errors that are not bugs, please post on the + [Image.sc forum](https://forum.image.sc/tags/deeplabcut). +- **Chat** — for short questions: + [![Gitter](https://badges.gitter.im/DeepLabCut/community.svg)](https://gitter.im/DeepLabCut/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +- **Social** — share results or follow updates: + [![Twitter Follow](https://img.shields.io/twitter/follow/DeepLabCut.svg?label=DeepLabCut&style=social)](https://x.com/DeepLabCut) +- **Bug reports** — please open an issue with a minimal reproducible example: + + +```{note} +We are not able to provide individual support for experiment design or custom data +analysis — the volume of such requests is too large to sustain. We welcome these +discussions on the forum, where the community can benefit collectively. Please read our +[Missions & Values statement](mission-and-values) to learn more. +``` diff --git a/docs/main-workflows/multi-animal-tracking.md b/docs/main-workflows/multi-animal-tracking.md new file mode 100644 index 0000000000..8e638c3768 --- /dev/null +++ b/docs/main-workflows/multi-animal-tracking.md @@ -0,0 +1,229 @@ +--- +deeplabcut: + last_content_updated: '2025-06-30' + last_metadata_updated: '2026-03-06' + ignore: false + visibility: online +--- + +(file:multi-animal-tracking)= + +# Multi-animal tracking + +```{contents} +--- +local: +depth: 3 +--- +``` + +The workflow for multi-animal projects is fully aligned with single animal projects. However, for multi-animal projects, +the video analysis is slightly more complex: besides bare keypoint estimation, the keypoints need to be assigned to one +of the different individuals and coherenty tracked across frames. This guide zooms in on the details of the tracking +step in the video analysis for multi-animal projects. The full workflow is discussed in the +{ref}`user guide `. + +```{important} +From version 3.0 onward, `deeplabcut.analyze_videos` runs the **full pose estimation + tracking pipeline** by default (`auto_track=True`), producing an .h5 file ready for downstream use. + +In prior versions of DeepLabCut, pose estimation and tracking were separate procedures. This behavior can still be obtained by setting `auto_track=False`. With `auto_track=False`, no `.h5` file is produced — only a `*_full.pickle` file containing the raw detections. If `auto_track=False`, one must run `convert_detections2tracklets` and +`stitch_tracklets` manually (see below), granting more control over the last steps of +the workflow (ideal for advanced users). +``` + +## Visualization before tracking + +### Visualize raw keypoint detections (without tracking) + +To validate raw pose estimation performance on a video before committing to the tracking +results, run: + +```python +videos_to_analyze = ['/fullpath/project/videos/testVideo.mp4'] +deeplabcut.analyze_videos( + config_path, + videos_to_analyze, + auto_track=False +) +deeplabcut.create_video_with_all_detections( + config_path, + videos_to_analyze +) +``` + +### Visualizing part-affinity fields (PAFs) + +For models predicting part-affinity fields, another sanity check may be to +examine the distributions of edge affinity costs using `deeplabcut.utils.plot_edge_affinity_distributions`. Easily separable distributions +indicate that the model has learned strong links to group keypoints into distinct +individuals — likely a necessary feature for the assembly stage (note that the amount of +overlap will also depend on the amount of interactions between your animals in the +dataset). All TensorFlow multi-animal models use part-affinity fields and PyTorch models +consisting of just a backbone name (e.g. `resnet_50`, `resnet_101`) use part-affinity +fields. If you're unsure whether your PyTorch model has a one, check +the **pytorch_config.yaml** for a `DLCRNetHead`. + +````{tip} +If these results do not look good, we recommend extracting and labeling more frames (even from more videos). Try to label close interactions of animals for best performance. Once you label more, you can create a new training set and train. + +You can either: + +1. extract more frames manually from existing or new videos and label as when initially building the training data set, or +1. let DeepLabCut find frames where keypoints were poorly detected and automatically extract those for you. All you need is + to run: + +```python +deeplabcut.find_outliers_in_raw_data(config_path, pickle_file, video_file) +``` + +where pickle_file is the `_full.pickle` one obtains after video analysis. +Flagged frames will be added to your collection of images in the corresponding labeled-data folders for you to label. +```` + +## Manually run tracking steps + +### Animal Assembly and Tracking across frames + +After pose estimation, now you perform assembly and tracking. + +You can validate the tracking parameters. Namely, you can iteratively change the +parameters, run `convert_detections2tracklets` then load them in the GUI +(`refine_tracklets`) if you want to look at the performance. If you want to edit these, +you will need to open the `inference_cfg.yaml` file (or click button in GUI). The +options are: + +```python +# Tracking: +#p/m pixels in width and height for increasing bounding boxes. +boundingboxslack : 0 +# Intersection over Union (IoU) threshold for linking two bounding boxes +iou_threshold: .2 +# maximum duration of a lost tracklet before it's considered a "new animal" (in frames) +max_age: 100 +# minimum number of consecutive frames before a detection is tracked +min_hits: 3 +``` + +If the network was trained with identity supervision (i.e., `identity=True` in +`config.yaml` before training), this information can be leveraged during: (i) animal +assembly, where body parts are grouped by predicted identity rather than keypoint +affinity; and (ii) tracking, where identity alone can be used in place of motion +trackers to form tracklets. + +To use this ID information, simply pass: + +```python +deeplabcut.convert_detections2tracklets(..., identity_only=True) +``` + +- **Note:** If only one individual is to be assembled and tracked, assembly and tracking are skipped, and detections are treated as in single-animal projects; i.e., it is the keypoints with highest confidence that are kept and accumulated over frames to form a single, long tracklet. No action is required from users, this is done automatically. + +**Animal assembly and tracking quality** can be assessed via `deeplabcut.utils.make_labeled_video.create_video_from_pickled_tracks`. This function provides an additional diagnostic tool before moving on to refining tracklets. + +If animal assemblies do not look pretty, an alternative to the outlier search described above is to pass the +`_assemblies.pickle` to `find_outliers_in_raw_data` in place of the `_full.pickle`. +This will focus the outlier search on unusual assemblies (i.e., animal skeletons that were oddly reconstructed). This may be a bit more sensitive with crowded scenes or frames where animals interact closely. +Note though that at that stage it is likely preferable anyway to carry on with the remaining steps, and extract outliers +from the final h5 file as was customary in single animal projects. + +\*\*Next, tracklets are stitched to form complete tracks with: + +```python +deeplabcut.stitch_tracklets( + config_path, + ['videofile_path'], + video_extensions='mp4', + shuffle=1, + trainingsetindex=0, +) +``` + +Note that the base signature of the function is identical to `analyze_videos` and `convert_detections2tracklets`. +If the number of tracks to reconstruct is different from the number of individuals +originally defined in the config.yaml, `n_tracks` (i.e., the number of animals you have in your video) +can be directly specified as follows: + +```python +deeplabcut.stitch_tracklets(..., n_tracks=n) +``` + +In such cases, file columns will default to dummy animal names (ind1, ind2, ..., up to indn). + +##### API Docs + +````{admonition} Click the button to see API Docs for analyze_videos +--- +class: dropdown +--- +```{eval-rst} +.. include:: ./api/deeplabcut.analyze_videos.rst +``` +```` + +````{admonition} Click the button to see API Docs for convert_detections2tracklets +--- +class: dropdown +--- +```{eval-rst} +.. include:: ./api/deeplabcut.convert_detections2tracklets.rst +``` +```` + +````{admonition} Click the button to see API Docs for stitch_tracklets +--- +class: dropdown +--- +```{eval-rst} +.. include:: ./api/deeplabcut.stitch_tracklets.rst +``` +```` + +##### Using Unsupervised Identity Tracking: + +In Lauer et al. 2022 we introduced a new method to do unsupervised reID of animals. +Here, you can use the tracklets to learn the identity of animals to enhance your +tracking performance. To use the code: + +```python +deeplabcut.transformer_reID(config, videos_to_analyze, n_tracks=None, video_extensions="mp4") +``` + +Note you should pass the n_tracks (number of animals) you expect to see in the video. + +##### Refine Tracklets: + +You can also optionally **refine the tracklets**. You can fix both "major" ID swaps, i.e. perhaps when animals cross, and you can micro-refine the individual body points. You will load the `...trackertype.pickle` or `.h5'` file that was created above, and then you can launch a GUI to interactively refine the data. This also has several options, so please check out the docstring. Upon saving the refined tracks you get an `.h5` file (akin to what you might be used to from standard DLC. You can also load (1) filter this to take care of small jitters, and (2) load this `.h5` this to refine (again) in case you find another issue, etc! + +```python +deeplabcut.refine_tracklets(config_path, pickle_or_h5_file, videofile_path, max_gap=0, min_swap_len=2, min_tracklet_len=2, trail_len=50) +``` + +If you use the GUI (or otherwise), here are some settings to consider: + +```{figure} https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1619628014395-BQ09VLLTKCLQQGRB5T9A/ke17ZwdGBToddI8pDm48kLMj_XrWI9gi4tVeBdgcB8p7gQa3H78H3Y0txjaiv_0fDoOvxcdMmMKkDsyUqMSsMWxHk725yiiHCCLfrh8O1z4YTzHvnKhyp6Da-NYroOW3ZGjoBKy3azqku80C789l0lt53wR20brczws2A6XSGt3kSTbW7uM0ncVKHWPvgHR4kN5Ka1TcK96ljy4ji9jPkQ/TrackletGUI.png?format=1000w +--- +name: fig-tracklet-gui +alt: Tracklet refinement GUI showing key settings +width: 950px +align: center +--- +Tracklet refinement GUI. Key settings to configure are described in the text. +``` + +\*note, setting `max_gap=0` can be used to fill in all frames across the video; otherwise, 1-n is the # of frames you want to fill in, i.e. maybe you want to fill in short gaps of 5 frames, but 15 frames indicates another issue, etc. You can test this in the GUI very easy by editing the value and then re-launch pop-up GUI. + +If you fill in gaps, they will be associated to an ultra low probability, 0.01, so you are aware this is not the networks best estimate, this is the human-override! Thus, if you create a video, you need to set your pcutoff to 0 if you want to see these filled in frames. + +[Read more here!](functionDetails.md#madeeplabcut-critical-point---assemble--refine-tracklets) + +Short demo: + +```{figure} https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1588690928000-90ZMRIM8SN6QE20ZOMNX/ke17ZwdGBToddI8pDm48kJ1oJoOIxBAgRD2ClXVCmKFZw-zPPgdn4jUwVcJE1ZvWQUxwkmyExglNqGp0IvTJZUJFbgE-7XRK3dMEBRBhUpxBw7VlGKDQO2xTcc51Yv6DahHgScLwHgvMZoEtbzk_9vMJY_JknNFgVzVQ2g0FD_s/refineDEMO.gif?format=750w +--- +name: fig-refine-tracklets-demo +alt: Animated demonstration of the tracklet refinement workflow +width: 70% +align: center +--- +Short demo of the tracklet refinement workflow. +``` diff --git a/docs/main-workflows/user-guide.md b/docs/main-workflows/user-guide.md new file mode 100644 index 0000000000..de1705d2e8 --- /dev/null +++ b/docs/main-workflows/user-guide.md @@ -0,0 +1,1711 @@ +--- +deeplabcut: + last_content_updated: '2025-06-30' + last_metadata_updated: '2026-03-06' + ignore: false + visibility: online +--- + +(file:dlc-userguide)= + +# DeepLabCut User Guide + +```{contents} +--- +local: +depth: 3 +--- +``` + +This guide covers the standard single-animal and multi-animal 2D pose estimation projects. + +## Getting started + +DeepLabCut offers two equivalent interfaces: a **GUI** for those who prefer a visual +workflow (no Python knowledge required), and a **Python API** for users who want +scripting flexibility or to integrate DeepLabCut into a larger pipeline. All workflow +steps are available in both. + +We assume you have DeepLabCut installed (if not, see {ref}`file:how-to-install`). +Open a terminal and activate your conda environment: + +```bash +conda activate DEEPLABCUT +``` + +```{important} +On Windows, always open the terminal with administrator privileges: right-click and +select "Run as administrator". +``` + +Choose your interface below to launch DeepLabCut: + +### GUI (recommended for beginners) + +```bash +python -m deeplabcut +``` + +```{figure} https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1572824438905-QY9XQKZ8LAJZG6BLPWOQ/ke17ZwdGBToddI8pDm48kIIa76w436aRzIF_cdFnEbEUqsxRUqqbr1mOJYKfIPR7LoDQ9mXPOjoJoqy81S2I8N_N4V1vUb5AoIIIbLZhVYxCRW4BPu10St3TBAUQYVKcLthF_aOEGVRewCT7qiippiAuU5PSJ9SSYal26FEts0MmqyMIhpMOn8vJAUvOV4MI/guilaunch.jpg?format=1000w +--- +name: fig-gui-launch +alt: The DeepLabCut Project Manager GUI after launch +width: 60% +align: center +--- +The DeepLabCut Project Manager GUI. +``` + +### Python API + +In an interactive Python session (e.g. `ipython`), import DeepLabCut: + +```python +import deeplabcut +``` + +As a reminder, the core functions are described in our +[Nature Protocols paper](https://www.nature.com/articles/s41596-019-0176-0) (published +at the time of DeepLabCut version 2.0.6). Additional functions and features are +continually added to the package; we recommend reading the protocol alongside this +documentation. + +## Workflow + +DeepLabCut's full workflow is described in steps (A)–(N) below. Code examples throughout this page use the Python API; if you are +using the GUI, the same steps are available in the corresponding panels of the +Project Manager. + +You should think of the workflow as 5 phases + +```{figure} ../images/dlc-workflow.png +--- +name: dlc-workflow-figure +alt: The 5 phases of the DeepLabCut workflow. +align: center +--- +The 5 phases of the DeepLabCut workflow. The expected outputs are indicated in the grey boxes. +``` + +1. Project setup: create and configure your new project. +1. Data preparation: select frames and annotate your training data. +1. Training and evaluation: configure, train and evaluate your neural network model. +1. Analysis: run inference with your trained model to create predictions and labeled videos. +1. Refinement (optional): improve your data quality for a next training iteration. + +```{admonition} Automated multi-animal tracking +--- +class-container: multi-animal +--- + For multi-animal projects, the video-analysis step contains an automated tracking step. More information is available in the {ref}`multi-animal tracking guide `. +``` + +### Phase 1 — Project setup + +#### (A) Create a New Project + +##### Overview + +The function `create_new_project` creates a new project directory, required subdirectories, and a basic project +configuration file. Each project is identified by the name of the project (e.g. Reaching), name of the experimenter +(e.g. YourName), as well as the date at creation. + +Thus, this function requires the user to input: + +- The name of the project +- The name of the experimenter +- The full path of the videos that are (initially) used to create the training dataset. +- Optional arguments specify: + - The working directory + - Where the project directory will be created + - **Recommended**: Whether to copy the videos to the project directory + - Whether to create a single- or multi-animal project + +```{note} +If the optional argument `working_directory` is unspecified, the +project directory is created in the current working directory. + +If `copy_videos` is unspecified symbolic links +for the videos are created in the videos directory. +Each symbolic link creates a reference to a video and thus +eliminates the need to copy the entire video to the video directory (if the videos remain at the original location). +This is why administrator privileges are required for Windows users, as creating symbolic links requires them. +``` + +##### Code example + +````{dropdown} +--- +class-container: single-animal +open: +--- +```python +deeplabcut.create_new_project( + "Name of the project", + "Name of the experimenter", + ["Full path of video 1", "Full path of video 2", "Full path of video 3"], + working_directory="Full path of the working directory", + copy_videos=True, + multianimal=False +) +``` +```` + +````{dropdown} +--- +class-container: multi-animal +open: +--- +```python +deeplabcut.create_new_project( + "Name of the project", + "Name of the experimenter", + ["Full path of video 1", "Full path of video 2", "Full path of video 3"], + working_directory="Full path of the working directory", + copy_videos=True, + multianimal=True +) +``` +```` + +###### Output & directory structure + +```{important} +On Windows, input paths as: +`r'C:\Users\computername\Videos\reachingvideo1.avi'` or +`'C:\\Users\\computername\\Videos\\reachingvideo1.avi'` +``` + +```{tip} +You can also place `config_path` in front of `deeplabcut.create_new_project` to create a variable that holds +the path to the config.yaml file, i.e. `config_path=deeplabcut.create_new_project(...)` +``` + +This set of arguments creates a project directory with the name: + +**`++`** + +in the **working directory** and creates the video copies in the videos directory. + +The project directory will have subdirectories: + +``` +++/ +├── dlc-models/ +│ ├── iteration-0/ +│ ├── iteration-1/ +│ └── ... +├── dlc-models-pytorch/ +│ ├── iteration-0/ +│ │ └── / +│ │ ├── train/ +│ │ └── test/ +│ ├── iteration-1/ +│ └── ... +├── labeled-data/ +│ └──