completion: autogenerate (#3921)

* cli: split main_parser

* cli: basic parser tree printing

* cli: major completion tidy

* completion: more missing root commands

* completion: much automagic such beauty

- fixes #2985 automatically generate bash completion

* tests: bash completion

* tests: completion: tidy script a little

* minor tidy

* command: positional name consistency

* completion: bash: add logging

* double-check renames and consistency

* cli: add completion subcommand

* ci: completion: update test

* minor logging

* more completion logging verbosity

* fix completion of files

* add shtab

* purge unneeded completion => shtab

* completion: neaten UI

* completion: use `shtab>=0.0.2`

* completion: purge hardcoded completions!

* revert unneeded renames for now

* fix isort

* completion: tidy CLI API

* fix snap build

* fix snap build again

* snap: internally build completion

* allow repo-less completion

* command: tidy headless logic

* use CmdBaseNoRepo

* better DVC-file completion

* merge command.choices -> command.completion

* drop choices class

* remove completion optionals

* tidy docs

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,6 @@
 * [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
 
-* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here. If the CLI API is changed, I have updated [tab completion scripts](https://github.com/iterative/dvc/tree/master/scripts/completion).
+* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
 
 * [ ] ❌ I will check DeepSource, CodeClimate, and other sanity checks below. (We consider them recommendatory and don't expect everything to be addressed. Please fix things that actually improve code or fix bugs.)
 

dvc/cli.py

@@ -8,6 +8,7 @@ from .command import (
     cache,
     checkout,
     commit,
+    completion,
     config,
     daemon,
     dag,
@@ -70,6 +71,7 @@ COMMANDS = [
     dag,
     daemon,
     commit,
+    completion,
     diff,
     version,
     update,
@@ -140,15 +142,7 @@ def get_parent_parser():
     return parent_parser
 
 
-def parse_args(argv=None):
-    """Parses CLI arguments.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Raises:
-        dvc.exceptions.DvcParserError: raised for argument parsing errors.
-    """
+def get_main_parser():
     parent_parser = get_parent_parser()
 
     # Main parser
@@ -196,6 +190,18 @@ def parse_args(argv=None):
     for cmd in COMMANDS:
         cmd.add_parser(subparsers, parent_parser)
 
-    args = parser.parse_args(argv)
+    return parser
 
+
+def parse_args(argv=None):
+    """Parses CLI arguments.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Raises:
+        dvc.exceptions.DvcParserError: raised for argument parsing errors.
+    """
+    parser = get_main_parser()
+    args = parser.parse_args(argv)
     return args

dvc/command/add.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
@@ -62,6 +63,9 @@ def add_parser(subparsers, parent_parser):
         metavar="<filename>",
     )
     parser.add_argument(
-        "targets", nargs="+", help="Input files/directories to add."
+        "targets",
+        nargs="+",
+        help="Input files/directories to add.",
+        choices=completion.Required.FILE,
     )
     parser.set_defaults(func=CmdAdd)

dvc/command/cache.py

@@ -1,5 +1,6 @@
 import argparse
 
+from dvc.command import completion
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
 
@@ -55,5 +56,6 @@ def add_parser(subparsers, parent_parser):
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
         "config file location.",
+        choices=completion.Required.DIR,
     )
     cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/checkout.py

@@ -4,6 +4,7 @@ import operator
 
 import colorama
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import CheckoutError
 from dvc.utils.humanize import get_summary
@@ -113,5 +114,6 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="DVC-files to checkout. Optional. "
         "(Finds all DVC-files in the workspace by default.)",
+        choices=completion.Optional.DVC_FILE,
     )
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -66,5 +67,6 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="DVC-files to commit. Optional. "
         "(Finds all DVC-files in the workspace by default.)",
+        choices=completion.Optional.DVC_FILE,
     )
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -0,0 +1,68 @@
+import argparse
+import logging
+
+import shtab
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+
+logger = logging.getLogger(__name__)
+CHOICE_FUNCTIONS = {
+    "bash": {"DVCFile": "_dvc_compgen_DVCFiles"},
+    "zsh": {"DVCFile": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'"},
+}
+PREAMBLE = {
+    "bash": """
+# $1=COMP_WORDS[1]
+_dvc_compgen_DVCFiles() {
+  compgen -d -S '/' -- $1  # recurse into subdirs
+  compgen -f -X '!*?.dvc' -- $1
+  compgen -f -X '!*Dvcfile' -- $1
+  compgen -f -X '!*dvc.yaml' -- $1
+}
+""",
+    "zsh": "",
+}
+
+
+class Optional(shtab.Optional):
+    DVC_FILE = [shtab.Choice("DVCFile", required=False)]
+
+
+class Required(shtab.Required):
+    DVC_FILE = [shtab.Choice("DVCFile", required=True)]
+
+
+class CmdCompletion(CmdBaseNoRepo):
+    def run(self):
+        from dvc.cli import get_main_parser
+
+        parser = get_main_parser()
+        shell = self.args.shell
+        script = shtab.complete(
+            parser,
+            shell=shell,
+            preamble=PREAMBLE[shell],
+            choice_functions=CHOICE_FUNCTIONS[shell],
+        )
+        print(script)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    COMPLETION_HELP = "Generate shell tab completion."
+    COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
+    completion_parser = subparsers.add_parser(
+        "completion",
+        parents=[parent_parser],
+        description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
+        help=COMPLETION_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    completion_parser.add_argument(
+        "-s",
+        "--shell",
+        help="Shell syntax for completions.",
+        default="bash",
+        choices=["bash", "zsh"],
+    )
+    completion_parser.set_defaults(func=CmdCompletion)

dvc/command/daemon.py

@@ -1,3 +1,4 @@
+from dvc.command import completion
 from dvc.command.base import CmdBaseNoRepo, fix_subparsers
 
 
@@ -64,5 +65,7 @@ def add_parser(subparsers, parent_parser):
         description=DAEMON_ANALYTICS_HELP,
         help=DAEMON_ANALYTICS_HELP,
     )
-    daemon_analytics_parser.add_argument("target", help="Analytics file.")
+    daemon_analytics_parser.add_argument(
+        "target", help="Analytics file.", choices=completion.Required.FILE
+    )
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/data_sync.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.command.checkout import log_changes
 from dvc.exceptions import CheckoutError, DvcException
@@ -104,6 +105,7 @@ def shared_parent_parser():
         nargs="*",
         help="Limit command scope to these DVC-files. "
         "Using -R, directories to search DVC-files in can also be given.",
+        choices=completion.Optional.DVC_FILE,
     )
 
     return parent_parser

dvc/command/freeze.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -39,7 +40,10 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     freeze_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to freeze."
+        "targets",
+        nargs="+",
+        help="DVC-files to freeze.",
+        choices=completion.Required.DVC_FILE,
     )
     freeze_parser.set_defaults(func=CmdFreeze)
 
@@ -52,6 +56,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unfreeze_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to unfreeze."
+        "targets",
+        nargs="+",
+        help="DVC-files to unfreeze.",
+        choices=completion.Required.DVC_FILE,
     )
     unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/get.py

@@ -3,6 +3,7 @@ import logging
 
 from dvc.exceptions import DvcException
 
+from . import completion
 from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
@@ -62,7 +63,9 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path", help="Path to a file or directory within the repository"
+        "path",
+        help="Path to a file or directory within the repository",
+        choices=completion.Required.FILE,
     )
     get_parser.add_argument(
         "-o",
@@ -70,6 +73,7 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
+        choices=completion.Optional.DIR,
     )
     get_parser.add_argument(
         "--rev",

dvc/command/get_url.py

@@ -3,6 +3,7 @@ import logging
 
 from dvc.exceptions import DvcException
 
+from . import completion
 from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
@@ -33,6 +34,9 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out", nargs="?", help="Destination path to put data to."
+        "out",
+        nargs="?",
+        help="Destination path to put data to.",
+        choices=completion.Optional.DIR,
     )
     get_parser.set_defaults(func=CmdGetUrl)

dvc/command/imp.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -43,7 +44,9 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path", help="Path to a file or directory within the repository"
+        "path",
+        help="Path to a file or directory within the repository",
+        choices=completion.Required.FILE,
     )
     import_parser.add_argument(
         "-o",
@@ -51,6 +54,7 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
+        choices=completion.Optional.DIR,
     )
     import_parser.add_argument(
         "--rev",

dvc/command/imp_url.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -54,11 +55,15 @@ def add_parser(subparsers, parent_parser):
         "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to."
+        "out",
+        nargs="?",
+        help="Destination path to put files to.",
+        choices=completion.Optional.DIR,
     )
     import_parser.add_argument(
         "--file",
         help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
+        choices=completion.Optional.DIR,
     )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/command/ls/__init__.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 import sys
 
+from dvc.command import completion
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.command.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
@@ -74,5 +75,6 @@ def add_parser(subparsers, parent_parser):
         "path",
         nargs="?",
         help="Path to directory within the repository to list outputs for",
+        choices=completion.Optional.DIR,
     )
     list_parser.set_defaults(func=CmdList)

dvc/command/metrics.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import BadMetricError, DvcException
 
@@ -169,6 +170,7 @@ def add_parser(subparsers, parent_parser):
             "Limit command scope to these metric files. Using -R, "
             "directories to search metric files in can also be given."
         ),
+        choices=completion.Optional.FILE,
     )
     metrics_show_parser.add_argument(
         "-a",
@@ -235,6 +237,7 @@ def add_parser(subparsers, parent_parser):
             "directories to search metric files in can also be given."
         ),
         metavar="<paths>",
+        choices=completion.Optional.FILE,
     )
     metrics_diff_parser.add_argument(
         "-R",

dvc/command/move.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -36,7 +37,11 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src", help="Source path to a data file or directory."
+        "src",
+        help="Source path to a data file or directory.",
+        choices=completion.Required.FILE,
+    )
+    move_parser.add_argument(
+        "dst", help="Destination path.", choices=completion.Required.FILE,
     )
-    move_parser.add_argument("dst", help="Destination path.")
     move_parser.set_defaults(func=CmdMove)

dvc/command/plots.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 import os
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
 from dvc.schema import PLOT_PROPS
@@ -128,6 +129,7 @@ def add_parser(subparsers, parent_parser):
         "targets",
         nargs="*",
         help="Plots files to visualize. Shows all plots by default.",
+        choices=completion.Optional.FILE,
     )
     _add_props_arguments(plots_show_parser)
     _add_output_arguments(plots_show_parser)
@@ -149,6 +151,7 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="Plots file to visualize. Shows all plots by default.",
         metavar="<path>",
+        choices=completion.Optional.FILE,
     )
     plots_diff_parser.add_argument(
         "revisions", nargs="*", default=None, help="Git commits to plot from",
@@ -166,7 +169,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_modify_parser.add_argument(
-        "target", help="Metric file to set properties to",
+        "target",
+        help="Metric file to set properties to",
+        choices=completion.Required.FILE,
     )
     _add_props_arguments(plots_modify_parser)
     plots_modify_parser.add_argument(
@@ -191,6 +196,7 @@ def _add_props_arguments(parser):
             )
         ),
         metavar="<path>",
+        choices=completion.Optional.FILE,
     )
     parser.add_argument(
         "-x", default=None, help="Field name for X axis.", metavar="<field>"
@@ -223,6 +229,7 @@ def _add_output_arguments(parser):
         default=None,
         help="Destination path to save plots to",
         metavar="<path>",
+        choices=completion.Optional.DIR,
     )
     parser.add_argument(
         "--show-vega",

dvc/command/remove.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -34,6 +35,9 @@ def add_parser(subparsers, parent_parser):
         help="Remove outputs as well.",
     )
     remove_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to remove."
+        "targets",
+        nargs="+",
+        help="DVC-files to remove.",
+        choices=completion.Required.DVC_FILE,
     )
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 import os
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
@@ -71,6 +72,7 @@ def add_parser(subparsers, parent_parser):
         "targets",
         nargs="*",
         help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
+        choices=completion.Optional.DVC_FILE,
     )
     repro_parser.add_argument(
         "-f",

dvc/command/unprotect.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -32,6 +33,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets", nargs="+", help="Data files/directories to unprotect."
+        "targets",
+        nargs="+",
+        help="Data files/directories to unprotect.",
+        choices=completion.Required.FILE,
     )
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -32,7 +33,10 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to update."
+        "targets",
+        nargs="+",
+        help="DVC-files to update.",
+        choices=completion.Required.DVC_FILE,
     )
     update_parser.add_argument(
         "--rev",

scripts/build_posix.sh

@@ -115,10 +115,10 @@ build_dvc() {
   # [1] https://github.com/iterative/dvc/issues/2585
   if [[ "$(uname)" == 'Linux' ]]; then
     mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
-    cp scripts/completion/dvc.bash $BUILD_DIR/$BASH_CMPLT_DIR/dvc
+    $LIB_DIR/dvc/dvc completion -s bash > $BUILD_DIR/$BASH_CMPLT_DIR/dvc
 
     mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
-    cp scripts/completion/dvc.zsh $BUILD_DIR/$ZSH_CMPLT_DIR
+    $LIB_DIR/dvc/dvc completion -s zsh > $BUILD_DIR/$ZSH_CMPLT_DIR/_dvc
   fi
 }
 

scripts/completion/dvc.bash

@@ -1,146 +0,0 @@
-#!/usr/bin/env bash
-# References:
-#   - https://www.gnu.org/software/bash/manual/html_node/Programmable-Completion.html
-#   - https://opensource.com/article/18/3/creating-bash-completion-script
-#   - https://stackoverflow.com/questions/12933362
-
-_dvc_commands='add cache checkout commit config dag destroy diff fetch freeze \
-  get-url get gc import-url import init install list metrics move params \
-  plots pull push remote remove repro root run status unfreeze unprotect \
-  update version'
-
-_dvc_options='-h --help -V --version'
-_dvc_global_options='-h --help -q --quiet -v --verbose'
-
-_dvc_add='-R --recursive -f --file --no-commit --external'
-_dvc_add_COMPGEN=_dvc_compgen_files
-_dvc_cache='dir'
-_dvc_cache_dir=' --global --system --local -u --unset'
-_dvc_checkout='-d --with-deps -R --recursive -f --force --relink --summary'
-_dvc_checkout_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_commit='-f --force -d --with-deps -R --recursive'
-_dvc_commit_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_config='-u --unset --local --system --global'
-_dvc_dag='--dot --full'
-_dvc_dag_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_destroy='-f --force'
-_dvc_diff='-t --show-json --show-hash --show-md'
-_dvc_fetch='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
-_dvc_fetch_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_gc='-a --all-branches --all-commits -T --all-tags -w --workspace -c --cloud -r --remote -f --force -p --projects -j --jobs'
-_dvc_get='-o --out --rev --show-url'
-_dvc_get_url=''
-_dvc_import='-o --out --rev'
-_dvc_import_url='-f --file'
-_dvc_init='--no-scm -f --force'
-_dvc_install=''
-_dvc_list='-R --recursive --dvc-only --rev'
-_dvc_list_COMPGEN=_dvc_compgen_files
-_dvc_freeze=''
-_dvc_freeze_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_metrics='diff show'
-_dvc_metrics_diff='--targets -t --type -x --xpath -R --show-json --show-md --no-path --old'
-_dvc_metrics_show='-t --type -x --xpath -a --all-branches -T --all-tags -R --recursive'
-_dvc_metrics_show_COMPGEN=_dvc_compgen_files
-_dvc_move=''
-_dvc_move_COMPGEN=_dvc_compgen_files
-_dvc_params='diff'
-_dvc_params_diff='--all --show-json --show-md --no-path'
-_dvc_plots='show diff'
-_dvc_plots_show='-t --template -o --out -x -y --show-vega --no-csv-header --title --x-label --y-label'
-_dvc_plots_diff='-t --template --targets -o --out -x -y --show-vega --no-csv-header --title --x-label --y-label'
-_dvc_plots_modify='-t --template -x -y --no-csv-header --title --x-label --y-label'
-_dvc_pull='-j --jobs -r --remote -a --all-branches -T --all-tags -f --force -d --with-deps -R --recursive'
-_dvc_pull_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_push='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
-_dvc_push_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_remote='add default list modify remove rename'
-_dvc_remote_add='--global --system --local -d --default -f --force'
-_dvc_remote_default='--global --system --local -u --unset'
-_dvc_remote_list='--global --system --local'
-_dvc_remote_modify='--global --system --local -u --unset'
-_dvc_remote_remove='--global --system --local'
-_dvc_remote_rename='--global --system --local'
-_dvc_remove='-o --outs -p --purge -f --force'
-_dvc_remove_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --no-run-cache --force-downstream --no-commit -R --recursive --downstream'
-_dvc_repro_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_root=''
-_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --no-run-cache --no-commit -w --wdir --external'
-_dvc_run_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud'
-_dvc_status_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_unfreeze_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_unprotect_COMPGEN=_dvc_compgen_files
-_dvc_update='--rev'
-_dvc_update_COMPGEN=_dvc_compgen_DVCFiles
-_dvc_version=''
-
-# $1=COMP_WORDS[1]
-_dvc_compgen_DVCFiles() {
-  compgen -f -X '!*?.dvc' -- $1
-  compgen -d -S '/' -- $1  # recurse into subdirs
-  # Note that the recurse into dirs is only for looking for DVC-files.
-  # Since dirs themselves are not required, we need `-o nospace` at the bottom
-  # unfortunately :(
-}
-
-# $1=COMP_WORDS[1]
-_dvc_compgen_files() {
-  compgen -f -- $1
-  compgen -d -S '/' -- $1  # recurse into subdirs
-}
-
-# $1=COMP_WORDS[1]
-_dvc_replace_hyphen() {
-  echo $1 | sed 's/-/_/g'
-}
-
-# $1=COMP_WORDS[1]
-_dvc_compgen_command() {
-  local flags_list="_dvc_$(_dvc_replace_hyphen $1)"
-  local args_gen="${flags_list}_COMPGEN"
-  COMPREPLY=( $(compgen -W "$_dvc_global_options ${!flags_list}" -- "$word"; [ -n "${!args_gen}" ] && ${!args_gen} "$word") )
-}
-
-# $1=COMP_WORDS[1]
-# $2=COMP_WORDS[2]
-_dvc_compgen_subcommand() {
-  local flags_list="_dvc_$(_dvc_replace_hyphen $1)_$(_dvc_replace_hyphen $2)"
-  local args_gen="${flags_list}_COMPGEN"
-  [ -n "${!args_gen}" ] && local opts_more="$(${!args_gen} "$word")"
-  local opts="${!flags_list}"
-  if [ -z "$opts$opts_more" ]; then
-    _dvc_compgen_command $1
-  else
-    COMPREPLY=( $(compgen -W "$_dvc_global_options $opts" -- "$word"; [ -n "$opts_more" ] && echo "$opts_more") )
-  fi
-}
-
-# Notes:
-# `COMPREPLY` contains what will be rendered after completion is triggered
-# `word` refers to the current typed word
-# `${!var}` is to evaluate the content of `var` and expand its content as a variable
-#       hello="world"
-#       x="hello"
-#       ${!x} ->  ${hello} ->  "world"
-_dvc() {
-  local word="${COMP_WORDS[COMP_CWORD]}"
-
-  COMPREPLY=()
-
-  if [ "${COMP_CWORD}" -eq 1 ]; then
-    case "$word" in
-      -*) COMPREPLY=($(compgen -W "$_dvc_options" -- "$word")) ;;
-      *) COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
-    esac
-  elif [ "${COMP_CWORD}" -eq 2 ]; then
-    _dvc_compgen_command ${COMP_WORDS[1]}
-  elif [ "${COMP_CWORD}" -ge 3 ]; then
-    _dvc_compgen_subcommand ${COMP_WORDS[1]} ${COMP_WORDS[2]}
-  fi
-
-  return 0
-}
-
-complete -o nospace -F _dvc dvc

scripts/completion/dvc.zsh

@@ -1,340 +0,0 @@
-#compdef dvc
-
-#----------------------------------------------------------
-# Repository:  https://github.com/iterative/dvc
-#
-# References:
-#   - https://github.com/zsh-users/zsh-completions
-#   - http://zsh.sourceforge.net/Doc/Release/Completion-System.html
-#   - https://mads-hartmann.com/2017/08/06/writing-zsh-completion-scripts.html
-#   - http://www.linux-mag.com/id/1106/
-#----------------------------------------------------------
-
-_dvc_commands() {
-  local _commands=(
-    "add:Track data files or directories with DVC."
-    "cache:Manage cache settings."
-    "checkout:Checkout data files from cache."
-    "commit:Save changed data to cache and update DVC-files."
-    "config:Get or set config settings."
-    "dag:Visualize DVC project DAG."
-    "destroy:Remove DVC-files, local DVC config and data cache."
-    "diff:Show added, modified, or deleted data between commits in the DVC repository, or between a commit and the workspace."
-    "fetch:Get files or directories tracked by DVC from remote storage into the cache."
-    "get-url:Download or copy files from URL."
-    "get:Download data from DVC repository."
-    "gc:Garbage collect unused objects from cache or remote storage."
-    "import-url:Download or copy file from URL and take it under DVC control."
-    "import:Download data from DVC repository and take it under DVC control."
-    "init:Initialize DVC in the current directory."
-    "install:Install DVC git hooks into the repository."
-    "list:List repository contents, including files and directories tracked by DVC and by Git."
-    "lock:Lock DVC-file."
-    "metrics:Commands to display and compare metrics."
-    "move:Rename or move a DVC controlled data file or a directory."
-    "params:Commands to display params."
-    "pull:Pull data files from a DVC remote storage."
-    "push:Push data files to a DVC remote storage."
-    "plots:Generate plot for metrics structured as JSON, CSV or TSV."
-    "remote:Manage remote storage configuration."
-    "remove:Remove outputs of DVC-file."
-    "repro:Check for changes and reproduce DVC-file and dependencies."
-    "root:Relative path to project's directory."
-    "run:Generate a stage file from a command and execute the command."
-    "status:Show changed stages, compare local cache and a remote storage."
-    "unlock:Unlock DVC-file."
-    "unprotect:Unprotect data file/directory."
-    "update:Update data artifacts imported from other DVC repositories."
-    "version:Show DVC version and system/environment information."
-  )
-
-  _describe 'dvc commands' _commands
-}
-
-_dvc_options=(
-  "(-)"{-h,--help}"[Show help message.]"
-  "(-)"{-V,--version}"[Show program's version]"
-)
-
-_dvc_global_options=(
-  "(-)"{-h,--help}"[Show help message related to the command.]"
-  "(-)"{-q,--quiet}"[Be quiet.]"
-  "(-)"{-v,--verbose}"[Be verbose.]"
-)
-
-_dvc_add=(
-  {-R,--recursive}"[Recursively add each file under the directory.]"
-  "--no-commit[Don't put files/directories into cache.]"
-  {-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
-  "--external[Allow targets that are outside of the DVC project.]"
-  "1:File:_files"
-)
-
-_dvc_cache=(
-  "1:Sub command:(dir)"
-)
-
-_dvc_checkout=(
-  {-d,--with-deps}"[Checkout all dependencies of the specified target.]"
-  {-R,--recursive}"[Checkout all subdirectories of the specified directory.]"
-  {-f,--force}"[Do not prompt when removing working directory files.]"
-  "--relink[Recreate links or copies from cache to workspace.]"
-  "--summary[Show summary of the changes.]"
-  "1:Stages:_files -g '(*.dvc|Dvcfile)'"
-)
-
-_dvc_commit=(
-  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-  {-f,--force}"[Commit even if checksums for dependencies/outputs changed.]"
-  {-d,--with-deps}"[Commit all dependencies of the specified target.]"
-  {-R,--recursive}"[Commit cache for subdirectories of the specified directory.]"
-)
-
-_dvc_config=(
-  "--global[Use global config.]"
-  "--system[Use system config.]"
-  "--local[Use local config.]"
-  {-u,--unset}"[Unset option.]"
-)
-
-_dvc_dag=(
-  "--dot[Print DAG in DOT format.]"
-  "--full[Show full DAG that the target belongs too, instead of showing DAG consisting only of ancestors.]"
-  "1:Stage:"
-)
-
-_dvc_destroy=(
-  {-f,--force}"[Force destruction.]"
-)
-
-_dvc_diff=(
-  "--show-json[Format the output into a JSON]"
-  "--show-hash[Display hash value for each entry]"
-  "--show-md[Format the output into a Markdown table]"
-  "1:Old Git commit to compare (defaults to HEAD):"
-  "2:New Git commit to compare (defaults to the current workspace):"
-)
-
-_dvc_fetch=(
-  {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
-  {-r,--remote}"[Remote repository to fetch from.]:Remote repository:"
-  {-a,--all-branches}"[Fetch cache for all branches.]"
-  {-T,--all-tags}"[Fetch cache for all tags.]"
-  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]"
-  {-R,--recursive}"[Fetch cache for subdirectories of specified directory.]"
-  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-)
-
-_dvc_geturl=(
-  "1:URL:"
-  "2:Output:"
-)
-
-_dvc_get=(
-  {-o,--out}"[Destination path to put data to.]:OUT:_files -/"
-  "--rev[Git revision (e.g. SHA, branch, tag)]:Revision:"
-  "--show-url[Returns path/url to the location in remote for given path]"
-  "1:URL:"
-  "2:Path:"
-)
-
-_dvc_gc=(
-  {-w,--workspace}"[Keep data files used in the current workspace.]"
-  {-a,--all-branches}"[Keep data files for the tips of all Git branches.]"
-  "--all-commits[Keep data files for all Git commits.]"
-  {-T,--all-tags}"[Keep data files for all Git tags.]"
-  {-c,--cloud}"[Collect garbage in remote repository.]"
-  {-r,--remote}"[Remote storage to collect garbage in.]:Remote repository:"
-  {-f,--force}"[Force garbage collection - automatically agree to all prompts.]:Repos:_files"
-  {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
-  {-p,--projects}"[Keep data files required by these projects in addition to the current one.]:Repos:_files"
-)
-
-_dvc_importurl=(
-  {-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
-  "1:URL:"
-  "2:Output:"
-)
-
-_dvc_import=(
-  {-o,--out}"[Destination path to put data to.]:OUT:_files -/"
-  "--rev[Git revision (e.g. SHA, branch, tag)]:Commit hash:"
-  "1:URL:"
-  "2:Path:"
-)
-
-_dvc_init=(
-  "--no-scm[Initiate dvc in directory that is not tracked by any scm tool.]"
-  {-f,--force}"[Overwrite existing '.dvc' directory. This operation removes local cache.]"
-)
-
-_dvc_install=()
-
-_dvc_freeze=(
-  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-)
-
-_dvc_list=(
-  "--rev[Git revision (e.g. branch, tag, SHA)]:Revision:"
-  {-R,--recursive}"[Recursively add each file under the directory.]"
-  "--dvc-only[Only outputs DVC-outs.]"
-  "1:URL:"
-  "2:Path:"
-)
-
-_dvc_metrics=(
-  "1:Sub command:(show diff)"
-)
-
-_dvc_move=(
-  "1:Source:_files"
-  "2:Destination:"
-)
-
-_dvc_params=(
-  "1:Sub command:(diff)"
-)
-
-_dvc_pull=(
-  {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
-  {-r,--remote}"[Remote repository to pull from.]:Remote repository:"
-  {-a,--all-branches}"[Fetch cache for all branches.]"
-  {-T,--all-tags}"[Fetch cache for all tags.]"
-  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]"
-  {-f,--force}"[Do not prompt when removing working directory files.]"
-  {-R,--recursive}"[Pull cache for subdirectories of the specified directory.]"
-  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-)
-
-_dvc_push=(
-  {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
-  {-r,--remote}"[Remote repository to push to.]:Remote repository:"
-  {-a,--all-branches}"[Push cache for all branches.]"
-  {-T,--all-tags}"[Push cache for all tags.]"
-  {-d,--with-deps}"[Push cache for all dependencies of the specified target.]"
-  {-R,--recursive}"[Push cache for subdirectories of specified directory.]"
-  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-)
-
-_dvc_plots=(
-  "1:Sub command:(show diff modify)"
-)
-
-_dvc_remote=(
-  "1:Sub command:(add default remove modify list rename)"
-)
-
-_dvc_remove=(
-  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-  "--dry[Only print the commands that would be executed without actually executing]"
-  {-o,--outs}"[Only remove DVC-file outputs. (Default)]"
-  {-p,--purge}"[Remove DVC-file and all its outputs.]"
-  {-f,--force}"[Force purge.]"
-)
-
-_dvc_repro=(
-  {-f,--force}"[Reproduce even if dependencies were not changed.]"
-  {-s,--single-item}"[Reproduce only single data item without recursive dependencies check.]"
-  {-c,--cwd}"[Directory within your repo to reproduce from.]:CWD:_files -/"
-  {-m,--metrics}"[Show metrics after reproduction.]"
-  "--dry[Only print the commands that would be executed without actually executing]"
-  {-i,--interactive}"[Ask for confirmation before reproducing each stage.]"
-  {-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]"
-  {-P,--all-pipelines}"[Reproduce all pipelines in the repo.]"
-  {-R,--recursive}"[Reproduce all stages in the specified directory.]"
-  "--force-downstream[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]"
-  "--no-run-cache[Run changed stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
-  "--no-commit[Don't put files/directories into cache.]"
-  "--downstream[Start from the specified stages when reproducing pipelines.]"
-  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-)
-
-_dvc_root=()
-
-_dvc_run=(
-  "*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files"
-  "*"{-o,--outs}"[Declare output file or directory.]:Output data:_files"
-  "*"{-O,--outs-no-cache}"[Declare output file or directory (do not put into DVC cache).]:Output regular:_files"
-  "*"{-m,--metrics}"[Declare output metric file or directory.]:Metrics:_files"
-  "*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache).]:Metrics (no cache):_files"
-  {-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
-  {-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/"
-  {-w,--wdir}"[Directory within your repo to run your command in.]:WDIR:_files -/"
-  "--no-exec[Only create stage file without actually running it.]"
-  {-y,--yes}"[Deprecated, use --overwrite-dvcfile instead]"
-  "--overwrite-dvcfile[Overwrite existing DVC-file without asking for confirmation.]"
-  "--no-run-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
-  "--remove-outs[Deprecated, this is now the default behavior]"
-  "--no-commit[Don't put files/directories into cache.]"
-  "--outs-persist[Declare output file or directory that will not be removed upon repro.]:Output persistent:_files"
-  "--outs-persist-no-cache[Declare output file or directory that will not be removed upon repro (do not put into DVC cache).]:Output persistent regular:_files"
-  "--external[Allow outputs that are outside of the DVC project.]"
-)
-
-_dvc_status=(
-  {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
-  {-q,--quiet}"[Suppresses all output. Exit with 0 if pipelines are up to date, otherwise 1.]"
-  {-c,--cloud}"[Show status of a local cache compared to a remote repository.]"
-  {-r,--remote}"[Remote repository to compare local cache to.]:Remote repository:"
-  {-a,--all-branches}"[Show status of a local cache compared to a remote repository for all branches.]"
-  {-T,--all-tags}"[Show status of a local cache compared to a remote repository for all tags.]"
-  {-d,--with-deps}"[Show status for all dependencies of the specified target.]"
-  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-)
-
-_dvc_unfreeze=(
-  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-)
-
-_dvc_unprotect=(
-  "*:Data files:_files"
-)
-
-_dvc_update=(
-  "--rev[Git revision (e.g. SHA, branch, tag)]:Revision:"
-  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-)
-
-typeset -A opt_args
-local context state line curcontext="$curcontext"
-
-_arguments \
-  $_dvc_options \
-  '1: :_dvc_commands' \
-  '*::args:->args'
-
-case $words[1] in
-  add) _arguments $_dvc_global_options $_dvc_add ;;
-  cache) _arguments $_dvc_global_options $_dvc_cache ;;
-  checkout) _arguments $_dvc_global_options $_dvc_checkout ;;
-  commit) _arguments $_dvc_global_options $_dvc_commit ;;
-  config) _arguments $_dvc_global_options $_dvc_config ;;
-  dag) _arguments $_dvc_global_options $_dvc_dag ;;
-  destroy) _arguments $_dvc_global_options $_dvc_destroy ;;
-  diff) _arguments $_dvc_global_options $_dvc_diff ;;
-  fetch) _arguments $_dvc_global_options $_dvc_fetch ;;
-  freeze) _arguments $_dvc_global_options $_dvc_freeze ;;
-  get-url) _arguments $_dvc_global_options $_dvc_geturl ;;
-  get) _arguments $_dvc_global_options $_dvc_get ;;
-  gc) _arguments $_dvc_global_options $_dvc_gc ;;
-  import-url) _arguments $_dvc_global_options $_dvc_importurl ;;
-  import) _arguments $_dvc_global_options $_dvc_import ;;
-  init) _arguments $_dvc_global_options $_dvc_init ;;
-  install) _arguments $_dvc_global_options $_dvc_install ;;
-  list) _arguments $_dvc_global_options $_dvc_list ;;
-  metrics) _arguments $_dvc_global_options $_dvc_metrics ;;
-  move) _arguments $_dvc_global_options $_dvc_move ;;
-  params) _arguments $_dvc_global_options $_dvc_params ;;
-  pull) _arguments $_dvc_global_options $_dvc_pull ;;
-  push) _arguments $_dvc_global_options $_dvc_push ;;
-  plots) _arguments $_dvc_global_options $_dvc_plots ;;
-  remote) _arguments $_dvc_global_options $_dvc_remote ;;
-  remove) _arguments $_dvc_global_options $_dvc_remove ;;
-  repro) _arguments $_dvc_global_options $_dvc_repro ;;
-  root) _arguments $_dvc_global_options $_dvc_root ;;
-  run) _arguments $_dvc_global_options $_dvc_run ;;
-  status) _arguments $_dvc_global_options $_dvc_status ;;
-  unfreeze) _arguments $_dvc_global_options $_dvc_unfreeze ;;
-  unprotect) _arguments $_dvc_global_options $_dvc_unprotect ;;
-  update) _arguments $_dvc_global_options $_dvc_update ;;
-esac

setup.cfg

@@ -17,7 +17,7 @@ count=true
 [isort]
 include_trailing_comma=true
 known_first_party=dvc,tests
-known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,tqdm,voluptuous,yaml,zc
+known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
 line_length=79
 force_grid_wrap=0
 use_parentheses=True

setup.py

@@ -78,6 +78,7 @@ install_requires = [
     "tabulate>=0.8.7",
     "pygtrie==2.3.2",
     "dpath>=2.0.1,<3",
+    "shtab>=0.0.2",
 ]
 
 

snap/snapcraft.yaml

@@ -31,17 +31,21 @@ parts:
         snapcraftctl pull
         snapcraftctl set-version $(cd $SNAPCRAFT_PART_SRC && git describe --tags)
         git diff --quiet || error_dirty_build
-        echo 'PKG = "snap"' > $SNAPCRAFT_PART_SRC/dvc/utils/build.py
+        echo 'PKG = "snap"' > dvc/utils/build.py
         # install all optional extras
-        sed -ri 's/(=install_requires)/\1+all_remotes+hdfs/' $SNAPCRAFT_PART_SRC/setup.py
+        sed -ri 's/(=install_requires)/\1+all_remotes+hdfs/' setup.py
         # ensure dvc knows the state isn't really dirty
-        sed -rin 's/.*git.*diff.*--quiet.*//' $SNAPCRAFT_PART_SRC/dvc/version.py
+        sed -rin 's/.*git.*diff.*--quiet.*//' dvc/version.py
     override-build: |
         snapcraftctl build
         # prevent user site packages interfering with this snap - reference:
         # https://github.com/snapcore/snapcraft/blob/19393ef36cd773a28131cec10cc0bfb3bf9c7e77/tools/snapcraft-override-build.sh#L18
         sed -ri 's/^(ENABLE_USER_SITE = )None$/\1False/' $SNAPCRAFT_PART_INSTALL/usr/lib/python*/site.py
-        cp $SNAPCRAFT_PART_BUILD/scripts/completion/dvc.bash $SNAPCRAFT_PART_INSTALL/completion.sh
+        # make plugin aware that a file (generated later) needs to be staged
+        touch $SNAPCRAFT_PART_INSTALL/completion.sh
+    override-stage: |
+        snapcraftctl stage
+        usr/bin/python3 -m dvc completion -s bash > completion.sh
 apps:
   dvc:
     command: bin/dvc