diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 000000000..55595a926 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,568 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=print-statement, + parameter-unpacking, + unpacking-in-except, + old-raise-syntax, + backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + non-ascii-bytes-literal, + raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + deprecated-itertools-function, + deprecated-types-field, + next-method-defined, + dict-items-not-iterating, + dict-keys-not-iterating, + dict-values-not-iterating, + deprecated-operator-function, + deprecated-urllib-function, + xreadlines-attribute, + deprecated-sys-function, + exception-escape, + comprehension-escape, + useless-object-inheritance + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[LOGGING] + +# Format style used to check logging format string. `old` means using % +# formatting, while `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package.. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma, + dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _, + s3, + gs, + x, y, + x0, y0, + x1, y1, + dx, dy + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement. +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception". +overgeneral-exceptions=Exception diff --git a/dvc/__init__.py b/dvc/__init__.py index 103342d99..16c9a5ddd 100644 --- a/dvc/__init__.py +++ b/dvc/__init__.py @@ -14,39 +14,50 @@ PACKAGEPATH = os.path.abspath(os.path.dirname(__file__)) HOMEPATH = os.path.dirname(PACKAGEPATH) VERSIONPATH = os.path.join(PACKAGEPATH, 'version.py') + +def _update_version_file(): + """Dynamically update version file.""" + from git import Repo + from git.exc import InvalidGitRepositoryError + + try: + repo = Repo(HOMEPATH) + except InvalidGitRepositoryError: + return __version__ + + sha = repo.head.object.hexsha + short_sha = repo.git.rev_parse(sha, short=6) + dirty = '.mod' if repo.is_dirty() else '' + ver = '{}+{}{}'.format(__version__, short_sha, dirty) + + # Write a helper file, that will be installed with the package + # and will provide a true version of the installed dvc + with open(VERSIONPATH, 'w+') as fobj: + fobj.write('# AUTOGENERATED by dvc/__init__.py\n') + fobj.write('version = "{}"\n'.format(ver)) + + return ver + + +def _remove_version_file(): + """Remove version.py so that it doesn't get into the release.""" + if os.path.exists(VERSIONPATH): + os.unlink(VERSIONPATH) + + if os.path.exists(os.path.join(HOMEPATH, 'setup.py')): # dvc is run directly from source without installation or # __version__ is called from setup.py if os.getenv('APPVEYOR_REPO_TAG', '').lower() != 'true' \ and os.getenv('TRAVIS_TAG', '') == '': - # Dynamically update version - try: - import git - repo = git.Repo(HOMEPATH) - sha = repo.head.object.hexsha - short_sha = repo.git.rev_parse(sha, short=6) - dirty = '.mod' if repo.is_dirty() else '' - __version__ = '{}+{}{}'.format(__version__, short_sha, dirty) - - # Write a helper file, that will be installed with the package - # and will provide a true version of the installed dvc - with open(VERSIONPATH, 'w+') as fd: - fd.write('# AUTOGENERATED by dvc/__init__.py\n') - fd.write('version = "{}"\n'.format(__version__)) - except Exception: # pragma: no cover - pass + __version__ = _update_version_file() else: # pragma: no cover - # Remove version.py so that it doesn't get into the release - if os.path.exists(VERSIONPATH): - os.unlink(VERSIONPATH) + _remove_version_file() else: # pragma: no cover # dvc was installed with pip or something. Hopefully we have our # auto-generated version.py to help us provide a true version - try: - from dvc.version import version - __version__ = version - except Exception: - pass + from dvc.version import version + __version__ = version VERSION = __version__ diff --git a/dvc/__main__.py b/dvc/__main__.py index 964065a73..10bdd5b51 100644 --- a/dvc/__main__.py +++ b/dvc/__main__.py @@ -1,3 +1,5 @@ +"""Main entry point for dvc command line tool.""" + import sys from dvc.main import main diff --git a/dvc/analytics.py b/dvc/analytics.py index 584233dae..620947ce5 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -1,3 +1,5 @@ +"""Collect and send usage analytics""" + import os import json import errno @@ -7,6 +9,11 @@ from dvc import VERSION class Analytics(object): + """Class for collecting and sending usage analytics. + + Args: + info (dict): optional existing analytics report. + """ URL = 'https://analytics.dvc.org' TIMEOUT_POST = 5 @@ -34,17 +41,20 @@ class Analytics(object): PARAM_CMD_CLASS = 'cmd_class' PARAM_CMD_RETURN_CODE = 'cmd_return_code' - def __init__(self, info={}): + def __init__(self, info=None): from dvc.config import Config from dvc.lock import Lock + if info is None: + info = {} + self.info = info cdir = Config.get_global_config_dir() try: os.makedirs(cdir) - except OSError as e: - if e.errno != errno.EEXIST: + except OSError as exc: + if exc.errno != errno.EEXIST: raise self.user_id_file = os.path.join(cdir, self.USER_ID_FILE) @@ -52,10 +62,15 @@ class Analytics(object): @staticmethod def load(path): + """Loads analytics report from json file specified by path. + + Args: + path (str): path to json file with analytics report. + """ with open(path, 'r') as fobj: - a = Analytics(info=json.load(fobj)) + analytics = Analytics(info=json.load(fobj)) os.unlink(path) - return a + return analytics def _write_user_id(self): import uuid @@ -93,7 +108,7 @@ class Analytics(object): def _collect_windows(self): import sys - version = sys.getwindowsversion() + version = sys.getwindowsversion() # pylint: disable=no-member info = {} info[self.PARAM_OS] = 'windows' info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major @@ -121,16 +136,20 @@ class Analytics(object): def _collect_system_info(self): import platform system = platform.system() + if system == 'Windows': return self._collect_windows() - elif system == 'Darwin': + + if system == 'Darwin': return self._collect_darwin() - elif system == 'Linux': + + if system == 'Linux': return self._collect_linux() - else: - raise NotImplementedError + + raise NotImplementedError def collect(self): + """Collect analytics report.""" from dvc.scm import SCM from dvc.utils import is_binary from dvc.project import Project @@ -143,12 +162,13 @@ class Analytics(object): self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info() try: - scm = SCM(root_dir=Project._find_root()) + scm = SCM(root_dir=Project.find_root()) self.info[self.PARAM_SCM_CLASS] = type(scm).__name__ except NotDvcProjectError: pass def collect_cmd(self, args, ret): + """Collect analytics info from a CLI command.""" from dvc.command.daemon import CmdDaemonAnalytics assert isinstance(ret, int) or ret is None @@ -161,6 +181,11 @@ class Analytics(object): self.info[self.PARAM_CMD_CLASS] = args.func.__name__ def dump(self): + """Save analytics report to a temporary file. + + Returns: + str: path to the temporary file that contains the analytics report. + """ import tempfile with tempfile.NamedTemporaryFile(delete=False, mode='w') as fobj: @@ -182,7 +207,7 @@ class Analytics(object): if cmd is None or not hasattr(cmd, 'config'): try: - dvc_dir = Project._find_dvc_dir() + dvc_dir = Project.find_dvc_dir() config = Config(dvc_dir) assert config is not None except NotDvcProjectError: @@ -192,7 +217,7 @@ class Analytics(object): config = cmd.config assert config is not None - core = config._config.get(Config.SECTION_CORE, {}) + core = config.config.get(Config.SECTION_CORE, {}) enabled = core.get(Config.SECTION_CORE_ANALYTICS, True) logger.debug("Analytics is {}abled." .format('en' if enabled else 'dis')) @@ -200,16 +225,23 @@ class Analytics(object): @staticmethod def send_cmd(cmd, args, ret): - from dvc.daemon import Daemon + """Collect and send analytics for CLI command. + + Args: + args (list): parsed args for the CLI command. + ret (int): return value of the CLI command. + """ + from dvc.daemon import daemon if not Analytics._is_enabled(cmd): return - a = Analytics() - a.collect_cmd(args, ret) - Daemon()(['analytics', a.dump()]) + analytics = Analytics() + analytics.collect_cmd(args, ret) + daemon(['analytics', analytics.dump()]) def send(self): + """Collect and send analytics.""" import requests if not self._is_enabled(): diff --git a/dvc/cache.py b/dvc/cache.py index 285adf69a..6d3cfd964 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -1,9 +1,17 @@ +"""Manages cache of a dvc project.""" + import os from dvc.config import Config class Cache(object): + """Class that manages cache locations of a dvc project. + + Args: + project (dvc.project.Project): project instance that this cache + belongs to. + """ CACHE_DIR = 'cache' def __init__(self, project): @@ -11,12 +19,12 @@ class Cache(object): self.project = project - config = project.config._config[Config.SECTION_CACHE] + config = project.config.config[Config.SECTION_CACHE] local = config.get(Config.SECTION_CACHE_LOCAL) if local: name = Config.SECTION_REMOTE_FMT.format(local) - sect = project.config._config[name] + sect = project.config.config[name] else: default_cache_dir = os.path.join(project.dvc_dir, self.CACHE_DIR) cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir) @@ -31,13 +39,43 @@ class Cache(object): Config.SECTION_CACHE_PROTECTED: protected } - self.local = Remote(project, sect) + self._local = Remote(project, sect) - self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3) - self.gs = self._get_remote(config, Config.SECTION_CACHE_GS) - self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH) - self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS) - self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE) + self._s3 = self._get_remote(config, Config.SECTION_CACHE_S3) + self._gs = self._get_remote(config, Config.SECTION_CACHE_GS) + self._ssh = self._get_remote(config, Config.SECTION_CACHE_SSH) + self._hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS) + self._azure = self._get_remote(config, Config.SECTION_CACHE_AZURE) + + @property + def local(self): + """Remote instance for local cache.""" + return self._local + + @property + def s3(self): + """Remote instance for AWS S3 cache.""" + return self._s3 + + @property + def gs(self): + """Remote instance for Google Cloud Storage cache.""" + return self._gs + + @property + def ssh(self): + """Remote instance for SSH cache.""" + return self._ssh + + @property + def hdfs(self): + """Remote instance for HDFS cache.""" + return self._hdfs + + @property + def azure(self): + """Remote instance for azure cache.""" + return self._azure def _get_remote(self, config, name): from dvc.remote import Remote @@ -47,5 +85,5 @@ class Cache(object): return None name = Config.SECTION_REMOTE_FMT.format(remote) - sect = self.project.config._config[name] + sect = self.project.config.config[name] return Remote(self.project, sect) diff --git a/dvc/cli.py b/dvc/cli.py index 7703e445a..afa8990d8 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -1,1120 +1,141 @@ +"""Command line interface for dvc.""" + from __future__ import print_function -import os import sys import argparse import dvc.logger as logger -from dvc.command.init import CmdInit -from dvc.command.destroy import CmdDestroy -from dvc.command.remove import CmdRemove -from dvc.command.move import CmdMove -from dvc.command.unprotect import CmdUnprotect -from dvc.command.run import CmdRun -from dvc.command.repro import CmdRepro -from dvc.command.data_sync import CmdDataPush, CmdDataPull, CmdDataFetch -from dvc.command.status import CmdDataStatus -from dvc.command.gc import CmdGC -from dvc.command.add import CmdAdd -from dvc.command.imp import CmdImport -from dvc.command.config import CmdConfig -from dvc.command.checkout import CmdCheckout -from dvc.command.remote import CmdRemoteAdd, CmdRemoteRemove -from dvc.command.remote import CmdRemoteModify, CmdRemoteList, CmdRemoteDefault -from dvc.command.cache import CmdCacheDir -from dvc.command.metrics import CmdMetricsShow, CmdMetricsAdd -from dvc.command.metrics import CmdMetricsRemove, CmdMetricsModify -from dvc.command.install import CmdInstall -from dvc.command.root import CmdRoot -from dvc.command.lock import CmdLock, CmdUnlock -from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList -from dvc.command.daemon import CmdDaemonUpdater, CmdDaemonAnalytics +from dvc.command.base import fix_subparsers +import dvc.command.init as init +import dvc.command.destroy as destroy +import dvc.command.remove as remove +import dvc.command.move as move +import dvc.command.unprotect as unprotect +import dvc.command.run as run +import dvc.command.repro as repro +import dvc.command.data_sync as data_sync +import dvc.command.gc as gc +import dvc.command.add as add +import dvc.command.imp as imp +import dvc.command.config as config +import dvc.command.checkout as checkout +import dvc.command.remote as remote +import dvc.command.cache as cache +import dvc.command.metrics as metrics +import dvc.command.install as install +import dvc.command.root as root +import dvc.command.lock as lock +import dvc.command.pipeline as pipeline +import dvc.command.daemon as daemon from dvc.exceptions import DvcParserError from dvc import VERSION -def _fix_subparsers(subparsers): - # NOTE: Workaround for bug in Python 3 - # More info at: - # https://bugs.python.org/issue16308 - # https://github.com/iterative/dvc/issues/769 - if sys.version_info[0] == 3: # pragma: no cover - subparsers.required = True - subparsers.dest = 'cmd' +COMMANDS = [ + init, + destroy, + add, + remove, + move, + unprotect, + run, + repro, + data_sync, + gc, + add, + imp, + config, + checkout, + remote, + cache, + metrics, + install, + root, + lock, + pipeline, + daemon, +] class DvcParser(argparse.ArgumentParser): + """Custom parser class for dvc CLI.""" def error(self, message): + """Custom error method. + Args: + message (str): error message. + + Raises: + dvc.exceptions.DvcParser: dvc parser exception. + """ logger.error(message) self.print_help() raise DvcParserError() class VersionAction(argparse.Action): # pragma: no cover + # pylint: disable=too-few-public-methods + """Shows dvc version and exits.""" def __call__(self, parser, namespace, values, option_string=None): print(VERSION) sys.exit(0) def parse_args(argv=None): + """Parses CLI arguments. + + Args: + argv: optional list of arguments to parse. sys.argv is used by default. + + Raises: + dvc.exceptions.DvcParserError: raised for argument parsing errors. + """ # Common args parent_parser = argparse.ArgumentParser(add_help=False) log_level_group = parent_parser.add_mutually_exclusive_group() log_level_group.add_argument( - '-q', - '--quiet', - action='store_true', - default=False, - help='Be quiet.') + '-q', + '--quiet', + action='store_true', + default=False, + help='Be quiet.') log_level_group.add_argument( - '-v', - '--verbose', - action='store_true', - default=False, - help='Be verbose.') + '-v', + '--verbose', + action='store_true', + default=False, + help='Be verbose.') # Main parser desc = 'Data Version Control' - parser = DvcParser(prog='dvc', - description=desc, - parents=[parent_parser], - formatter_class=argparse.RawTextHelpFormatter) + parser = DvcParser( + prog='dvc', + description=desc, + parents=[parent_parser], + formatter_class=argparse.RawTextHelpFormatter) # NOTE: On some python versions action='version' prints to stderr # instead of stdout https://bugs.python.org/issue18920 - parser.add_argument('-V', - '--version', - action=VersionAction, - nargs=0, - help="Show program's version.") + parser.add_argument( + '-V', + '--version', + action=VersionAction, + nargs=0, + help="Show program's version.") # Sub commands subparsers = parser.add_subparsers( - title='Available Commands', - metavar='COMMAND', - dest='cmd', - help='Use dvc COMMAND --help for command-specific ' - 'help.') + title='Available Commands', + metavar='COMMAND', + dest='cmd', + help='Use dvc COMMAND --help for command-specific help.') - _fix_subparsers(subparsers) + fix_subparsers(subparsers) - # Init - INIT_HELP = 'Initialize dvc over a directory ' \ - '(should already be a git dir).' - init_parser = subparsers.add_parser( - 'init', - parents=[parent_parser], - description=INIT_HELP, - help=INIT_HELP) - init_parser.add_argument( - '--no-scm', - action='store_true', - default=False, - help="Initiate dvc in directory that is " - "not tracked by any scm tool (e.g. git).") - init_parser.add_argument( - '-f', - '--force', - action='store_true', - default=False, - help="Overwrite '.dvc' if it exists. Will remove " - "all local cache.") - init_parser.set_defaults(func=CmdInit) - - # Destroy - DESTROY_HELP = "Destroy dvc. Will remove all project's information, " \ - "data files and cache." - destroy_parser = subparsers.add_parser( - 'destroy', - parents=[parent_parser], - description=DESTROY_HELP, - help=DESTROY_HELP) - destroy_parser.add_argument( - '-f', - '--force', - action='store_true', - default=False, - help='Force destruction.') - destroy_parser.set_defaults(func=CmdDestroy) - - # Add - ADD_HELP = 'Add files/directories to dvc.' - add_parser = subparsers.add_parser( - 'add', - parents=[parent_parser], - description=ADD_HELP, - help=ADD_HELP) - add_parser.add_argument( - '-R', - '--recursive', - action='store_true', - default=False, - help='Recursively add each file under the directory.') - add_parser.add_argument( - 'targets', - nargs='+', - help='Input files/directories.') - add_parser.set_defaults(func=CmdAdd) - - # Import - IMPORT_HELP = 'Import files from URL.' - import_parser = subparsers.add_parser( - 'import', - parents=[parent_parser], - description=IMPORT_HELP, - help=IMPORT_HELP) - import_parser.add_argument( - 'url', - help="URL. Supported urls: " - "'/path/to/file', " - "'C:\\\\path\\to\\file', " - "'https://example.com/path/to/file', " - "'s3://bucket/path/to/file', " - "'gs://bucket/path/to/file', " - "'hdfs://example.com/path/to/file', " - "'ssh://example.com:/path/to/file', " - "'remote://myremote/path/to/file'(see " - "`dvc remote` commands). ") - import_parser.add_argument( - 'out', - nargs='?', - help='Output.') - import_parser.set_defaults(func=CmdImport) - - # Checkout - CHECKOUT_HELP = 'Checkout data files from cache.' - checkout_parser = subparsers.add_parser( - 'checkout', - parents=[parent_parser], - description=CHECKOUT_HELP, - help=CHECKOUT_HELP) - checkout_parser.add_argument( - '-d', - '--with-deps', - action='store_true', - default=False, - help='Checkout all dependencies of the ' - 'specified target.') - checkout_parser.add_argument( - '-f', - '--force', - action='store_true', - default=False, - help='Do not prompt when removing ' - 'working directory files.') - checkout_parser.add_argument( - 'targets', - nargs='*', - help='DVC files.') - checkout_parser.set_defaults(func=CmdCheckout) - - # Run - RUN_HELP = 'Generate a stage file from a given ' \ - 'command and execute the command.' - run_parser = subparsers.add_parser( - 'run', - parents=[parent_parser], - description=RUN_HELP, - help=RUN_HELP) - run_parser.add_argument( - '-d', - '--deps', - action='append', - default=[], - help='Declare dependencies for reproducible cmd.') - run_parser.add_argument( - '-o', - '--outs', - action='append', - default=[], - help='Declare output data file or data directory.') - run_parser.add_argument( - '-O', - '--outs-no-cache', - action='append', - default=[], - help='Declare output regular file or ' - 'directory (sync to Git, not DVC cache).') - run_parser.add_argument( - '-M', - '--metrics-no-cache', - action='append', - default=[], - help='Declare output metric file or ' - 'directory (not cached by DVC).') - run_parser.add_argument( - '-f', - '--file', - help="Specify name of the stage file. It should be " - "either 'Dvcfile' or have a '.dvc' suffix (e.g. " - "'prepare.dvc', 'clean.dvc', etc) in order for " - "dvc to be able to find it later. By default " - "the first output basename + .dvc is used as " - "a stage filename.") - run_parser.add_argument( - '-c', - '--cwd', - default=os.path.curdir, - help='Directory within your project to run your ' - 'command and place stage file in.') - run_parser.add_argument( - '--no-exec', - action='store_true', - default=False, - help="Only create stage file without actually " - "running it.") - run_parser.add_argument( - '-y', - '--yes', - action='store_true', - default=False, - help="(OBSOLETED, use --overwrite-dvcfile instead) " - "Automatic 'yes' answer to all prompts. E.g. " - "when '.dvc' file exists and dvc asks if you " - "want to overwrite it.") - run_parser.add_argument( - '--overwrite-dvcfile', - action='store_true', - default=False, - help="Overwrite existing dvc file without asking " - "for confirmation.") - run_parser.add_argument( - '--ignore-build-cache', - action='store_true', - default=False, - help="Run this stage even if it has been already " - "ran with the same command/dependencies/outputs/etc " - "before.") - run_parser.add_argument( - '--remove-outs', - action='store_true', - default=False, - help="Remove outputs before running the command.") - run_parser.add_argument( - 'command', - nargs=argparse.REMAINDER, - help='Command to execute.') - run_parser.set_defaults(func=CmdRun) - - # Parent parser used in pull/push/status - parent_cache_parser = argparse.ArgumentParser( - add_help=False, - parents=[parent_parser]) - parent_cache_parser.add_argument( - '-j', - '--jobs', - type=int, - default=None, - help='Number of jobs to run simultaneously.') - parent_cache_parser.add_argument( - '--show-checksums', - action='store_true', - default=False, - help='Show checksums instead of file names.') - parent_cache_parser.add_argument( - 'targets', - nargs='*', - default=None, - help='DVC files.') - - # Pull - PULL_HELP = 'Pull data files from the cloud.' - pull_parser = subparsers.add_parser( - 'pull', - parents=[parent_cache_parser], - description=PULL_HELP, - help=PULL_HELP) - pull_parser.add_argument( - '-r', - '--remote', - help='Remote repository to pull from.') - pull_parser.add_argument( - '-a', - '--all-branches', - action='store_true', - default=False, - help='Fetch cache for all branches.') - pull_parser.add_argument( - '-T', - '--all-tags', - action='store_true', - default=False, - help='Fetch cache for all tags.') - pull_parser.add_argument( - '-d', - '--with-deps', - action='store_true', - default=False, - help='Fetch cache for all dependencies of the ' - 'specified target.') - pull_parser.add_argument( - '-f', - '--force', - action='store_true', - default=False, - help='Do not prompt when removing ' - 'working directory files.') - pull_parser.add_argument( - '-R', - '--recursive', - help='Pull cache for subdirectories of the specified ' - 'directory.') - pull_parser.set_defaults(func=CmdDataPull) - - # Push - PUSH_HELP = 'Push data files to the cloud.' - push_parser = subparsers.add_parser( - 'push', - parents=[parent_cache_parser], - description=PUSH_HELP, - help=PUSH_HELP) - push_parser.add_argument( - '-r', - '--remote', - help='Remote repository to push to.') - push_parser.add_argument( - '-a', - '--all-branches', - action='store_true', - default=False, - help='Push cache for all branches.') - push_parser.add_argument( - '-T', - '--all-tags', - action='store_true', - default=False, - help='Push cache for all tags.') - push_parser.add_argument( - '-d', - '--with-deps', - action='store_true', - default=False, - help='Push cache for all dependencies of the ' - 'specified target.') - push_parser.add_argument( - '-R', - '--recursive', - help='Push cache from subdirectories of specified ' - 'directory.') - push_parser.set_defaults(func=CmdDataPush) - - # Fetch - FETCH_HELP = 'Fetch data files from the cloud.' - fetch_parser = subparsers.add_parser( - 'fetch', - parents=[parent_cache_parser], - description=FETCH_HELP, - help=FETCH_HELP) - fetch_parser.add_argument( - '-r', - '--remote', - help='Remote repository to fetch from.') - fetch_parser.add_argument( - '-a', - '--all-branches', - action='store_true', - default=False, - help='Fetch cache for all branches.') - fetch_parser.add_argument( - '-T', - '--all-tags', - action='store_true', - default=False, - help='Fetch cache for all tags.') - fetch_parser.add_argument( - '-d', - '--with-deps', - action='store_true', - default=False, - help='Fetch cache for all dependencies of the ' - 'specified target.') - fetch_parser.add_argument( - '-R', - '--recursive', - help='Fetch cache for subdirectories of specified ' - 'directory.') - fetch_parser.set_defaults(func=CmdDataFetch) - - # Status - STATUS_HELP = 'Show the project status.' - status_parser = subparsers.add_parser( - 'status', - parents=[parent_cache_parser], - description=STATUS_HELP, - help=STATUS_HELP) - status_parser.add_argument( - '-c', - '--cloud', - action='store_true', - default=False, - help='Show status of a local cache compared to a ' - 'remote repository.') - status_parser.add_argument( - '-r', - '--remote', - help='Remote repository to compare local cache to.') - status_parser.add_argument( - '-a', - '--all-branches', - action='store_true', - default=False, - help='Show status of a local cache compared to a ' - 'remote repository for all branches.') - status_parser.add_argument( - '-T', - '--all-tags', - action='store_true', - default=False, - help='Show status of a local cache compared to a ' - 'remote repository for all tags.') - status_parser.add_argument( - '-d', - '--with-deps', - action='store_true', - default=False, - help='Show status for all dependencies of the ' - 'specified target.') - status_parser.set_defaults(func=CmdDataStatus) - - # Repro - REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'." - repro_parser = subparsers.add_parser( - 'repro', - parents=[parent_parser], - description=REPRO_HELP, - help=REPRO_HELP) - repro_parser.add_argument( - 'targets', - nargs='*', - help='DVC file to reproduce.') - repro_parser.add_argument( - '-f', - '--force', - action='store_true', - default=False, - help='Reproduce even if dependencies were not ' - 'changed.') - repro_parser.add_argument( - '-s', - '--single-item', - action='store_true', - default=False, - help='Reproduce only single data item without ' - 'recursive dependencies check.') - repro_parser.add_argument( - '-c', - '--cwd', - default=os.path.curdir, - help='Directory within your project to ' - 'reroduce from.') - repro_parser.add_argument( - '-m', - '--metrics', - action='store_true', - default=False, - help='Show metrics after reproduction.') - repro_parser.add_argument( - '--dry', - action='store_true', - default=False, - help='Only print the commands that would be executed ' - 'without actually executing.') - repro_parser.add_argument( - '-i', - '--interactive', - action='store_true', - default=False, - help='Ask for confirmation before reproducing each ' - 'stage.') - repro_parser.add_argument( - '-p', - '--pipeline', - action='store_true', - default=False, - help='Reproduce the whole pipeline that the ' - 'specified stage file belongs to.') - repro_parser.add_argument( - '-P', - '--all-pipelines', - action='store_true', - default=False, - help='Reproduce all pipelines in the project.') - repro_parser.add_argument( - '--ignore-build-cache', - action='store_true', - default=False, - help='Reproduce all descendants of a changed stage' - "even if their direct dependencies didn't " - 'change.') - repro_parser.set_defaults(func=CmdRepro) - - # Remove - REMOVE_HELP = 'Remove outputs of DVC file.' - remove_parser = subparsers.add_parser( - 'remove', - parents=[parent_parser], - description=REMOVE_HELP, - help=REMOVE_HELP) - remove_parser_group = remove_parser.add_mutually_exclusive_group() - remove_parser_group.add_argument( - '-o', - '--outs', - action='store_true', - default=True, - help='Only remove DVC file outputs.(default)') - remove_parser_group.add_argument( - '-p', - '--purge', - action='store_true', - default=False, - help='Remove DVC file and all its outputs.') - remove_parser.add_argument( - '-f', - '--force', - action='store_true', - default=False, - help='Force purge.') - remove_parser.add_argument( - 'targets', - nargs='+', - help='DVC files.') - remove_parser.set_defaults(func=CmdRemove) - - # Move - MOVE_HELP = 'Move output of DVC file.' - move_parser = subparsers.add_parser( - 'move', - parents=[parent_parser], - description=MOVE_HELP, - help=MOVE_HELP) - move_parser.add_argument( - 'src', - help='Source.') - move_parser.add_argument( - 'dst', - help='Destination.') - move_parser.set_defaults(func=CmdMove) - - # Unprotect - UNPROTECT_HELP = 'Unprotect data file/directory.' - unprotect_parser = subparsers.add_parser( - 'unprotect', - parents=[parent_parser], - description=UNPROTECT_HELP, - help=UNPROTECT_HELP) - unprotect_parser.add_argument( - 'targets', - nargs='+', - help='Data files/directory.') - unprotect_parser.set_defaults(func=CmdUnprotect) - - # Garbage collector - GC_HELP = 'Collect garbage.' - gc_parser = subparsers.add_parser( - 'gc', - parents=[parent_parser], - description=GC_HELP, - help=GC_HELP) - gc_parser.add_argument( - '-a', - '--all-branches', - action='store_true', - default=False, - help='Collect garbage for all branches.') - gc_parser.add_argument( - '-T', - '--all-tags', - action='store_true', - default=False, - help='Collect garbage for all tags.') - gc_parser.add_argument( - '-c', - '--cloud', - action='store_true', - default=False, - help='Collect garbage in remote repository.') - gc_parser.add_argument( - '-r', - '--remote', - help='Remote repository to collect garbage in.') - gc_parser.add_argument( - '-f', - '--force', - action='store_true', - default=False, - help='Force garbage collection.') - gc_parser.add_argument( - '-j', - '--jobs', - type=int, - default=None, - help='Number of jobs to run simultaneously.') - gc_parser.add_argument( - '-p', - '--projects', - type=str, - nargs='*', - default=None, - help='Collect garbage for all given projects.') - gc_parser.set_defaults(func=CmdGC) - - # Parent parser used in config/remote - parent_config_parser = argparse.ArgumentParser( - add_help=False, - parents=[parent_parser]) - parent_config_parser.add_argument( - '--global', - dest='glob', - action='store_true', - default=False, - help='Use global config.') - parent_config_parser.add_argument( - '--system', - action='store_true', - default=False, - help='Use system config.') - parent_config_parser.add_argument( - '--local', - action='store_true', - default=False, - help='Use local config.') - - # Config - CONFIG_HELP = 'Get or set config options.' - config_parser = subparsers.add_parser( - 'config', - parents=[parent_config_parser], - description=CONFIG_HELP, - help=CONFIG_HELP) - config_parser.add_argument( - '-u', - '--unset', - default=False, - action='store_true', - help='Unset option.') - config_parser.add_argument( - 'name', - help='Option name.') - config_parser.add_argument( - 'value', - nargs='?', - default=None, - help='Option value.') - config_parser.set_defaults(func=CmdConfig) - - # Remote - REMOTE_HELP = 'Manage set of tracked repositories.' - remote_parser = subparsers.add_parser( - 'remote', - parents=[parent_parser], - description=REMOTE_HELP, - help=REMOTE_HELP) - - remote_subparsers = remote_parser.add_subparsers( - dest='cmd', - help='Use dvc remote CMD --help for ' - 'command-specific help.') - - _fix_subparsers(remote_subparsers) - - REMOTE_ADD_HELP = 'Add remote.' - remote_add_parser = remote_subparsers.add_parser( - 'add', - parents=[parent_config_parser], - description=REMOTE_ADD_HELP, - help=REMOTE_ADD_HELP) - remote_add_parser.add_argument( - 'name', - help='Name.') - remote_add_parser.add_argument( - 'url', - help='URL. See full list of supported urls at ' - 'https://dvc.org/doc/commands-reference/remote') - remote_add_parser.add_argument( - '-d', - '--default', - action='store_true', - default=False, - help='Set as default remote.') - remote_add_parser.set_defaults(func=CmdRemoteAdd) - - REMOTE_DEFAULT_HELP = 'Set/unset default remote.' - remote_default_parser = remote_subparsers.add_parser( - 'default', - parents=[parent_config_parser], - description=REMOTE_DEFAULT_HELP, - help=REMOTE_DEFAULT_HELP) - remote_default_parser.add_argument( - 'name', - nargs='?', - help='Name of the remote.') - remote_default_parser.add_argument( - '-u', - '--unset', - action='store_true', - default=False, - help='Unset default remote.') - remote_default_parser.set_defaults(func=CmdRemoteDefault) - - REMOTE_REMOVE_HELP = 'Remove remote.' - remote_remove_parser = remote_subparsers.add_parser( - 'remove', - parents=[parent_config_parser], - description=REMOTE_REMOVE_HELP, - help=REMOTE_REMOVE_HELP) - remote_remove_parser.add_argument( - 'name', - help='Name') - remote_remove_parser.set_defaults(func=CmdRemoteRemove) - - REMOTE_MODIFY_HELP = 'Modify remote.' - remote_modify_parser = remote_subparsers.add_parser( - 'modify', - parents=[parent_config_parser], - description=REMOTE_MODIFY_HELP, - help=REMOTE_MODIFY_HELP) - remote_modify_parser.add_argument( - 'name', - help='Name.') - remote_modify_parser.add_argument( - 'option', - help='Option.') - remote_modify_parser.add_argument( - 'value', - nargs='?', - help='Value.') - remote_modify_parser.add_argument( - '-u', - '--unset', - default=False, - action='store_true', - help='Unset option.') - remote_modify_parser.set_defaults(func=CmdRemoteModify) - - REMOTE_LIST_HELP = 'List remotes.' - remote_list_parser = remote_subparsers.add_parser( - 'list', - parents=[parent_config_parser], - description=REMOTE_LIST_HELP, - help=REMOTE_LIST_HELP) - remote_list_parser.set_defaults(func=CmdRemoteList) - - # Cache - CACHE_HELP = 'Manage cache settings.' - cache_parser = subparsers.add_parser( - 'cache', - parents=[parent_parser], - description=CACHE_HELP, - help=CACHE_HELP) - - cache_subparsers = cache_parser.add_subparsers( - dest='cmd', - help='Use dvc cache CMD --help for ' - 'command-specific help.') - - _fix_subparsers(cache_subparsers) - - parent_cache_config_parser = argparse.ArgumentParser( - add_help=False, - parents=[parent_config_parser]) - CACHE_DIR_HELP = 'Configure cache directory location.' - cache_dir_parser = cache_subparsers.add_parser( - 'dir', - parents=[parent_cache_config_parser], - description=CACHE_DIR_HELP, - help=CACHE_DIR_HELP) - cache_dir_parser.add_argument( - '-u', - '--unset', - default=False, - action='store_true', - help='Unset option.') - cache_dir_parser.add_argument( - 'value', - nargs='?', - default=None, - help='Path to cache directory. Relative paths are ' - 'resolved relative to the current directory and ' - 'saved to config relative to the config file ' - 'location.') - cache_dir_parser.set_defaults(func=CmdCacheDir) - - # Metrics - METRICS_HELP = 'Get metrics from all branches.' - metrics_parser = subparsers.add_parser( - 'metrics', - parents=[parent_parser], - description=METRICS_HELP, - help=METRICS_HELP) - - metrics_subparsers = metrics_parser.add_subparsers( - dest='cmd', - help='Use dvc metrics CMD --help for ' - 'command-specific help.') - - _fix_subparsers(metrics_subparsers) - - METRICS_SHOW_HELP = 'Show metrics.' - metrics_show_parser = metrics_subparsers.add_parser( - 'show', - parents=[parent_parser], - description=METRICS_SHOW_HELP, - help=METRICS_SHOW_HELP) - metrics_show_parser.add_argument( - 'path', - nargs='?', - help='Path to metrics file.') - metrics_show_parser.add_argument( - '-t', - '--type', - help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).') - metrics_show_parser.add_argument( - '-x', - '--xpath', - help='JSON/TSV/HTSV/CSV/HCSV path.') - metrics_show_group = metrics_show_parser.add_mutually_exclusive_group() - metrics_show_group.add_argument( - '--json-path', - help='JSON path.') - metrics_show_group.add_argument( - '--tsv-path', - help="TSV path 'row,column' (e.g. '1,2').") - metrics_show_group.add_argument( - '--htsv-path', - help="Headed TSV path 'row,column (e.g. 'Name,3').") - metrics_show_group.add_argument( - '--csv-path', - help="CSV path 'row,column' (e.g. '1,2').") - metrics_show_group.add_argument( - '--hcsv-path', - help="Headed CSV path 'row,column' (e.g. 'Name,3').") - metrics_show_parser.add_argument( - '-a', - '--all-branches', - action='store_true', - default=False, - help='Show metrics for all branches.') - metrics_show_parser.add_argument( - '-T', - '--all-tags', - action='store_true', - default=False, - help='Show metrics for all tags.') - metrics_show_parser.set_defaults(func=CmdMetricsShow) - - METRICS_ADD_HELP = 'Add metrics.' - metrics_add_parser = metrics_subparsers.add_parser( - 'add', - parents=[parent_parser], - description=METRICS_ADD_HELP, - help=METRICS_ADD_HELP) - metrics_add_parser.add_argument( - '-t', - '--type', - help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).') - metrics_add_parser.add_argument( - '-x', - '--xpath', - help='JSON/TSV/HTSV/CSV/HCSV path.') - metrics_add_parser.add_argument( - 'path', - help='Path to metrics file.') - metrics_add_parser.set_defaults(func=CmdMetricsAdd) - - METRICS_MODIFY_HELP = 'Modify metrics.' - metrics_modify_parser = metrics_subparsers.add_parser( - 'modify', - parents=[parent_parser], - description=METRICS_MODIFY_HELP, - help=METRICS_MODIFY_HELP) - metrics_modify_parser.add_argument( - '-t', - '--type', - help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).') - metrics_modify_parser.add_argument( - '-x', - '--xpath', - help='JSON/TSV/HTSV/CSV/HCSV path.') - metrics_modify_parser.add_argument( - 'path', - help='Metrics file.') - metrics_modify_parser.set_defaults(func=CmdMetricsModify) - - METRICS_REMOVE_HELP = 'Remove metrics.' - metrics_remove_parser = metrics_subparsers.add_parser( - 'remove', - parents=[parent_parser], - description=METRICS_REMOVE_HELP, - help=METRICS_REMOVE_HELP) - metrics_remove_parser.add_argument( - 'path', - help='Path to metrics file.') - metrics_remove_parser.set_defaults(func=CmdMetricsRemove) - - # Install - INSTALL_HELP = 'Install dvc hooks into the repository.' - install_parser = subparsers.add_parser( - 'install', - parents=[parent_parser], - description=INSTALL_HELP, - help=INSTALL_HELP) - install_parser.set_defaults(func=CmdInstall) - - # Root - ROOT_HELP = "Relative path to project's directory." - root_parser = subparsers.add_parser( - 'root', - parents=[parent_parser], - description=ROOT_HELP, - help=ROOT_HELP) - root_parser.set_defaults(func=CmdRoot) - - # Lock - LOCK_HELP = 'Lock DVC file.' - lock_parser = subparsers.add_parser( - 'lock', - parents=[parent_parser], - description=LOCK_HELP, - help=LOCK_HELP) - lock_parser.add_argument( - 'targets', - nargs='+', - help='DVC files.') - lock_parser.set_defaults(func=CmdLock) - - # Unlock - UNLOCK_HELP = 'Unlock DVC file.' - unlock_parser = subparsers.add_parser( - 'unlock', - parents=[parent_parser], - description=UNLOCK_HELP, - help=UNLOCK_HELP) - unlock_parser.add_argument( - 'targets', - nargs='+', - help='DVC files.') - unlock_parser.set_defaults(func=CmdUnlock) - - # Pipeline - PIPELINE_HELP = 'Manage pipeline.' - pipeline_parser = subparsers.add_parser( - 'pipeline', - parents=[parent_parser], - description=PIPELINE_HELP, - help=PIPELINE_HELP) - - pipeline_subparsers = pipeline_parser.add_subparsers( - dest='cmd', - help='Use dvc pipeline CMD --help ' - 'for command-specific help.') - - _fix_subparsers(pipeline_subparsers) - - PIPELINE_SHOW_HELP = 'Show pipeline.' - pipeline_show_parser = pipeline_subparsers.add_parser( - 'show', - parents=[parent_parser], - description=PIPELINE_SHOW_HELP, - help=PIPELINE_SHOW_HELP) - pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group() - pipeline_show_group.add_argument( - '-c', - '--commands', - action='store_true', - default=False, - help='Print commands instead of paths to DVC files.') - pipeline_show_group.add_argument( - '-o', - '--outs', - action='store_true', - default=False, - help='Print output files instead of ' - 'paths to DVC files.') - pipeline_show_parser.add_argument( - '--ascii', - action='store_true', - default=False, - help='Output DAG as ASCII.') - pipeline_show_parser.add_argument( - '--dot', - help='Write DAG in .dot format.' - ) - pipeline_show_parser.add_argument( - 'targets', - nargs='*', - help="DVC files. 'Dvcfile' by default.") - pipeline_show_parser.set_defaults(func=CmdPipelineShow) - - PIPELINE_LIST_HELP = 'List pipelines.' - pipeline_list_parser = pipeline_subparsers.add_parser( - 'list', - parents=[parent_parser], - description=PIPELINE_LIST_HELP, - help=PIPELINE_LIST_HELP) - pipeline_list_parser.set_defaults(func=CmdPipelineList) - - # Daemon - DAEMON_HELP = 'Service daemon.' - daemon_parser = subparsers.add_parser( - 'daemon', - parents=[parent_parser], - description=DAEMON_HELP, - help=DAEMON_HELP) - - daemon_subparsers = daemon_parser.add_subparsers( - dest='cmd', - help='Use dvc daemon CMD --help ' - 'for command-specific help.') - - _fix_subparsers(daemon_subparsers) - - DAEMON_UPDATER_HELP = 'Fetch latest available version.' - daemon_updater_parser = daemon_subparsers.add_parser( - 'updater', - parents=[parent_parser], - description=DAEMON_UPDATER_HELP, - help=DAEMON_UPDATER_HELP) - daemon_updater_parser.set_defaults(func=CmdDaemonUpdater) - - DAEMON_ANALYTICS_HELP = 'Send dvc usage analytics.' - daemon_analytics_parser = daemon_subparsers.add_parser( - 'analytics', - parents=[parent_parser], - description=DAEMON_ANALYTICS_HELP, - help=DAEMON_ANALYTICS_HELP) - daemon_analytics_parser.add_argument( - 'target', - help="Analytics file.") - daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics) + for cmd in COMMANDS: + cmd.add_parser(subparsers, parent_parser) args = parser.parse_args(argv) - if (issubclass(args.func, CmdRepro) - or issubclass(args.func, CmdPipelineShow)) \ - and hasattr(args, 'targets') \ - and len(args.targets) == 0 \ - and not (hasattr(args, 'all_pipelines') - and args.all_pipelines): # pragma: no cover - if hasattr(args, 'cwd'): - cwd = args.cwd - else: - cwd = os.curdir - path = os.path.join(cwd, 'Dvcfile') - if not os.path.exists(path): - msg = "default target '{}' does not exist.".format(path) - if issubclass(args.func, CmdRepro): - repro_parser.error(msg) - elif issubclass(args.func, CmdPipelineShow): - pipeline_show_parser.error(msg) - args.targets = ['Dvcfile'] - - if issubclass(args.func, CmdRun) \ - and len(args.deps) == 0 \ - and len(args.outs) == 0 \ - and len(args.outs_no_cache) == 0 \ - and len(args.command) == 0: # pragma: no cover - run_parser.error("Too few arguments. Specify at least one: " - "'-d', '-o', '-O', 'command'.") - return args diff --git a/dvc/command/add.py b/dvc/command/add.py index 33677828b..cd3b6dac1 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -12,3 +12,23 @@ class CmdAdd(CmdBase): logger.error('failed to add file') return 1 return 0 + + +def add_parser(subparsers, parent_parser): + ADD_HELP = 'Add files/directories to dvc.' + add_parser = subparsers.add_parser( + 'add', + parents=[parent_parser], + description=ADD_HELP, + help=ADD_HELP) + add_parser.add_argument( + '-R', + '--recursive', + action='store_true', + default=False, + help='Recursively add each file under the directory.') + add_parser.add_argument( + 'targets', + nargs='+', + help='Input files/directories.') + add_parser.set_defaults(func=CmdAdd) diff --git a/dvc/command/base.py b/dvc/command/base.py index 81e05d90a..5beff9809 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -1,6 +1,21 @@ +import sys + import dvc.logger as logger +def fix_subparsers(subparsers): + """Workaround for bug in Python 3. See more info at: + https://bugs.python.org/issue16308 + https://github.com/iterative/dvc/issues/769 + + Args: + subparsers: subparsers to fix. + """ + if sys.version_info[0] == 3: # pragma: no cover + subparsers.required = True + subparsers.dest = 'cmd' + + class CmdBase(object): def __init__(self, args): from dvc.project import Project @@ -8,10 +23,19 @@ class CmdBase(object): self.project = Project() self.config = self.project.config self.args = args - self._set_loglevel(args) + self.set_loglevel(args) + + @property + def default_targets(self): + """Default targets for `dvc repro` and `dvc pipeline`.""" + from dvc.stage import Stage + msg = "assuming default target '{}'.".format(Stage.STAGE_FILE) + logger.warning(msg) + return [Stage.STAGE_FILE] @staticmethod - def _set_loglevel(args): + def set_loglevel(args): + """Sets log level from CLI arguments.""" if args.quiet: logger.be_quiet() diff --git a/dvc/command/cache.py b/dvc/command/cache.py index 1d31030b5..dee60f046 100644 --- a/dvc/command/cache.py +++ b/dvc/command/cache.py @@ -1,3 +1,6 @@ +import argparse + +from dvc.command.base import fix_subparsers from dvc.command.remote import CmdRemoteAdd from dvc.command.config import CmdConfig @@ -9,3 +12,44 @@ class CmdCacheDir(CmdConfig): self.configobj.filename) return super(CmdCacheDir, self).run() + + +def add_parser(subparsers, parent_parser): + from dvc.command.config import parent_config_parser + + CACHE_HELP = 'Manage cache settings.' + cache_parser = subparsers.add_parser( + 'cache', + parents=[parent_parser], + description=CACHE_HELP, + help=CACHE_HELP) + + cache_subparsers = cache_parser.add_subparsers( + dest='cmd', + help='Use dvc cache CMD --help for command-specific help.') + + fix_subparsers(cache_subparsers) + + parent_cache_config_parser = argparse.ArgumentParser( + add_help=False, + parents=[parent_config_parser]) + CACHE_DIR_HELP = 'Configure cache directory location.' + cache_dir_parser = cache_subparsers.add_parser( + 'dir', + parents=[parent_cache_config_parser], + description=CACHE_DIR_HELP, + help=CACHE_DIR_HELP) + cache_dir_parser.add_argument( + '-u', + '--unset', + default=False, + action='store_true', + help='Unset option.') + cache_dir_parser.add_argument( + 'value', + nargs='?', + default=None, + help="Path to cache directory. Relative paths are resolved relative " + "to the current directory and saved to config relative to the " + "config file location.") + cache_dir_parser.set_defaults(func=CmdCacheDir) diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index 55f6c53cc..89673ac8c 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -11,3 +11,29 @@ class CmdCheckout(CmdBase): with_deps=self.args.with_deps, force=self.args.force) return 0 + + +def add_parser(subparsers, parent_parser): + CHECKOUT_HELP = 'Checkout data files from cache.' + checkout_parser = subparsers.add_parser( + 'checkout', + parents=[parent_parser], + description=CHECKOUT_HELP, + help=CHECKOUT_HELP) + checkout_parser.add_argument( + '-d', + '--with-deps', + action='store_true', + default=False, + help='Checkout all dependencies of the specified target.') + checkout_parser.add_argument( + '-f', + '--force', + action='store_true', + default=False, + help='Do not prompt when removing working directory files.') + checkout_parser.add_argument( + 'targets', + nargs='*', + help='DVC files.') + checkout_parser.set_defaults(func=CmdCheckout) diff --git a/dvc/command/config.py b/dvc/command/config.py index 9e0ce946a..cc06b22fa 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -1,4 +1,5 @@ import os +import argparse import dvc.logger as logger from dvc.command.base import CmdBase @@ -13,7 +14,7 @@ class CmdConfig(CmdBase): self.args = args try: - dvc_dir = os.path.join(Project._find_root(), Project.DVC_DIR) + dvc_dir = os.path.join(Project.find_root(), Project.DVC_DIR) saved_exc = None except NotDvcProjectError as exc: dvc_dir = None @@ -75,3 +76,46 @@ class CmdConfig(CmdBase): return self._show(section, opt) else: return self._set(section, opt, self.args.value) + + +parent_config_parser = argparse.ArgumentParser(add_help=False) +parent_config_parser.add_argument( + '--global', + dest='glob', + action='store_true', + default=False, + help='Use global config.') +parent_config_parser.add_argument( + '--system', + action='store_true', + default=False, + help='Use system config.') +parent_config_parser.add_argument( + '--local', + action='store_true', + default=False, + help='Use local config.') + + +def add_parser(subparsers, parent_parser): + CONFIG_HELP = 'Get or set config options.' + config_parser = subparsers.add_parser( + 'config', + parents=[parent_config_parser, parent_parser], + description=CONFIG_HELP, + help=CONFIG_HELP) + config_parser.add_argument( + '-u', + '--unset', + default=False, + action='store_true', + help='Unset option.') + config_parser.add_argument( + 'name', + help='Option name.') + config_parser.add_argument( + 'value', + nargs='?', + default=None, + help='Option value.') + config_parser.set_defaults(func=CmdConfig) diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index eb3802577..01228880a 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -1,11 +1,11 @@ -from dvc.command.base import CmdBase +from dvc.command.base import CmdBase, fix_subparsers class CmdDaemonBase(CmdBase): def __init__(self, args): self.args = args self.config = None - self._set_loglevel(args) + self.set_loglevel(args) def run_cmd(self): return self.run() @@ -17,7 +17,7 @@ class CmdDaemonUpdater(CmdDaemonBase): from dvc.project import Project from dvc.updater import Updater - root_dir = Project._find_root() + root_dir = Project.find_root() dvc_dir = os.path.join(root_dir, Project.DVC_DIR) updater = Updater(dvc_dir) updater.fetch(detach=False) @@ -33,3 +33,37 @@ class CmdDaemonAnalytics(CmdDaemonBase): analytics.send() return 0 + + +def add_parser(subparsers, parent_parser): + DAEMON_HELP = 'Service daemon.' + daemon_parser = subparsers.add_parser( + 'daemon', + parents=[parent_parser], + description=DAEMON_HELP, + help=DAEMON_HELP) + + daemon_subparsers = daemon_parser.add_subparsers( + dest='cmd', + help='Use dvc daemon CMD --help for command-specific help.') + + fix_subparsers(daemon_subparsers) + + DAEMON_UPDATER_HELP = 'Fetch latest available version.' + daemon_updater_parser = daemon_subparsers.add_parser( + 'updater', + parents=[parent_parser], + description=DAEMON_UPDATER_HELP, + help=DAEMON_UPDATER_HELP) + daemon_updater_parser.set_defaults(func=CmdDaemonUpdater) + + DAEMON_ANALYTICS_HELP = 'Send dvc usage analytics.' + daemon_analytics_parser = daemon_subparsers.add_parser( + 'analytics', + parents=[parent_parser], + description=DAEMON_ANALYTICS_HELP, + help=DAEMON_ANALYTICS_HELP) + daemon_analytics_parser.add_argument( + 'target', + help="Analytics file.") + daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics) diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 31398d2d0..26806d29b 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -1,5 +1,7 @@ -from dvc.command.base import CmdBase +import argparse + import dvc.logger as logger +from dvc.command.base import CmdBase class CmdDataBase(CmdBase): @@ -67,3 +69,179 @@ class CmdDataFetch(CmdDataBase): logger.error('failed to fetch data from the cloud') return 1 return 0 + + +def add_parser(subparsers, parent_parser): + from dvc.command.status import CmdDataStatus + + # Parent parser used in pull/push/status + parent_cache_parser = argparse.ArgumentParser( + add_help=False, + parents=[parent_parser]) + parent_cache_parser.add_argument( + '-j', + '--jobs', + type=int, + default=None, + help='Number of jobs to run simultaneously.') + parent_cache_parser.add_argument( + '--show-checksums', + action='store_true', + default=False, + help='Show checksums instead of file names.') + parent_cache_parser.add_argument( + 'targets', + nargs='*', + default=None, + help='DVC files.') + + # Pull + PULL_HELP = 'Pull data files from the cloud.' + pull_parser = subparsers.add_parser( + 'pull', + parents=[parent_cache_parser], + description=PULL_HELP, + help=PULL_HELP) + pull_parser.add_argument( + '-r', + '--remote', + help='Remote repository to pull from.') + pull_parser.add_argument( + '-a', + '--all-branches', + action='store_true', + default=False, + help='Fetch cache for all branches.') + pull_parser.add_argument( + '-T', + '--all-tags', + action='store_true', + default=False, + help='Fetch cache for all tags.') + pull_parser.add_argument( + '-d', + '--with-deps', + action='store_true', + default=False, + help='Fetch cache for all dependencies of the specified target.') + pull_parser.add_argument( + '-f', + '--force', + action='store_true', + default=False, + help='Do not prompt when removing working directory files.') + pull_parser.add_argument( + '-R', + '--recursive', + help='Pull cache for subdirectories of the specified directory.') + pull_parser.set_defaults(func=CmdDataPull) + + # Push + PUSH_HELP = 'Push data files to the cloud.' + push_parser = subparsers.add_parser( + 'push', + parents=[parent_cache_parser], + description=PUSH_HELP, + help=PUSH_HELP) + push_parser.add_argument( + '-r', + '--remote', + help='Remote repository to push to.') + push_parser.add_argument( + '-a', + '--all-branches', + action='store_true', + default=False, + help='Push cache for all branches.') + push_parser.add_argument( + '-T', + '--all-tags', + action='store_true', + default=False, + help='Push cache for all tags.') + push_parser.add_argument( + '-d', + '--with-deps', + action='store_true', + default=False, + help='Push cache for all dependencies of the specified target.') + push_parser.add_argument( + '-R', + '--recursive', + help='Push cache from subdirectories of specified directory.') + push_parser.set_defaults(func=CmdDataPush) + + # Fetch + FETCH_HELP = 'Fetch data files from the cloud.' + fetch_parser = subparsers.add_parser( + 'fetch', + parents=[parent_cache_parser], + description=FETCH_HELP, + help=FETCH_HELP) + fetch_parser.add_argument( + '-r', + '--remote', + help='Remote repository to fetch from.') + fetch_parser.add_argument( + '-a', + '--all-branches', + action='store_true', + default=False, + help='Fetch cache for all branches.') + fetch_parser.add_argument( + '-T', + '--all-tags', + action='store_true', + default=False, + help='Fetch cache for all tags.') + fetch_parser.add_argument( + '-d', + '--with-deps', + action='store_true', + default=False, + help='Fetch cache for all dependencies of the ' + 'specified target.') + fetch_parser.add_argument( + '-R', + '--recursive', + help='Fetch cache for subdirectories of specified directory.') + fetch_parser.set_defaults(func=CmdDataFetch) + + # Status + STATUS_HELP = 'Show the project status.' + status_parser = subparsers.add_parser( + 'status', + parents=[parent_cache_parser], + description=STATUS_HELP, + help=STATUS_HELP) + status_parser.add_argument( + '-c', + '--cloud', + action='store_true', + default=False, + help='Show status of a local cache compared to a remote repository.') + status_parser.add_argument( + '-r', + '--remote', + help='Remote repository to compare local cache to.') + status_parser.add_argument( + '-a', + '--all-branches', + action='store_true', + default=False, + help='Show status of a local cache compared to a remote repository ' + 'for all branches.') + status_parser.add_argument( + '-T', + '--all-tags', + action='store_true', + default=False, + help='Show status of a local cache compared to a remote repository ' + 'for all tags.') + status_parser.add_argument( + '-d', + '--with-deps', + action='store_true', + default=False, + help='Show status for all dependencies of the specified target.') + status_parser.set_defaults(func=CmdDataStatus) diff --git a/dvc/command/destroy.py b/dvc/command/destroy.py index afd7a0449..c7676b53d 100644 --- a/dvc/command/destroy.py +++ b/dvc/command/destroy.py @@ -25,3 +25,20 @@ class CmdDestroy(CmdBase): logger.error('failed to destroy DVC') return 1 return 0 + + +def add_parser(subparsers, parent_parser): + DESTROY_HELP = "Destroy dvc. Will remove all project's information, " \ + "data files and cache." + destroy_parser = subparsers.add_parser( + 'destroy', + parents=[parent_parser], + description=DESTROY_HELP, + help=DESTROY_HELP) + destroy_parser.add_argument( + '-f', + '--force', + action='store_true', + default=False, + help='Force destruction.') + destroy_parser.set_defaults(func=CmdDestroy) diff --git a/dvc/command/gc.py b/dvc/command/gc.py index 84c6cd4e5..697024420 100644 --- a/dvc/command/gc.py +++ b/dvc/command/gc.py @@ -39,3 +39,54 @@ class CmdGC(CmdBase): jobs=self.args.jobs, projects=self.args.projects) return 0 + + +def add_parser(subparsers, parent_parser): + GC_HELP = 'Collect garbage.' + gc_parser = subparsers.add_parser( + 'gc', + parents=[parent_parser], + description=GC_HELP, + help=GC_HELP) + gc_parser.add_argument( + '-a', + '--all-branches', + action='store_true', + default=False, + help='Collect garbage for all branches.') + gc_parser.add_argument( + '-T', + '--all-tags', + action='store_true', + default=False, + help='Collect garbage for all tags.') + gc_parser.add_argument( + '-c', + '--cloud', + action='store_true', + default=False, + help='Collect garbage in remote repository.') + gc_parser.add_argument( + '-r', + '--remote', + help='Remote repository to collect garbage in.') + gc_parser.add_argument( + '-f', + '--force', + action='store_true', + default=False, + help='Force garbage collection.') + gc_parser.add_argument( + '-j', + '--jobs', + type=int, + default=None, + help='Number of jobs to run simultaneously.') + gc_parser.add_argument( + '-p', + '--projects', + type=str, + nargs='*', + default=None, + help='Collect garbage for all given projects.') + gc_parser.set_defaults(func=CmdGC) diff --git a/dvc/command/imp.py b/dvc/command/imp.py index 46e497490..c4eaac4b7 100644 --- a/dvc/command/imp.py +++ b/dvc/command/imp.py @@ -22,3 +22,29 @@ class CmdImport(CmdBase): .format(self.args.url)) return 1 return 0 + + +def add_parser(subparsers, parent_parser): + IMPORT_HELP = 'Import files from URL.' + import_parser = subparsers.add_parser( + 'import', + parents=[parent_parser], + description=IMPORT_HELP, + help=IMPORT_HELP) + import_parser.add_argument( + 'url', + help="URL. Supported urls: " + "'/path/to/file', " + "'C:\\\\path\\to\\file', " + "'https://example.com/path/to/file', " + "'s3://bucket/path/to/file', " + "'gs://bucket/path/to/file', " + "'hdfs://example.com/path/to/file', " + "'ssh://example.com:/path/to/file', " + "'remote://myremote/path/to/file'(see " + "`dvc remote` commands). ") + import_parser.add_argument( + 'out', + nargs='?', + help='Output.') + import_parser.set_defaults(func=CmdImport) diff --git a/dvc/command/init.py b/dvc/command/init.py index 932d27916..952f05573 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -17,3 +17,27 @@ class CmdInit(object): logger.error('failed to initiate dvc') return 1 return 0 + + +def add_parser(subparsers, parent_parser): + """Setup parser for `dvc init`.""" + INIT_HELP = 'Initialize dvc over a directory ' \ + '(should already be a git dir).' + init_parser = subparsers.add_parser( + 'init', + parents=[parent_parser], + description=INIT_HELP, + help=INIT_HELP) + init_parser.add_argument( + '--no-scm', + action='store_true', + default=False, + help="Initiate dvc in directory that is " + "not tracked by any scm tool (e.g. git).") + init_parser.add_argument( + '-f', + '--force', + action='store_true', + default=False, + help="Overwrite '.dvc' if it exists. Will remove all local cache.") + init_parser.set_defaults(func=CmdInit) diff --git a/dvc/command/install.py b/dvc/command/install.py index 1a2e0dda1..47be9eb31 100644 --- a/dvc/command/install.py +++ b/dvc/command/install.py @@ -10,3 +10,13 @@ class CmdInstall(CmdBase): logger.error('failed to install dvc hooks') return 1 return 0 + + +def add_parser(subparsers, parent_parser): + INSTALL_HELP = 'Install dvc hooks into the repository.' + install_parser = subparsers.add_parser( + 'install', + parents=[parent_parser], + description=INSTALL_HELP, + help=INSTALL_HELP) + install_parser.set_defaults(func=CmdInstall) diff --git a/dvc/command/lock.py b/dvc/command/lock.py index faaa23e42..9ae847b01 100644 --- a/dvc/command/lock.py +++ b/dvc/command/lock.py @@ -26,3 +26,29 @@ class CmdLock(CmdLockBase): class CmdUnlock(CmdLockBase): def run(self): return super(CmdUnlock, self).run(True) + + +def add_parser(subparsers, parent_parser): + LOCK_HELP = 'Lock DVC file.' + lock_parser = subparsers.add_parser( + 'lock', + parents=[parent_parser], + description=LOCK_HELP, + help=LOCK_HELP) + lock_parser.add_argument( + 'targets', + nargs='+', + help='DVC files.') + lock_parser.set_defaults(func=CmdLock) + + UNLOCK_HELP = 'Unlock DVC file.' + unlock_parser = subparsers.add_parser( + 'unlock', + parents=[parent_parser], + description=UNLOCK_HELP, + help=UNLOCK_HELP) + unlock_parser.add_argument( + 'targets', + nargs='+', + help='DVC files.') + unlock_parser.set_defaults(func=CmdUnlock) diff --git a/dvc/command/metrics.py b/dvc/command/metrics.py index 98012eeeb..36a5b4be7 100644 --- a/dvc/command/metrics.py +++ b/dvc/command/metrics.py @@ -1,6 +1,6 @@ import dvc.logger as logger -from dvc.command.base import CmdBase from dvc.exceptions import DvcException +from dvc.command.base import CmdBase, fix_subparsers class CmdMetricsShow(CmdBase): @@ -75,3 +75,115 @@ class CmdMetricsRemove(CmdBase): return 1 return 0 + + +def add_parser(subparsers, parent_parser): + METRICS_HELP = 'Get metrics from all branches.' + metrics_parser = subparsers.add_parser( + 'metrics', + parents=[parent_parser], + description=METRICS_HELP, + help=METRICS_HELP) + + metrics_subparsers = metrics_parser.add_subparsers( + dest='cmd', + help="Use dvc metrics CMD --help for command-specific help.") + + fix_subparsers(metrics_subparsers) + + METRICS_SHOW_HELP = 'Show metrics.' + metrics_show_parser = metrics_subparsers.add_parser( + 'show', + parents=[parent_parser], + description=METRICS_SHOW_HELP, + help=METRICS_SHOW_HELP) + metrics_show_parser.add_argument( + 'path', + nargs='?', + help='Path to metrics file.') + metrics_show_parser.add_argument( + '-t', + '--type', + help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).') + metrics_show_parser.add_argument( + '-x', + '--xpath', + help='JSON/TSV/HTSV/CSV/HCSV path.') + metrics_show_group = metrics_show_parser.add_mutually_exclusive_group() + metrics_show_group.add_argument( + '--json-path', + help='JSON path.') + metrics_show_group.add_argument( + '--tsv-path', + help="TSV path 'row,column' (e.g. '1,2').") + metrics_show_group.add_argument( + '--htsv-path', + help="Headed TSV path 'row,column (e.g. 'Name,3').") + metrics_show_group.add_argument( + '--csv-path', + help="CSV path 'row,column' (e.g. '1,2').") + metrics_show_group.add_argument( + '--hcsv-path', + help="Headed CSV path 'row,column' (e.g. 'Name,3').") + metrics_show_parser.add_argument( + '-a', + '--all-branches', + action='store_true', + default=False, + help='Show metrics for all branches.') + metrics_show_parser.add_argument( + '-T', + '--all-tags', + action='store_true', + default=False, + help='Show metrics for all tags.') + metrics_show_parser.set_defaults(func=CmdMetricsShow) + + METRICS_ADD_HELP = 'Add metrics.' + metrics_add_parser = metrics_subparsers.add_parser( + 'add', + parents=[parent_parser], + description=METRICS_ADD_HELP, + help=METRICS_ADD_HELP) + metrics_add_parser.add_argument( + '-t', + '--type', + help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).') + metrics_add_parser.add_argument( + '-x', + '--xpath', + help='JSON/TSV/HTSV/CSV/HCSV path.') + metrics_add_parser.add_argument( + 'path', + help='Path to metrics file.') + metrics_add_parser.set_defaults(func=CmdMetricsAdd) + + METRICS_MODIFY_HELP = 'Modify metrics.' + metrics_modify_parser = metrics_subparsers.add_parser( + 'modify', + parents=[parent_parser], + description=METRICS_MODIFY_HELP, + help=METRICS_MODIFY_HELP) + metrics_modify_parser.add_argument( + '-t', + '--type', + help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).') + metrics_modify_parser.add_argument( + '-x', + '--xpath', + help='JSON/TSV/HTSV/CSV/HCSV path.') + metrics_modify_parser.add_argument( + 'path', + help='Metrics file.') + metrics_modify_parser.set_defaults(func=CmdMetricsModify) + + METRICS_REMOVE_HELP = 'Remove metrics.' + metrics_remove_parser = metrics_subparsers.add_parser( + 'remove', + parents=[parent_parser], + description=METRICS_REMOVE_HELP, + help=METRICS_REMOVE_HELP) + metrics_remove_parser.add_argument( + 'path', + help='Path to metrics file.') + metrics_remove_parser.set_defaults(func=CmdMetricsRemove) diff --git a/dvc/command/move.py b/dvc/command/move.py index a76706fe5..803e4d4a5 100644 --- a/dvc/command/move.py +++ b/dvc/command/move.py @@ -13,3 +13,15 @@ class CmdMove(CmdBase): logger.error(msg) return 1 return 0 + + +def add_parser(subparsers, parent_parser): + MOVE_HELP = 'Move output of DVC file.' + move_parser = subparsers.add_parser( + 'move', + parents=[parent_parser], + description=MOVE_HELP, + help=MOVE_HELP) + move_parser.add_argument('src', help='Source.') + move_parser.add_argument('dst', help='Destination.') + move_parser.set_defaults(func=CmdMove) diff --git a/dvc/command/pipeline.py b/dvc/command/pipeline.py index 37ceeb172..e15f664c9 100644 --- a/dvc/command/pipeline.py +++ b/dvc/command/pipeline.py @@ -2,7 +2,7 @@ import os import dvc.logger as logger from dvc.exceptions import DvcException -from dvc.command.base import CmdBase +from dvc.command.base import CmdBase, fix_subparsers class CmdPipelineShow(CmdBase): @@ -70,15 +70,14 @@ class CmdPipelineShow(CmdBase): return nodes, edges def _show_ascii(self, target, commands, outs): - from dvc.dagascii import Dagascii + from dvc.dagascii import draw nodes, edges = self.__build_graph(target, commands, outs) if not nodes: return - d = Dagascii(nodes, edges) - d.draw() + draw(nodes, edges) def __write_dot(self, target, commands, outs, filename): import networkx @@ -92,6 +91,9 @@ class CmdPipelineShow(CmdBase): write_dot(simple_g, filename) def run(self, unlock=False): + if not self.args.targets: + self.args.targets = self.default_targets + for target in self.args.targets: try: if self.args.ascii: @@ -128,3 +130,59 @@ class CmdPipelineList(CmdBase): logger.info('{} pipeline(s) total'.format(len(pipelines))) return 0 + + +def add_parser(subparsers, parent_parser): + PIPELINE_HELP = 'Manage pipeline.' + pipeline_parser = subparsers.add_parser( + 'pipeline', + parents=[parent_parser], + description=PIPELINE_HELP, + help=PIPELINE_HELP) + + pipeline_subparsers = pipeline_parser.add_subparsers( + dest='cmd', + help="Use dvc pipeline CMD --help for command-specific help.") + + fix_subparsers(pipeline_subparsers) + + PIPELINE_SHOW_HELP = 'Show pipeline.' + pipeline_show_parser = pipeline_subparsers.add_parser( + 'show', + parents=[parent_parser], + description=PIPELINE_SHOW_HELP, + help=PIPELINE_SHOW_HELP) + pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group() + pipeline_show_group.add_argument( + '-c', + '--commands', + action='store_true', + default=False, + help='Print commands instead of paths to DVC files.') + pipeline_show_group.add_argument( + '-o', + '--outs', + action='store_true', + default=False, + help='Print output files instead of paths to DVC files.') + pipeline_show_parser.add_argument( + '--ascii', + action='store_true', + default=False, + help='Output DAG as ASCII.') + pipeline_show_parser.add_argument( + '--dot', + help='Write DAG in .dot format.') + pipeline_show_parser.add_argument( + 'targets', + nargs='*', + help="DVC files. 'Dvcfile' by default.") + pipeline_show_parser.set_defaults(func=CmdPipelineShow) + + PIPELINE_LIST_HELP = 'List pipelines.' + pipeline_list_parser = pipeline_subparsers.add_parser( + 'list', + parents=[parent_parser], + description=PIPELINE_LIST_HELP, + help=PIPELINE_LIST_HELP) + pipeline_list_parser.set_defaults(func=CmdPipelineList) diff --git a/dvc/command/remote.py b/dvc/command/remote.py index b0b50ec0a..9bf83b03d 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -2,6 +2,7 @@ import os import re import dvc.logger as logger +from dvc.command.base import fix_subparsers from dvc.config import Config from dvc.remote import _get, RemoteLOCAL from dvc.command.config import CmdConfig @@ -104,3 +105,103 @@ class CmdRemoteList(CmdConfig): '') logger.info('{}\t{}'.format(name, url)) return 0 + + +def add_parser(subparsers, parent_parser): + from dvc.command.config import parent_config_parser + + REMOTE_HELP = 'Manage set of tracked repositories.' + remote_parser = subparsers.add_parser( + 'remote', + parents=[parent_parser], + description=REMOTE_HELP, + help=REMOTE_HELP) + + remote_subparsers = remote_parser.add_subparsers( + dest='cmd', + help='Use dvc remote CMD --help for ' + 'command-specific help.') + + fix_subparsers(remote_subparsers) + + REMOTE_ADD_HELP = 'Add remote.' + remote_add_parser = remote_subparsers.add_parser( + 'add', + parents=[parent_config_parser, parent_parser], + description=REMOTE_ADD_HELP, + help=REMOTE_ADD_HELP) + remote_add_parser.add_argument( + 'name', + help='Name.') + remote_add_parser.add_argument( + 'url', + help='URL. See full list of supported urls at ' + 'https://dvc.org/doc/commands-reference/remote') + remote_add_parser.add_argument( + '-d', + '--default', + action='store_true', + default=False, + help='Set as default remote.') + remote_add_parser.set_defaults(func=CmdRemoteAdd) + + REMOTE_DEFAULT_HELP = 'Set/unset default remote.' + remote_default_parser = remote_subparsers.add_parser( + 'default', + parents=[parent_config_parser, parent_parser], + description=REMOTE_DEFAULT_HELP, + help=REMOTE_DEFAULT_HELP) + remote_default_parser.add_argument( + 'name', + nargs='?', + help='Name of the remote.') + remote_default_parser.add_argument( + '-u', + '--unset', + action='store_true', + default=False, + help='Unset default remote.') + remote_default_parser.set_defaults(func=CmdRemoteDefault) + + REMOTE_REMOVE_HELP = 'Remove remote.' + remote_remove_parser = remote_subparsers.add_parser( + 'remove', + parents=[parent_config_parser, parent_parser], + description=REMOTE_REMOVE_HELP, + help=REMOTE_REMOVE_HELP) + remote_remove_parser.add_argument( + 'name', + help='Name') + remote_remove_parser.set_defaults(func=CmdRemoteRemove) + + REMOTE_MODIFY_HELP = 'Modify remote.' + remote_modify_parser = remote_subparsers.add_parser( + 'modify', + parents=[parent_config_parser, parent_parser], + description=REMOTE_MODIFY_HELP, + help=REMOTE_MODIFY_HELP) + remote_modify_parser.add_argument( + 'name', + help='Name.') + remote_modify_parser.add_argument( + 'option', + help='Option.') + remote_modify_parser.add_argument( + 'value', + nargs='?', + help='Value.') + remote_modify_parser.add_argument( + '-u', + '--unset', + default=False, + action='store_true', + help='Unset option.') + remote_modify_parser.set_defaults(func=CmdRemoteModify) + + REMOTE_LIST_HELP = 'List remotes.' + remote_list_parser = remote_subparsers.add_parser( + 'list', + parents=[parent_config_parser, parent_parser], + description=REMOTE_LIST_HELP, + help=REMOTE_LIST_HELP) + remote_list_parser.set_defaults(func=CmdRemoteList) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index 3a2396ba1..2f0a5594a 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -34,3 +34,36 @@ class CmdRemove(CmdBase): logger.error('failed to remove {}'.format(target)) return 1 return 0 + + +def add_parser(subparsers, parent_parser): + REMOVE_HELP = 'Remove outputs of DVC file.' + remove_parser = subparsers.add_parser( + 'remove', + parents=[parent_parser], + description=REMOVE_HELP, + help=REMOVE_HELP) + remove_parser_group = remove_parser.add_mutually_exclusive_group() + remove_parser_group.add_argument( + '-o', + '--outs', + action='store_true', + default=True, + help='Only remove DVC file outputs.(default)') + remove_parser_group.add_argument( + '-p', + '--purge', + action='store_true', + default=False, + help='Remove DVC file and all its outputs.') + remove_parser.add_argument( + '-f', + '--force', + action='store_true', + default=False, + help='Force purge.') + remove_parser.add_argument( + 'targets', + nargs='+', + help='DVC files.') + remove_parser.set_defaults(func=CmdRemove) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 41679c338..b7afe0452 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -16,6 +16,8 @@ class CmdRepro(CmdBase): # Dirty hack so the for loop below can at least enter once if self.args.all_pipelines: self.args.targets = [None] + elif not self.args.targets: + self.args.targets = self.default_targets ret = 0 for target in self.args.targets: @@ -42,3 +44,72 @@ class CmdRepro(CmdBase): os.chdir(saved_dir) return ret + + +def add_parser(subparsers, parent_parser): + REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'." + repro_parser = subparsers.add_parser( + 'repro', + parents=[parent_parser], + description=REPRO_HELP, + help=REPRO_HELP) + repro_parser.add_argument( + 'targets', + nargs='*', + help='DVC file to reproduce.') + repro_parser.add_argument( + '-f', + '--force', + action='store_true', + default=False, + help='Reproduce even if dependencies were not changed.') + repro_parser.add_argument( + '-s', + '--single-item', + action='store_true', + default=False, + help='Reproduce only single data item without recursive dependencies ' + 'check.') + repro_parser.add_argument( + '-c', + '--cwd', + default=os.path.curdir, + help='Directory within your project to reroduce from.') + repro_parser.add_argument( + '-m', + '--metrics', + action='store_true', + default=False, + help='Show metrics after reproduction.') + repro_parser.add_argument( + '--dry', + action='store_true', + default=False, + help='Only print the commands that would be executed without ' + 'actually executing.') + repro_parser.add_argument( + '-i', + '--interactive', + action='store_true', + default=False, + help='Ask for confirmation before reproducing each stage.') + repro_parser.add_argument( + '-p', + '--pipeline', + action='store_true', + default=False, + help='Reproduce the whole pipeline that the specified stage file ' + 'belongs to.') + repro_parser.add_argument( + '-P', + '--all-pipelines', + action='store_true', + default=False, + help='Reproduce all pipelines in the project.') + repro_parser.add_argument( + '--ignore-build-cache', + action='store_true', + default=False, + help="Reproduce all descendants of a changed stage even if their " + "direct dependencies didn't change.") + repro_parser.set_defaults(func=CmdRepro) diff --git a/dvc/command/root.py b/dvc/command/root.py index c4cac440e..46d177f51 100644 --- a/dvc/command/root.py +++ b/dvc/command/root.py @@ -11,3 +11,13 @@ class CmdRoot(CmdBase): def run(self): logger.info(os.path.relpath(self.project.root_dir)) return 0 + + +def add_parser(subparsers, parent_parser): + ROOT_HELP = "Relative path to project's directory." + root_parser = subparsers.add_parser( + 'root', + parents=[parent_parser], + description=ROOT_HELP, + help=ROOT_HELP) + root_parser.set_defaults(func=CmdRoot) diff --git a/dvc/command/run.py b/dvc/command/run.py index f487523da..983a26bd5 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -1,3 +1,6 @@ +import os +import argparse + import dvc.logger as logger from dvc.command.base import CmdBase from dvc.exceptions import DvcException @@ -7,6 +10,14 @@ class CmdRun(CmdBase): def run(self): overwrite = (self.args.yes or self.args.overwrite_dvcfile) + if not any([self.args.deps, + self.args.outs, + self.args.outs_no_cache, + self.args.command]): # pragma: no cover + logger.error("too few arguments. Specify at least one: '-d', " + "'-o', '-O', 'command'.") + return 1 + try: self.project.run(cmd=self._parsed_cmd(), outs=self.args.outs, @@ -45,3 +56,88 @@ class CmdRun(CmdBase): return argument return '"{}"'.format(argument) + + +def add_parser(subparsers, parent_parser): + RUN_HELP = 'Generate a stage file from a given ' \ + 'command and execute the command.' + run_parser = subparsers.add_parser( + 'run', + parents=[parent_parser], + description=RUN_HELP, + help=RUN_HELP) + run_parser.add_argument( + '-d', + '--deps', + action='append', + default=[], + help='Declare dependencies for reproducible cmd.') + run_parser.add_argument( + '-o', + '--outs', + action='append', + default=[], + help='Declare output data file or data directory.') + run_parser.add_argument( + '-O', + '--outs-no-cache', + action='append', + default=[], + help='Declare output regular file or directory ' + '(sync to Git, not DVC cache).') + run_parser.add_argument( + '-M', + '--metrics-no-cache', + action='append', + default=[], + help='Declare output metric file or ' + 'directory (not cached by DVC).') + run_parser.add_argument( + '-f', + '--file', + help="Specify name of the stage file. It should be " + "either 'Dvcfile' or have a '.dvc' suffix (e.g. " + "'prepare.dvc', 'clean.dvc', etc) in order for " + "dvc to be able to find it later. By default " + "the first output basename + .dvc is used as " + "a stage filename.") + run_parser.add_argument( + '-c', + '--cwd', + default=os.path.curdir, + help="Directory within your project to run your command and place " + "stage file in.") + run_parser.add_argument( + '--no-exec', + action='store_true', + default=False, + help="Only create stage file without actually running it.") + run_parser.add_argument( + '-y', + '--yes', + action='store_true', + default=False, + help="(OBSOLETED, use --overwrite-dvcfile instead) Automatic 'yes' " + "answer to all prompts. E.g. when '.dvc' file exists and dvc " + "asks if you want to overwrite it.") + run_parser.add_argument( + '--overwrite-dvcfile', + action='store_true', + default=False, + help="Overwrite existing dvc file without asking for confirmation.") + run_parser.add_argument( + '--ignore-build-cache', + action='store_true', + default=False, + help="Run this stage even if it has been already ran with the same " + "command/dependencies/outputs/etc before.") + run_parser.add_argument( + '--remove-outs', + action='store_true', + default=False, + help="Remove outputs before running the command.") + run_parser.add_argument( + 'command', + nargs=argparse.REMAINDER, + help='Command to execute.') + run_parser.set_defaults(func=CmdRun) diff --git a/dvc/command/unprotect.py b/dvc/command/unprotect.py index cb1efcb01..98e910c87 100644 --- a/dvc/command/unprotect.py +++ b/dvc/command/unprotect.py @@ -13,3 +13,17 @@ class CmdUnprotect(CmdBase): logger.error(msg) return 1 return 0 + + +def add_parser(subparsers, parent_parser): + UNPROTECT_HELP = 'Unprotect data file/directory.' + unprotect_parser = subparsers.add_parser( + 'unprotect', + parents=[parent_parser], + description=UNPROTECT_HELP, + help=UNPROTECT_HELP) + unprotect_parser.add_argument( + 'targets', + nargs='+', + help='Data files/directory.') + unprotect_parser.set_defaults(func=CmdUnprotect) diff --git a/dvc/config.py b/dvc/config.py index 84463d1ee..aed75bebe 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -11,46 +11,111 @@ from dvc.exceptions import DvcException class ConfigError(DvcException): - """ DVC config exception """ + """DVC config exception. + + Args: + msg (str): error message. + ex (Exception): optional exception that has caused this error. + """ def __init__(self, msg, ex=None): super(ConfigError, self).__init__('config file error: {}'.format(msg), ex) def supported_cache_type(types): + """Checks if link type config option has a valid value. + + Args: + types (list/string): type(s) of links that dvc should try out. + """ if isinstance(types, str): - types = [t.strip() for t in types.split(',')] - for t in types: - if t not in ['reflink', 'hardlink', 'symlink', 'copy']: + types = [typ.strip() for typ in types.split(',')] + for typ in types: + if typ not in ['reflink', 'hardlink', 'symlink', 'copy']: return False return True def supported_loglevel(level): + """Checks if log level config option has a valid value. + + Args: + level (str): log level name. + """ return level in ['info', 'debug', 'warning', 'error'] def supported_cloud(cloud): + """Checks if obsoleted cloud option has a valid value. + + Args: + cloud (str): cloud type name. + """ return cloud in ['aws', 'gcp', 'local', ''] def is_bool(val): + """Checks that value is a boolean. + + Args: + val (str): string value verify. + + Returns: + bool: True if value stands for boolean, False otherwise. + """ return val.lower() in ['true', 'false'] def to_bool(val): + """Converts value to boolean. + + Args: + val (str): string to convert to boolean. + + Returns: + bool: True if value.lower() == 'true', False otherwise. + """ return val.lower() == 'true' def is_whole(val): + """Checks that value is a whole integer. + + Args: + val (str): number string to verify. + + Returns: + bool: True if val is a whole number, False otherwise. + """ return int(val) >= 0 def is_percent(val): + """Checks that value is a percent. + + Args: + val (str): number string to verify. + + Returns: + bool: True if 0<=value<=100, False otherwise. + """ return int(val) >= 0 and int(val) <= 100 -class Config(object): +class Config(object): # pylint: disable=too-many-instance-attributes + """Class that manages configuration files for a dvc project. + + Args: + dvc_dir (str): optional path to `.dvc` directory, that is used to + access project-specific configs like .dvc/config and + .dvc/config.local. + validate (bool): optional flag to tell dvc if it should validate the + config or just load it as is. 'True' by default. + + + Raises: + ConfigError: thrown when config has an invalid format. + """ APPNAME = 'dvc' APPAUTHOR = 'iterative' @@ -220,22 +285,47 @@ class Config(object): self.config_file = None self.config_local_file = None + self._system_config = None + self._global_config = None + self._project_config = None + self._local_config = None + + self.config = None + self.load(validate=validate) @staticmethod def get_global_config_dir(): + """Returns global config location. E.g. ~/.config/dvc/config. + + Returns: + str: path to the global config directory. + """ from appdirs import user_config_dir return user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) @staticmethod def get_system_config_dir(): + """Returns system config location. E.g. /etc/dvc.conf. + + Returns: + str: path to the system config directory. + """ from appdirs import site_config_dir return site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) @staticmethod def init(dvc_dir): + """Initializes dvc config. + + Args: + dvc_dir (str): path to .dvc directory. + + Returns: + dvc.config.Config: config object. + """ config_file = os.path.join(dvc_dir, Config.CONFIG) open(config_file, 'w+').close() return Config(dvc_dir) @@ -254,7 +344,7 @@ class Config(object): else: self._local_config = configobj.ConfigObj() - self._config = None + self.config = None def _load_config(self, path): config = configobj.ConfigObj(path) @@ -281,48 +371,67 @@ class Config(object): def _resolve_paths(self, config, fname): self._resolve_cache_path(config, fname) - for name, section in config.items(): + for section in config.values(): if self.SECTION_REMOTE_URL not in section.keys(): continue section[self.PRIVATE_CWD] = os.path.dirname(fname) def load(self, validate=True): + """Loads config from all the config files. + + Args: + validate (bool): optional flag to tell dvc if it should validate + the config or just load it as is. 'True' by default. + + + Raises: + dvc.config.ConfigError: thrown if config has invalid format. + """ self._load() try: - self._config = self._load_config(self.system_config_file) + self.config = self._load_config(self.system_config_file) user = self._load_config(self.global_config_file) config = self._load_config(self.config_file) local = self._load_config(self.config_local_file) # NOTE: schema doesn't support ConfigObj.Section validation, so we # need to convert our config to dict before passing it to - for c in [user, config, local]: - self._config = self._merge(self._config, c) + for conf in [user, config, local]: + self.config = self._merge(self.config, conf) if validate: - self._config = Schema(self.SCHEMA).validate(self._config) + self.config = Schema(self.SCHEMA).validate(self.config) # NOTE: now converting back to ConfigObj - self._config = configobj.ConfigObj(self._config, - write_empty_values=True) - self._config.filename = self.config_file - self._resolve_paths(self._config, self.config_file) + self.config = configobj.ConfigObj(self.config, + write_empty_values=True) + self.config.filename = self.config_file + self._resolve_paths(self.config, self.config_file) except Exception as ex: raise ConfigError(ex) - def _get_key(self, d, name, add=False): - for k in d.keys(): + @staticmethod + def _get_key(conf, name, add=False): + for k in conf.keys(): if k.lower() == name.lower(): return k if add: - d[name] = {} + conf[name] = {} return name return None def save(self, config=None): + """Saves config to config files. + + Args: + config (configobj.ConfigObj): optional config object to save. + + Raises: + dvc.config.ConfigError: thrown if failed to write config file. + """ if config is not None: clist = [config] else: @@ -348,7 +457,15 @@ class Config(object): msg = "failed to write config '{}'".format(conf.filename) raise ConfigError(msg, exc) - def unset(self, config, section, opt=None): + @staticmethod + def unset(config, section, opt=None): + """Unsets specified option and/or section in the config. + + Args: + config (configobj.ConfigObj): config to work on. + section (str): section name. + opt (str): optional option name. + """ if section not in config.keys(): raise ConfigError("section '{}' doesn't exist".format(section)) @@ -361,16 +478,33 @@ class Config(object): opt)) del config[section][opt] - if len(config[section]) == 0: + if not config[section]: del config[section] - def set(self, config, section, opt, value): + @staticmethod + def set(config, section, opt, value): + """Sets specified option in the config. + + Args: + config (configobj.ConfigObj): config to work on. + section (str): section name. + opt (str): option name. + value: value to set option to. + """ if section not in config.keys(): config[section] = {} config[section][opt] = value - def show(self, config, section, opt): + @staticmethod + def show(config, section, opt): + """Prints option value from the config. + + Args: + config (configobj.ConfigObj): config to work on. + section (str): section name. + opt (str): option name. + """ if section not in config.keys(): raise ConfigError("section '{}' doesn't exist".format(section)) @@ -385,10 +519,10 @@ class Config(object): res = {} sections = list(first.keys()) + list(second.keys()) for section in sections: - f = first.get(section, {}).copy() - s = second.get(section, {}).copy() - f.update(s) - res[section] = f + first_copy = first.get(section, {}).copy() + second_copy = second.get(section, {}).copy() + first_copy.update(second_copy) + res[section] = first_copy return res @staticmethod diff --git a/dvc/daemon.py b/dvc/daemon.py index e093bd497..86966db6d 100644 --- a/dvc/daemon.py +++ b/dvc/daemon.py @@ -1,3 +1,5 @@ +"""Launch `dvc daemon` command in a separate detached process.""" + import os import sys from subprocess import Popen @@ -6,75 +8,77 @@ import dvc.logger as logger from dvc.utils import is_binary, fix_env -class Daemon(object): # pragma: no cover - def _spawn_windows(self, cmd): - from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW +CREATE_NEW_PROCESS_GROUP = 0x00000200 +DETACHED_PROCESS = 0x00000008 - CREATE_NEW_PROCESS_GROUP = 0x00000200 - DETACHED_PROCESS = 0x00000008 - creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS - startupinfo = STARTUPINFO() - startupinfo.dwFlags |= STARTF_USESHOWWINDOW +def _spawn_windows(cmd): + from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW - p = Popen(cmd, - env=fix_env(), - close_fds=True, - shell=False, - creationflags=creationflags, - startupinfo=startupinfo) + creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS - p.communicate() + startupinfo = STARTUPINFO() + startupinfo.dwFlags |= STARTF_USESHOWWINDOW - def _spawn_posix(self, cmd): - # NOTE: using os._exit instead of sys.exit, because dvc built - # with PyInstaller has trouble with SystemExit exeption and throws - # errors such as "[26338] Failed to execute script __main__" - try: - pid = os.fork() - if pid > 0: - return - except OSError: - logger.error("failed at first fork") - os._exit(1) + Popen(cmd, + env=fix_env(), + close_fds=True, + shell=False, + creationflags=creationflags, + startupinfo=startupinfo).communicate() - os.setsid() - os.umask(0) - try: - pid = os.fork() - if pid > 0: - os._exit(0) - except OSError: - logger.error("failed at second fork") - os._exit(1) +@staticmethod +def _spawn_posix(cmd): + # NOTE: using os._exit instead of sys.exit, because dvc built + # with PyInstaller has trouble with SystemExit exeption and throws + # errors such as "[26338] Failed to execute script __main__" + try: + pid = os.fork() + if pid > 0: + return + except OSError: + logger.error("failed at first fork") + os._exit(1) # pylint: disable=protected-access - sys.stdin.close() - sys.stdout.close() - sys.stderr.close() + os.setsid() + os.umask(0) - p = Popen(cmd, - env=fix_env(), - close_fds=True, - shell=False) + try: + pid = os.fork() + if pid > 0: + os._exit(0) # pylint: disable=protected-access + except OSError: + logger.error("failed at second fork") + os._exit(1) # pylint: disable=protected-access - p.communicate() + sys.stdin.close() + sys.stdout.close() + sys.stderr.close() - os._exit(0) + Popen(cmd, env=fix_env(), close_fds=True, shell=False).communicate() - def __call__(self, args): - cmd = [sys.executable] - if not is_binary(): - cmd += ['-m', 'dvc'] - cmd += ['daemon', '-q'] + args + os._exit(0) # pylint: disable=protected-access - logger.debug("Trying to spawn '{}'".format(cmd)) - if os.name == 'nt': - self._spawn_windows(cmd) - elif os.name == 'posix': - self._spawn_posix(cmd) - else: - raise NotImplementedError +def daemon(args): + """Launch a `dvc daemon` command in a detached process. - logger.debug("Spawned '{}'".format(cmd)) + Args: + args (list): list of arguments to append to `dvc daemon` command. + """ + cmd = [sys.executable] + if not is_binary(): + cmd += ['-m', 'dvc'] + cmd += ['daemon', '-q'] + args + + logger.debug("Trying to spawn '{}'".format(cmd)) + + if os.name == 'nt': + _spawn_windows(cmd) + elif os.name == 'posix': + _spawn_posix(cmd) + else: + raise NotImplementedError + + logger.debug("Spawned '{}'".format(cmd)) diff --git a/dvc/dagascii.py b/dvc/dagascii.py index 68db1601a..6f0c458e5 100644 --- a/dvc/dagascii.py +++ b/dvc/dagascii.py @@ -1,3 +1,4 @@ +"""Draws DAG in ASCII.""" import sys import math @@ -6,7 +7,38 @@ from grandalf.layouts import SugiyamaLayout from grandalf.routing import route_with_lines, EdgeViewer +class VertexViewer(object): + """Class to define vertex box boundaries that will be accounted for during + graph building by grandalf. + + Args: + name (str): name of the vertex. + """ + HEIGHT = 3 # top and bottom box edges + text + + def __init__(self, name): + # pylint: disable=invalid-name + self._h = self.HEIGHT # top and bottom box edges + text + self._w = len(name) + 2 # right and left bottom edges + text + + @property + def h(self): # pylint: disable=invalid-name + """Height of the box.""" + return self._h + + @property + def w(self): # pylint: disable=invalid-name + """Width of the box.""" + return self._w + + class AsciiCanvas(object): + """Class for drawing in ASCII. + + Args: + cols (int): number of columns in the canvas. Should be > 1. + lines (int): number of lines in the canvas. Should be > 1. + """ TIMEOUT = 10 def __init__(self, cols, lines): @@ -19,6 +51,7 @@ class AsciiCanvas(object): self.canvas = [[' '] * cols for l in range(lines)] def draw(self): + """Draws ASCII canvas on the screen.""" if sys.stdout.isatty(): # pragma: no cover from asciimatics.screen import Screen Screen.wrapper(self._do_draw) @@ -27,6 +60,8 @@ class AsciiCanvas(object): print(''.join(line)) def _do_draw(self, screen): # pragma: no cover + # pylint: disable=too-many-locals + # pylint: disable=too-many-branches, too-many-statements from dvc.system import System from asciimatics.event import KeyboardEvent @@ -56,7 +91,7 @@ class AsciiCanvas(object): x_index = offset_x + x if len(self.canvas) > y_index \ and len(self.canvas[y_index]) > x_index: - line.append(self.canvas[y_index][x_index]) + line.append(self.canvas[y_index][x_index]) else: line.append(' ') assert len(line) == (smaxcol + 1) @@ -109,6 +144,17 @@ class AsciiCanvas(object): offset_x = 0 def point(self, x, y, char): + """Create a point on ASCII canvas. + + Args: + x (int): x coordinate. Should be >= 0 and < number of columns in + the canvas. + y (int): y coordinate. Should be >= 0 an < number of lines in the + canvas. + char (str): character to place in the specified point on the + canvas. + """ + assert len(char) == 1 assert x >= 0 assert x < self.cols assert y >= 0 @@ -117,6 +163,16 @@ class AsciiCanvas(object): self.canvas[y][x] = char def line(self, x0, y0, x1, y1, char): + """Create a line on ASCII canvas. + + Args: + x0 (int): x coordinate where the line should start. + y0 (int): y coordinate where the line should start. + x1 (int): x coordinate where the line should end. + y1 (int): y coordinate where the line should end. + char (str): character to draw the line with. + """ + # pylint: disable=too-many-arguments, too-many-branches if x0 > x1: x1, x0 = x0, x1 y1, y0 = y0, y1 @@ -149,138 +205,157 @@ class AsciiCanvas(object): self.point(x, y, char) def text(self, x, y, text): - for i, c in enumerate(text): - self.point(x + i, y, c) + """Print a text on ASCII canvas. - def box(self, x0, y0, w, h): - assert w > 1 - assert h > 1 + Args: + x (int): x coordinate where the text should start. + y (int): y coordinate where the text should start. + text (str): string that should be printed. + """ + for i, char in enumerate(text): + self.point(x + i, y, char) - w -= 1 - h -= 1 + def box(self, x0, y0, width, height): + """Create a box on ASCII canvas. - for x in range(x0, x0 + w): + Args: + x0 (int): x coordinate of the box corner. + y0 (int): y coordinate of the box corner. + width (int): box width. + height (int): box height. + """ + assert width > 1 + assert height > 1 + + width -= 1 + height -= 1 + + for x in range(x0, x0 + width): self.point(x, y0, '-') - self.point(x, y0 + h, '-') + self.point(x, y0 + height, '-') - for y in range(y0, y0 + h): + for y in range(y0, y0 + height): self.point(x0, y, '|') - self.point(x0 + w, y, '|') + self.point(x0 + width, y, '|') self.point(x0, y0, '+') - self.point(x0 + w, y0, '+') - self.point(x0, y0 + h, '+') - self.point(x0 + w, y0 + h, '+') + self.point(x0 + width, y0, '+') + self.point(x0, y0 + height, '+') + self.point(x0 + width, y0 + height, '+') -class Dagascii(object): - def __init__(self, vertexes, edges): - # - # Just a reminder about naming conventions: - # +------------X - # | - # | - # | - # | - # Y - # +def _build_sugiyama_layout(vertexes, edges): + # + # Just a reminder about naming conventions: + # +------------X + # | + # | + # | + # | + # Y + # - V = {v: Vertex(" {} ".format(v)) for v in vertexes} - # NOTE: reverting edges to correctly orientate the graph - E = [Edge(V[e], V[s]) for s, e in edges] - V = V.values() - g = Graph(V, E) + vertexes = {v: Vertex(" {} ".format(v)) for v in vertexes} + # NOTE: reverting edges to correctly orientate the graph + edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges] + vertexes = vertexes.values() + graph = Graph(vertexes, edges) - class VertexViewer(object): - h = 3 # top and bottom box edges + text + for vertex in vertexes: + vertex.view = VertexViewer(vertex.data) - def __init__(self, name): - self.w = len(name) + 2 # right and left bottom edges + text + # NOTE: determine min box length to create the best layout + minw = min([v.view.w for v in vertexes]) - for v in V: - v.view = VertexViewer(v.data) + for edge in edges: + edge.view = EdgeViewer() - # NOTE: determine min box length to create the best layout - minw = min([v.view.w for v in V]) + sug = SugiyamaLayout(graph.C[0]) + graph = graph.C[0] + roots = list(filter(lambda x: len(x.e_in()) == 0, graph.sV)) - for e in E: - e.view = EdgeViewer() + sug.init_all(roots=roots, optimize=True) - sug = SugiyamaLayout(g.C[0]) - gr = g.C[0] - r = list(filter(lambda x: len(x.e_in()) == 0, gr.sV)) + sug.yspace = VertexViewer.HEIGHT + sug.xspace = minw + sug.route_edge = route_with_lines - sug.init_all(roots=r, optimize=True) + sug.draw() - sug.yspace = VertexViewer.h - sug.xspace = minw - sug.route_edge = route_with_lines + return sug - sug.draw() - self.sug = sug +def draw(vertexes, edges): + """Build a DAG and draw it in ASCII. - def draw(self): - # NOTE: coordinates might me negative, so we need to shift - # everything to the positive plane before we actually draw it. - Xs = [] - Ys = [] + Args: + vertexes (list): list of graph vertexes. + edges (list): list of graph edges. + """ + # pylint: disable=too-many-locals + # NOTE: coordinates might me negative, so we need to shift + # everything to the positive plane before we actually draw it. + Xs = [] # pylint: disable=invalid-name + Ys = [] # pylint: disable=invalid-name - for v in self.sug.g.sV: - # NOTE: moving boxes w/2 to the left - Xs.append(v.view.xy[0] - v.view.w/2.0) - Xs.append(v.view.xy[0] + v.view.w/2.0) - Ys.append(v.view.xy[1]) - Ys.append(v.view.xy[1] + v.view.h) + sug = _build_sugiyama_layout(vertexes, edges) - for e in self.sug.g.sE: - for xy in e.view._pts: - Xs.append(xy[0]) - Ys.append(xy[1]) + for vertex in sug.g.sV: + # NOTE: moving boxes w/2 to the left + Xs.append(vertex.view.xy[0] - vertex.view.w/2.0) + Xs.append(vertex.view.xy[0] + vertex.view.w/2.0) + Ys.append(vertex.view.xy[1]) + Ys.append(vertex.view.xy[1] + vertex.view.h) - minx = min(Xs) - miny = min(Ys) - maxx = max(Xs) - maxy = max(Ys) + for edge in sug.g.sE: + for x, y in edge.view._pts: # pylint: disable=protected-access + Xs.append(x) + Ys.append(y) - canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1 - canvas_lines = int(round(maxy - miny)) + minx = min(Xs) + miny = min(Ys) + maxx = max(Xs) + maxy = max(Ys) - canvas = AsciiCanvas(canvas_cols, canvas_lines) + canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1 + canvas_lines = int(round(maxy - miny)) - # NOTE: first draw edges so that node boxes could overwrite them - for e in self.sug.g.sE: - assert len(e.view._pts) > 1 - for index in range(1, len(e.view._pts)): - start = e.view._pts[index - 1] - end = e.view._pts[index] + canvas = AsciiCanvas(canvas_cols, canvas_lines) - start_x = int(round(start[0] - minx)) - start_y = int(round(start[1] - miny)) - end_x = int(round(end[0] - minx)) - end_y = int(round(end[1] - miny)) + # NOTE: first draw edges so that node boxes could overwrite them + for edge in sug.g.sE: + # pylint: disable=protected-access + assert len(edge.view._pts) > 1 + for index in range(1, len(edge.view._pts)): + start = edge.view._pts[index - 1] + end = edge.view._pts[index] - assert start_x >= 0 - assert start_y >= 0 - assert end_x >= 0 - assert end_y >= 0 + start_x = int(round(start[0] - minx)) + start_y = int(round(start[1] - miny)) + end_x = int(round(end[0] - minx)) + end_y = int(round(end[1] - miny)) - canvas.line(start_x, start_y, - end_x, end_y, - '*') + assert start_x >= 0 + assert start_y >= 0 + assert end_x >= 0 + assert end_y >= 0 - for v in self.sug.g.sV: - # NOTE: moving boxes w/2 to the left - x = v.view.xy[0] - v.view.w/2.0 - y = v.view.xy[1] + canvas.line(start_x, start_y, + end_x, end_y, + '*') - canvas.box(int(round(x - minx)), - int(round(y - miny)), - v.view.w, - v.view.h) + for vertex in sug.g.sV: + # NOTE: moving boxes w/2 to the left + x = vertex.view.xy[0] - vertex.view.w/2.0 + y = vertex.view.xy[1] - canvas.text(int(round(x - minx)) + 1, - int(round(y - miny)) + 1, - v.data) + canvas.box(int(round(x - minx)), + int(round(y - miny)), + vertex.view.w, + vertex.view.h) - canvas.draw() + canvas.text(int(round(x - minx)) + 1, + int(round(y - miny)) + 1, + vertex.data) + + canvas.draw() diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index be78c584c..cd37fcae5 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -1,3 +1,5 @@ +"""Manages dvc remotes that user can use with push/pull/status comamnds.""" + import dvc.logger as logger from dvc.config import Config, ConfigError @@ -12,6 +14,16 @@ from dvc.remote.http import RemoteHTTP class DataCloud(object): + """Class that manages dvc remotes. + + Args: + project (dvc.project.Project): project instance that belongs to the + project that we are working on. + config (dict): config of the project. + + Raises: + config.ConfigError: thrown when config has invalid format. + """ CLOUD_MAP = { 'aws': RemoteS3, 'gcp': RemoteGS, @@ -55,7 +67,7 @@ class DataCloud(object): name = self._core.get(Config.SECTION_CORE_CLOUD, '').strip().lower() if name == '': self._cloud = None - return + return None cloud_type = self.CLOUD_MAP.get(name, None) if not cloud_type: @@ -97,24 +109,52 @@ class DataCloud(object): .format(cmd) ) - def push(self, targets, jobs=1, remote=None, show_checksums=False): - """Push data items in a cloud-agnostic way.""" + def push(self, targets, jobs=None, remote=None, show_checksums=False): + """Push data items in a cloud-agnostic way. + + Args: + targets (list): list of targets to push to the cloud. + jobs (int): number of jobs that can be running simultaneously. + remote (dvc.remote.base.RemoteBase): optional remote to push to. + By default remote from core.remote config option is used. + show_checksums (bool): show checksums instead of file names in + information messages. + """ return self.project.cache.local.push(targets, jobs=jobs, remote=self._get_cloud(remote, 'push'), show_checksums=show_checksums) - def pull(self, targets, jobs=1, remote=None, show_checksums=False): - """Pull data items in a cloud-agnostic way.""" + def pull(self, targets, jobs=None, remote=None, show_checksums=False): + """Pull data items in a cloud-agnostic way. + + Args: + targets (list): list of targets to pull from the cloud. + jobs (int): number of jobs that can be running simultaneously. + remote (dvc.remote.base.RemoteBase): optional remote to pull from. + By default remote from core.remote config option is used. + show_checksums (bool): show checksums instead of file names in + information messages. + """ return self.project.cache.local.pull(targets, jobs=jobs, remote=self._get_cloud(remote, 'pull'), show_checksums=show_checksums) - def status(self, targets, jobs=1, remote=None, show_checksums=False): - """Check status of data items in a cloud-agnostic way.""" + def status(self, targets, jobs=None, remote=None, show_checksums=False): + """Check status of data items in a cloud-agnostic way. + + Args: + targets (list): list of targets to check status for. + jobs (int): number of jobs that can be running simultaneously. + remote (dvc.remote.base.RemoteBase): optional remote to compare + targets to. By default remote from core.remote config option + is used. + show_checksums (bool): show checksums instead of file names in + information messages. + """ cloud = self._get_cloud(remote, 'status') return self.project.cache.local.status(targets, jobs=jobs, diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 8139fb734..0025060ec 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -47,7 +47,7 @@ def _get(stage, p, info): parsed = urlparse(p) if parsed.scheme == 'remote': name = Config.SECTION_REMOTE_FMT.format(parsed.netloc) - sect = stage.project.config._config[name] + sect = stage.project.config.config[name] remote = Remote(stage.project, sect) return DEP_MAP[remote.scheme](stage, p, info, remote=remote) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 81eea2fa5..b26481fc6 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -1,7 +1,15 @@ +"""Exceptions raised by the dvc.""" + import traceback class DvcException(Exception): + """Base class for all dvc exceptions. + + Args: + msg (str): message for this exception. + cause (Exception): optional cause exception. + """ def __init__(self, msg, cause=None): # NOTE: unlike python 3, python 2 doesn't have built-in support # for chained exceptions, so we are using our own implementation. @@ -10,30 +18,37 @@ class DvcException(Exception): if cause: try: self.cause_tb = traceback.format_exc() - except Exception: # pragma: no cover + except AttributeError: # pragma: no cover pass super(DvcException, self).__init__(msg) -class UnsupportedRemoteError(DvcException): - def __init__(self, config): - msg = "remote '{}' is not supported.".format(config) - super(UnsupportedRemoteError, self).__init__(msg) - - class OutputDuplicationError(DvcException): + """Thrown if a file/directory is specified as an output in more than one + stage. + + Args: + output (str): path to the file/directory. + stages (list): list of paths to stages. + """ def __init__(self, output, stages): assert isinstance(output, str) assert isinstance(stages, list) assert all(isinstance(stage, str) for stage in stages) - msg = "file '{}' is specified as an output in more than one stage:{}" - s = "" - for stage in stages: - s += "\n " + stage - super(OutputDuplicationError, self).__init__(msg.format(output, s)) + msg = ("file/directory '{}' is specified as an output in more than one" + "stage: {}").format(output, "\n ".join(stages)) + super(OutputDuplicationError, self).__init__(msg) class WorkingDirectoryAsOutputError(DvcException): + """Thrown if directory that stage is going to be saved in is specified as + an output of anothe stage. + + Args: + cwd (str): path to the directory. + fname (str): path to the stage file that has cwd specified as an + output. + """ def __init__(self, cwd, fname): assert isinstance(cwd, str) assert isinstance(fname, str) @@ -46,13 +61,26 @@ class WorkingDirectoryAsOutputError(DvcException): class CircularDependencyError(DvcException): + """Thrown if a file/directory specified both as an output and as a + dependency. + + Args: + dependency (str): path to the dependency. + """ def __init__(self, dependency): assert isinstance(dependency, str) - msg = "file '{}' is specified as an output and as a dependency." + msg = ("file/directory '{}' is specified as an output and as a " + "dependency.") super(CircularDependencyError, self).__init__(msg.format(dependency)) class ArgumentDuplicationError(DvcException): + """Thrown if a file/directory is specified as a dependency/output more + than once. + + Args: + path (str): path to the file/directory. + """ def __init__(self, path): assert isinstance(path, str) msg = "file '{}' is specified more than once." @@ -60,6 +88,12 @@ class ArgumentDuplicationError(DvcException): class MoveNotDataSourceError(DvcException): + """Thrown if attempted to move a file/directory that is not an output + in a data source stage. + + Args: + path (str): path to the file/directory. + """ def __init__(self, path): msg = "move is not permitted for stages that are not data sources. " \ "You need to either move '{path}' to a new location and edit " \ @@ -69,11 +103,17 @@ class MoveNotDataSourceError(DvcException): class NotDvcProjectError(DvcException): + """Thrown if a directory is not a dvc project. + + Args: + root (str): path to the directory. + """ def __init__(self, root): msg = "not a dvc repository (checked up to mount point '{}')" super(NotDvcProjectError, self).__init__(msg.format(root)) class DvcParserError(DvcException): + """Base class for CLI parser errors.""" def __init__(self): - super(DvcException, self).__init__("parser error") + super(DvcParserError, self).__init__("parser error") diff --git a/dvc/istextfile.py b/dvc/istextfile.py index 310d4738b..6f0f46fae 100644 --- a/dvc/istextfile.py +++ b/dvc/istextfile.py @@ -1,19 +1,23 @@ +"""Use heuristics to guess if it is a text file or a binary file.""" + # Based on https://eli.thegreenplace.net/2011/10/19/ # perls-guess-if-file-is-text-or-binary-implemented-in-python import sys PY3 = sys.version_info[0] == 3 + # A function that takes an integer in the 8-bit range and returns # a single-character byte object in py3 / a single-character string # in py2. # -int2byte = (lambda x: bytes((x,))) if PY3 else chr +def _int2byte(i): + if PY3: + return bytes((i,)) + return chr(i) -_text_characters = ( - b''.join(int2byte(i) for i in range(32, 127)) + - b'\n\r\t\f\b') +TEXT_CHARS = (b''.join(_int2byte(i) for i in range(32, 127)) + b'\n\r\t\f\b') def istextfile(fname, blocksize=512): @@ -22,17 +26,18 @@ def istextfile(fname, blocksize=512): If more than 30% of the chars in the block are non-text, or there are NUL ('\x00') bytes in the block, assume this is a binary file. """ - with open(fname, 'rb') as fd: - block = fd.read(blocksize) + with open(fname, 'rb') as fobj: + block = fobj.read(blocksize) + + if not block: + # An empty file is considered a valid text file + return True if b'\x00' in block: # Files with null bytes are binary return False - elif not block: - # An empty file is considered a valid text file - return True # Use translate's 'deletechars' argument to efficiently remove all - # occurrences of _text_characters from the block - nontext = block.translate(None, _text_characters) + # occurrences of TEXT_CHARS from the block + nontext = block.translate(None, TEXT_CHARS) return float(len(nontext)) / len(block) <= 0.30 diff --git a/dvc/lock.py b/dvc/lock.py index a8109b21d..473f9d7ff 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -1,3 +1,5 @@ +"""Manages dvc lock file.""" + import os import time import zc.lockfile @@ -6,10 +8,17 @@ from dvc.exceptions import DvcException class LockError(DvcException): - pass + """Thrown when unable to acquire the lock for dvc project.""" class Lock(object): + """Class for dvc project lock. + + Args: + dvc_dir (str): path to the directory that the lock should be created + in. + name (str): name of the lock file. + """ LOCK_FILE = 'lock' TIMEOUT = 5 @@ -25,6 +34,7 @@ class Lock(object): 'locked. Please retry the cmd later.') def lock(self): + """Acquire lock for dvc project.""" try: self._do_lock() return @@ -34,11 +44,12 @@ class Lock(object): self._do_lock() def unlock(self): + """Release lock for dvc project.""" self._lock.close() self._lock = None def __enter__(self): self.lock() - def __exit__(self, type, value, tb): + def __exit__(self, typ, value, tbck): self.unlock() diff --git a/dvc/logger.py b/dvc/logger.py index 7624b3e74..8ea17d304 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -1,17 +1,22 @@ +"""Manages logger for dvc project.""" + +import re import sys import logging -import colorama import traceback -import re from contextlib import contextmanager +import colorama + def info(message): + """Prints an info message.""" logger.info(message) def debug(message): + """Prints a debug message.""" prefix = colorize('Debug', color='blue') out = '{prefix}: {message}'.format(prefix=prefix, message=message) @@ -20,6 +25,7 @@ def debug(message): def warning(message): + """Prints a warning message.""" prefix = colorize('Warning', color='yellow') out = '{prefix}: {message}'.format(prefix=prefix, message=message) @@ -28,6 +34,7 @@ def warning(message): def error(message=None): + """Prints an error message.""" prefix = colorize('Error', color='red') exception, stack_trace = _parse_exc() @@ -49,6 +56,12 @@ def error(message=None): def box(message, border_color=None): + """Prints a message in a box. + + Args: + message (str): message to print. + border_color (str): name of a color to outline the box with. + """ lines = message.split('\n') max_width = max(_visual_width(line) for line in lines) @@ -84,20 +97,27 @@ def box(message, border_color=None): ) for line in lines ] - box = "{margin}{padding}{content}{padding}{margin}".format( - margin=colorize(margin, color=border_color), - padding=''.join(padding_lines), - content=''.join(content_lines), - ) + box_str = "{margin}{padding}{content}{padding}{margin}".format( + margin=colorize(margin, color=border_color), + padding=''.join(padding_lines), + content=''.join(content_lines), + ) - logger.info(box) + logger.info(box_str) def level(): + """Returns current log level.""" return logger.getEffectiveLevel() def set_level(level_name): + """Sets log level. + + Args: + level_name (str): log level name. E.g. info, debug, warning, error, + critical. + """ if not level_name: return @@ -113,15 +133,18 @@ def set_level(level_name): def be_quiet(): + """Disables all messages except critical ones.""" logger.setLevel(logging.CRITICAL) def be_verbose(): + """Enables all messages.""" logger.setLevel(logging.DEBUG) @contextmanager def verbose(): + """Enables verbose mode for the context.""" previous_level = level() be_verbose() yield @@ -130,6 +153,7 @@ def verbose(): @contextmanager def quiet(): + """Enables quiet mode for the context.""" previous_level = level() be_quiet() yield @@ -137,14 +161,19 @@ def quiet(): def is_quiet(): + """Returns whether or not all messages except critical ones are + disabled. + """ return level() == logging.CRITICAL def is_verbose(): + """Returns whether or not all messages are enabled.""" return level() == logging.DEBUG def colorize(message, color=None): + """Returns a message in a specified color.""" if not color: return message @@ -166,21 +195,23 @@ def _init_colorama(): colorama.init() -def _set_default_level(): +def set_default_level(): + """Sets default log level.""" logger.setLevel(logging.INFO) def _add_handlers(): formatter = '%(message)s' - class LogLevelFilter(logging.Filter): + class _LogLevelFilter(logging.Filter): + # pylint: disable=too-few-public-methods def filter(self, record): return record.levelno <= logging.WARNING sh_out = logging.StreamHandler(sys.stdout) sh_out.setFormatter(logging.Formatter(formatter)) sh_out.setLevel(logging.DEBUG) - sh_out.addFilter(LogLevelFilter()) + sh_out.addFilter(_LogLevelFilter()) sh_err = logging.StreamHandler(sys.stderr) sh_err.setFormatter(logging.Formatter(formatter)) @@ -258,8 +289,8 @@ def _visual_center(line, width): return (left_padding * ' ') + line + (right_padding * ' ') -logger = logging.getLogger('dvc') +logger = logging.getLogger('dvc') # pylint: disable=invalid-name -_set_default_level() +set_default_level() _add_handlers() _init_colorama() diff --git a/dvc/main.py b/dvc/main.py index c9a136405..789350425 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -1,3 +1,5 @@ +"""Main entry point for dvc CLI.""" + import dvc.logger as logger from dvc.cli import parse_args from dvc.command.base import CmdBase @@ -6,6 +8,14 @@ from dvc.exceptions import NotDvcProjectError, DvcParserError def main(argv=None): + """Run dvc CLI command. + + Args: + argv: optional list of arguments to parse. sys.argv is used by default. + + Returns: + int: command's return code. + """ args = None cmd = None try: @@ -13,7 +23,7 @@ def main(argv=None): # Init loglevel early in case we'll run # into errors before setting it properly - CmdBase._set_loglevel(args) + CmdBase.set_loglevel(args) cmd = args.func(args) @@ -26,7 +36,7 @@ def main(argv=None): ret = 253 except DvcParserError: ret = 254 - except Exception: + except Exception: # pylint: disable=broad-except logger.error('unexpected error') ret = 255 diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index e17c21831..aa283de20 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -53,7 +53,7 @@ def _get(stage, p, info, cache, metric): parsed = urlparse(p) if parsed.scheme == 'remote': name = Config.SECTION_REMOTE_FMT.format(parsed.netloc) - sect = stage.project.config._config[name] + sect = stage.project.config.config[name] remote = Remote(stage.project, sect) return OUTS_MAP[remote.scheme](stage, p, diff --git a/dvc/progress.py b/dvc/progress.py index a4d10ab93..432ceaeec 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -1,3 +1,5 @@ +"""Manages progress bars for dvc project.""" + from __future__ import print_function import sys import threading @@ -15,11 +17,13 @@ class Progress(object): self._line = None def set_n_total(self, total): + """Sets total number of targets.""" self._n_total = total self._n_finished = 0 @property def is_finished(self): + """Returns if all targets have finished.""" return self._n_total == self._n_finished def _clearln(self): @@ -31,6 +35,7 @@ class Progress(object): sys.stdout.flush() def refresh(self, line=None): + """Refreshes progress bar.""" # Just go away if it is locked. Will update next time if not self._lock.acquire(False): return @@ -45,17 +50,19 @@ class Progress(object): self._lock.release() def update_target(self, name, current, total): + """Updates progress bar for a specified target.""" self.refresh(self._bar(name, current, total)) def finish_target(self, name): + """Finishes progress bar for a specified target.""" # We have to write a msg about finished target with self._lock: - bar = self._bar(name, 100, 100) + pbar = self._bar(name, 100, 100) if sys.stdout.isatty(): self._clearln() - self._print(bar) + self._print(pbar) self._n_finished += 1 self._line = None @@ -68,25 +75,26 @@ class Progress(object): bar_len = 30 if total is None: - progress = 0 + state = 0 percent = "?% " else: total = int(total) - progress = int((100 * current)/total) if current < total else 100 - percent = str(progress) + "% " + state = int((100 * current)/total) if current < total else 100 + percent = str(state) + "% " if self._n_total > 1: num = "({}/{}): ".format(self._n_finished + 1, self._n_total) else: num = "" - n_sh = int((progress * bar_len)/100) + n_sh = int((state * bar_len)/100) n_sp = bar_len - n_sh - bar = "[" + '#'*n_sh + ' '*n_sp + "] " + pbar = "[" + '#'*n_sh + ' '*n_sp + "] " - return num + bar + percent + target_name + return num + pbar + percent + target_name - def _print(self, *args, **kwargs): + @staticmethod + def _print(*args, **kwargs): if logger.is_quiet(): return @@ -97,10 +105,10 @@ class Progress(object): if self._line is not None: self._clearln() - def __exit__(self, type, value, tb): + def __exit__(self, typ, value, tbck): if self._line is not None: self.refresh() self._lock.release() -progress = Progress() +progress = Progress() # pylint: disable=invalid-name diff --git a/dvc/project.py b/dvc/project.py index 90110200f..e752aab20 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -31,7 +31,7 @@ class Project(object): from dvc.data_cloud import DataCloud from dvc.updater import Updater - root_dir = self._find_root(root_dir) + root_dir = self.find_root(root_dir) self.root_dir = os.path.abspath(os.path.realpath(root_dir)) self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) @@ -42,14 +42,14 @@ class Project(object): self.lock = Lock(self.dvc_dir) # NOTE: storing state and link_state in the repository itself to avoid # any possible state corruption in 'shared cache dir' scenario. - self.state = State(self, self.config._config) + self.state = State(self, self.config.config) - core = self.config._config[Config.SECTION_CORE] + core = self.config.config[Config.SECTION_CORE] logger.set_level(core.get(Config.SECTION_CORE_LOGLEVEL)) self.cache = Cache(self) - self.cloud = DataCloud(self, config=self.config._config) + self.cloud = DataCloud(self, config=self.config.config) self.updater = Updater(self.dvc_dir) self._files_to_git_add = [] @@ -62,7 +62,7 @@ class Project(object): return "Project: '{root_dir}'".format(root_dir=self.root_dir) @staticmethod - def _find_root(root=None): + def find_root(root=None): if root is None: root = os.getcwd() else: @@ -78,8 +78,8 @@ class Project(object): raise NotDvcProjectError(root) @staticmethod - def _find_dvc_dir(root=None): - root_dir = Project._find_root(root) + def find_dvc_dir(root=None): + root_dir = Project.find_root(root) return os.path.join(root_dir, Project.DVC_DIR) def _remind_to_git_add(self): @@ -459,7 +459,7 @@ class Project(object): if not interactive: config = self.config - core = config._config[config.SECTION_CORE] + core = config.config[config.SECTION_CORE] interactive = core.get(config.SECTION_CORE_INTERACTIVE, False) targets = [] @@ -1244,9 +1244,9 @@ class Project(object): outs.append(out.path + out.sep) stages.append(stage) - def filter_dirs(dname): + def filter_dirs(dname, root=root): path = os.path.join(root, dname) - if path == self.dvc_dir or path == self.scm.dir: + if path in (self.dvc_dir, self.scm.dir): return False for out in outs: if path == os.path.normpath(out) or path.startswith(out): diff --git a/dvc/updater.py b/dvc/updater.py index 5523067d9..24227a45e 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -64,10 +64,10 @@ class Updater(object): # pragma: no cover self._notify() def fetch(self, detach=True): - from dvc.daemon import Daemon + from dvc.daemon import daemon if detach: - Daemon()(['updater']) + daemon(['updater']) return self._with_lock(self._get_latest_version, 'fetching') diff --git a/tests/test_logger.py b/tests/test_logger.py index 4ba48548e..25ed6cdc4 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -20,7 +20,7 @@ class TestLogger(TestCase): def setUp(self): logger.logger.handlers[0].stream = StringIO() logger.logger.handlers[1].stream = StringIO() - logger._set_default_level() + logger.set_default_level() self.color_patch.start() def tearDown(self): @@ -178,10 +178,10 @@ class TestLogger(TestCase): verbose = False args = A() - CmdBase._set_loglevel(args) + CmdBase.set_loglevel(args) self.assertEqual(logger.logger.getEffectiveLevel(), logging.CRITICAL) args.quiet = False args.verbose = True - CmdBase._set_loglevel(args) + CmdBase.set_loglevel(args) self.assertEqual(logger.logger.getEffectiveLevel(), logging.DEBUG) diff --git a/tests/test_state.py b/tests/test_state.py index 706745813..5846e7a04 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -12,7 +12,7 @@ class TestState(TestDvc): path = os.path.join(self.dvc.root_dir, self.FOO) md5 = file_md5(path)[0] - state = State(self.dvc, self.dvc.config._config) + state = State(self.dvc, self.dvc.config.config) with state: entry_md5 = state.update(path)