diff --git a/.appveyor.yml b/.appveyor.yml index f498c8f23..cd270f5c7 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,8 +1,3 @@ -branches: - only: - - master - - devel - cache: - C:\ProgramData\chocolatey\bin -> appveyor.yml - C:\ProgramData\chocolatey\lib -> appveyor.yml @@ -32,8 +27,8 @@ environment: RUBY: C:\Ruby25\bin install: - - SET PATH=%RUBY%;%PATH% - - git clone -q https://github.com/nanaya-tachibana/oss-emulator.git + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%RUBY%;%PATH%" + - git clone -q https://github.com/iterative/oss-emulator.git - gem install thor builder - ps: cd oss-emulator - ps: $OSSProcess = Start-Process -FilePath ruby -ArgumentList "bin\emulator -r store -p 50004" -PassThru -NoNewWindow diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 000000000..3bcc0cc37 --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,15 @@ +version = 1 + +test_patterns = [ + '*/tests/*' +] + +exclude_patterns = [ + +] + +[[analyzers]] +name = 'python' +enabled = true +runtime_version = '3.x.x' + diff --git a/README.rst b/README.rst index fabda7040..e93869c9f 100644 --- a/README.rst +++ b/README.rst @@ -31,8 +31,8 @@ | -**Data Science Version Control** or **DVC** is an **open-source** tool for data science and -machine learning projects. Key features: +**Data Version Control** or **DVC** is an **open-source** tool for data science and machine +learning projects. Key features: #. simple **command line** Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services; diff --git a/dvc/cache.py b/dvc/cache.py index e1b9d7b5d..c834ed0ae 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -7,6 +7,20 @@ import os from dvc.config import Config +class CacheConfig(object): + def __init__(self, config): + self.config = config + + def set_dir(self, dname, level=None): + from dvc.remote.config import RemoteConfig + + configobj = self.config.get_configobj(level) + path = RemoteConfig.resolve_path(dname, configobj.filename) + self.config.set( + Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level + ) + + class Cache(object): """Class that manages cache locations of a dvc repo. @@ -42,7 +56,7 @@ class Cache(object): Config.SECTION_CACHE_PROTECTED: protected, } - self.local = Remote(repo, settings) + self.local = Remote(repo, **settings) self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3) self.gs = self._get_remote(config, Config.SECTION_CACHE_GS) self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH) @@ -87,5 +101,4 @@ class Cache(object): if not remote: return None - settings = self.repo.config.get_remote_settings(remote) - return Remote(self.repo, settings) + return Remote(self.repo, name=remote) diff --git a/dvc/command/add.py b/dvc/command/add.py index 43a1d2b51..45c74c06f 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -54,7 +54,7 @@ def add_parser(subparsers, parent_parser): help="Don't put files/directories into cache.", ) add_parser.add_argument( - "-f", "--file", help="Specify name of the DVC file it generates." + "-f", "--file", help="Specify name of the DVC-file it generates." ) add_parser.add_argument( "targets", nargs="+", help="Input files/directories." diff --git a/dvc/command/cache.py b/dvc/command/cache.py index 80b189cae..b774aca8b 100644 --- a/dvc/command/cache.py +++ b/dvc/command/cache.py @@ -3,18 +3,18 @@ from __future__ import unicode_literals import argparse from dvc.command.base import fix_subparsers, append_doc_link -from dvc.command.remote import CmdRemoteAdd from dvc.command.config import CmdConfig +from dvc.cache import CacheConfig class CmdCacheDir(CmdConfig): - def run(self): - self.args.name = "cache.dir" - self.args.value = CmdRemoteAdd.resolve_path( - self.args.value, self.configobj.filename - ) + def __init__(self, args): + super(CmdCacheDir, self).__init__(args) + self.config = CacheConfig(self.config) - return super(CmdCacheDir, self).run() + def run(self): + self.config.set_dir(self.args.value, level=self.args.level) + return 0 def add_parser(subparsers, parent_parser): diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index 5a9aa05eb..99d376542 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -51,5 +51,5 @@ def add_parser(subparsers, parent_parser): default=False, help="Checkout all subdirectories of the specified directory.", ) - checkout_parser.add_argument("targets", nargs="*", help="DVC files.") + checkout_parser.add_argument("targets", nargs="*", help="DVC-files.") checkout_parser.set_defaults(func=CmdCheckout) diff --git a/dvc/command/commit.py b/dvc/command/commit.py index fd8f017e9..8ba3b99f3 100644 --- a/dvc/command/commit.py +++ b/dvc/command/commit.py @@ -34,7 +34,7 @@ class CmdCommit(CmdBase): def add_parser(subparsers, parent_parser): - COMMIT_HELP = "Save changed data to cache and update DVC files." + COMMIT_HELP = "Save changed data to cache and update DVC-files." commit_parser = subparsers.add_parser( "commit", @@ -65,6 +65,6 @@ def add_parser(subparsers, parent_parser): help="Commit cache for subdirectories of the specified directory.", ) commit_parser.add_argument( - "targets", nargs="*", default=None, help="DVC files." + "targets", nargs="*", default=None, help="DVC-files." ) commit_parser.set_defaults(func=CmdCommit) diff --git a/dvc/command/config.py b/dvc/command/config.py index f62ac5d7d..8e3bdba40 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -1,12 +1,10 @@ from __future__ import unicode_literals -import os import argparse import logging from dvc.command.base import CmdBaseNoRepo, append_doc_link from dvc.config import Config -from dvc.exceptions import DvcException logger = logging.getLogger(__name__) @@ -14,87 +12,48 @@ logger = logging.getLogger(__name__) class CmdConfig(CmdBaseNoRepo): def __init__(self, args): - from dvc.repo import Repo, NotDvcRepoError - super(CmdConfig, self).__init__(args) - try: - dvc_dir = os.path.join(Repo.find_root(), Repo.DVC_DIR) - saved_exc = None - except NotDvcRepoError as exc: - dvc_dir = None - saved_exc = exc - - self.config = Config(dvc_dir, validate=False) - if self.args.system: - self.configobj = self.config._system_config - elif self.args.glob: - self.configobj = self.config._global_config - elif self.args.local: - if dvc_dir is None: - raise saved_exc - self.configobj = self.config._local_config - else: - if dvc_dir is None: - raise saved_exc - self.configobj = self.config._repo_config - - def _unset(self, section, opt=None, configobj=None): - if configobj is None: - configobj = self.configobj - - try: - self.config.unset(configobj, section, opt) - self.config.save(configobj) - except DvcException: - logger.exception("failed to unset '{}'".format(self.args.name)) - return 1 - return 0 - - def _show(self, section, opt): - try: - self.config.show(self.configobj, section, opt) - except DvcException: - logger.exception("failed to show '{}'".format(self.args.name)) - return 1 - return 0 - - def _set(self, section, opt, value): - try: - self.config.set(self.configobj, section, opt, value) - self.config.save(self.configobj) - except DvcException: - logger.exception( - "failed to set '{}.{}' to '{}'".format(section, opt, value) - ) - return 1 - return 0 + self.config = Config(validate=False) def run(self): section, opt = self.args.name.lower().strip().split(".", 1) if self.args.unset: - return self._unset(section, opt) + self.config.unset(section, opt, level=self.args.level) elif self.args.value is None: - return self._show(section, opt) + logger.info(self.config.get(section, opt, level=self.args.level)) else: - return self._set(section, opt, self.args.value) + self.config.set( + section, opt, self.args.value, level=self.args.level + ) + + return 0 parent_config_parser = argparse.ArgumentParser(add_help=False) parent_config_parser.add_argument( "--global", - dest="glob", - action="store_true", - default=False, + dest="level", + action="store_const", + const=Config.LEVEL_GLOBAL, help="Use global config.", ) parent_config_parser.add_argument( - "--system", action="store_true", default=False, help="Use system config." + "--system", + dest="level", + action="store_const", + const=Config.LEVEL_SYSTEM, + help="Use system config.", ) parent_config_parser.add_argument( - "--local", action="store_true", default=False, help="Use local config." + "--local", + dest="level", + action="store_const", + const=Config.LEVEL_LOCAL, + help="Use local config.", ) +parent_config_parser.set_defaults(level=Config.LEVEL_REPO) def add_parser(subparsers, parent_parser): diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 6a9d4e89a..ef7d5b1f0 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -113,7 +113,11 @@ def shared_parent_parser(): help="Show checksums instead of file names.", ) shared_parent_parser.add_argument( +<<<<<<< HEAD "targets", nargs="*", default=None, help="Limit the scope to these stage files." +======= + "targets", nargs="*", default=None, help="DVC-files." +>>>>>>> upstream/master ) return shared_parent_parser diff --git a/dvc/command/destroy.py b/dvc/command/destroy.py index 0ffacc127..1fad83cbf 100644 --- a/dvc/command/destroy.py +++ b/dvc/command/destroy.py @@ -35,7 +35,7 @@ class CmdDestroy(CmdBase): def add_parser(subparsers, parent_parser): - DESTROY_HELP = "Remove DVC files, local DVC config and data cache." + DESTROY_HELP = "Remove DVC-files, local DVC config and data cache." destroy_parser = subparsers.add_parser( "destroy", diff --git a/dvc/command/imp.py b/dvc/command/imp.py index b549bac4e..b284ebb28 100644 --- a/dvc/command/imp.py +++ b/dvc/command/imp.py @@ -65,6 +65,6 @@ def add_parser(subparsers, parent_parser): "out", nargs="?", help="Destination path to put files to." ) import_parser.add_argument( - "-f", "--file", help="Specify name of the DVC file it generates." + "-f", "--file", help="Specify name of the DVC-file it generates." ) import_parser.set_defaults(func=CmdImport) diff --git a/dvc/command/lock.py b/dvc/command/lock.py index e435e8dcc..e934380ed 100644 --- a/dvc/command/lock.py +++ b/dvc/command/lock.py @@ -37,7 +37,7 @@ class CmdUnlock(CmdLockBase): def add_parser(subparsers, parent_parser): - LOCK_HELP = "Lock DVC file." + LOCK_HELP = "Lock DVC-file." lock_parser = subparsers.add_parser( "lock", parents=[parent_parser], @@ -45,10 +45,10 @@ def add_parser(subparsers, parent_parser): help=LOCK_HELP, formatter_class=argparse.RawDescriptionHelpFormatter, ) - lock_parser.add_argument("targets", nargs="+", help="DVC files.") + lock_parser.add_argument("targets", nargs="+", help="DVC-files.") lock_parser.set_defaults(func=CmdLock) - UNLOCK_HELP = "Unlock DVC file." + UNLOCK_HELP = "Unlock DVC-file." unlock_parser = subparsers.add_parser( "unlock", parents=[parent_parser], @@ -56,5 +56,5 @@ def add_parser(subparsers, parent_parser): help=UNLOCK_HELP, formatter_class=argparse.RawDescriptionHelpFormatter, ) - unlock_parser.add_argument("targets", nargs="+", help="DVC files.") + unlock_parser.add_argument("targets", nargs="+", help="DVC-files.") unlock_parser.set_defaults(func=CmdUnlock) diff --git a/dvc/command/move.py b/dvc/command/move.py index 4fce351f0..1991e2dca 100644 --- a/dvc/command/move.py +++ b/dvc/command/move.py @@ -27,7 +27,7 @@ def add_parser(subparsers, parent_parser): MOVE_HELP = "Rename or move a DVC controlled data file or a directory." MOVE_DESCRIPTION = ( "Rename or move a DVC controlled data file or a directory.\n" - "It renames and modifies the corresponding DVC file to reflect the" + "It renames and modifies the corresponding DVC-file to reflect the" " changes." ) diff --git a/dvc/command/pipeline.py b/dvc/command/pipeline.py index 949015dfb..fe71ecb8a 100644 --- a/dvc/command/pipeline.py +++ b/dvc/command/pipeline.py @@ -209,14 +209,14 @@ def add_parser(subparsers, parent_parser): "--commands", action="store_true", default=False, - help="Print commands instead of paths to DVC files.", + help="Print commands instead of paths to DVC-files.", ) pipeline_show_group.add_argument( "-o", "--outs", action="store_true", default=False, - help="Print output files instead of paths to DVC files.", + help="Print output files instead of paths to DVC-files.", ) pipeline_show_parser.add_argument( "-l", @@ -244,7 +244,7 @@ def add_parser(subparsers, parent_parser): help="Output DAG as Dependencies Tree.", ) pipeline_show_parser.add_argument( - "targets", nargs="*", help="DVC files. 'Dvcfile' by default." + "targets", nargs="*", help="DVC-files. 'Dvcfile' by default." ) pipeline_show_parser.set_defaults(func=CmdPipelineShow) diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 803592ef4..df1251d0e 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -1,130 +1,62 @@ from __future__ import unicode_literals import argparse -import os -import re import logging from dvc.command.base import append_doc_link, fix_subparsers from dvc.command.config import CmdConfig -from dvc.config import Config - +from dvc.remote.config import RemoteConfig logger = logging.getLogger(__name__) -class CmdRemoteAdd(CmdConfig): - @staticmethod - def resolve_path(path, config_file): - """Resolve path relative to config file location. +class CmdRemoteConfig(CmdConfig): + def __init__(self, args): + super(CmdRemoteConfig, self).__init__(args) + self.config = RemoteConfig(self.config) - Args: - path: Path to be resolved. - config_file: Path to config file, which `path` is specified - relative to. - - Returns: - Path relative to the `config_file` location. If `path` is an - absolute path then it will be returned without change. - - """ - if os.path.isabs(path): - return path - return os.path.relpath(path, os.path.dirname(config_file)) +class CmdRemoteAdd(CmdRemoteConfig): def run(self): - from dvc.remote import _get, RemoteLOCAL - - remote = _get({Config.SECTION_REMOTE_URL: self.args.url}) - if remote == RemoteLOCAL and not self.args.url.startswith("remote://"): - self.args.url = self.resolve_path( - self.args.url, self.configobj.filename - ) - - section = Config.SECTION_REMOTE_FMT.format(self.args.name) - if (section in self.configobj.keys()) and not self.args.force: - logger.error( - "Remote with name {} already exists. " - "Use -f (--force) to overwrite remote " - "with new value".format(self.args.name) - ) - return 1 - - ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url) - if ret != 0: - return ret - - if self.args.default: - msg = "Setting '{}' as a default remote.".format(self.args.name) - logger.info(msg) - ret = self._set( - Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, self.args.name - ) - - return ret - - -class CmdRemoteRemove(CmdConfig): - def _remove_default(self, config): - core = config.get(Config.SECTION_CORE, None) - if core is None: - return 0 - - default = core.get(Config.SECTION_CORE_REMOTE, None) - if default is None: - return 0 - - if default == self.args.name: - return self._unset( - Config.SECTION_CORE, - opt=Config.SECTION_CORE_REMOTE, - configobj=config, - ) - - def run(self): - section = Config.SECTION_REMOTE_FMT.format(self.args.name) - ret = self._unset(section) - if ret != 0: - return ret - - for configobj in [ - self.config._local_config, - self.config._repo_config, - self.config._global_config, - self.config._system_config, - ]: - self._remove_default(configobj) - self.config.save(configobj) - if configobj == self.configobj: - break - + self.config.add( + self.args.name, + self.args.url, + force=self.args.force, + default=self.args.default, + level=self.args.level, + ) return 0 -class CmdRemoteModify(CmdConfig): +class CmdRemoteRemove(CmdRemoteConfig): def run(self): - section = Config.SECTION_REMOTE_FMT.format(self.args.name) - self.args.name = "{}.{}".format(section, self.args.option) - return super(CmdRemoteModify, self).run() + self.config.remove(self.args.name, level=self.args.level) + return 0 -class CmdRemoteDefault(CmdConfig): +class CmdRemoteModify(CmdRemoteConfig): def run(self): - self.args.value = self.args.name - self.args.name = "core.remote" - return super(CmdRemoteDefault, self).run() + self.config.modify( + self.args.name, + self.args.option, + self.args.value, + level=self.args.level, + ) + return 0 -class CmdRemoteList(CmdConfig): +class CmdRemoteDefault(CmdRemoteConfig): def run(self): - for section in self.configobj.keys(): - r = re.match(Config.SECTION_REMOTE_REGEX, section) - if r: - name = r.group("name") - url = self.configobj[section].get( - Config.SECTION_REMOTE_URL, "" - ) - logger.info("{}\t{}".format(name, url)) + self.config.set_default( + self.args.name, unset=self.args.unset, level=self.args.level + ) + return 0 + + +class CmdRemoteList(CmdRemoteConfig): + def run(self): + for name, url in self.config.list(level=self.args.level).items(): + logger.info("{}\t{}".format(name, url)) return 0 diff --git a/dvc/command/remove.py b/dvc/command/remove.py index aa455702b..4a006a575 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -43,7 +43,7 @@ class CmdRemove(CmdBase): def add_parser(subparsers, parent_parser): - REMOVE_HELP = "Remove outputs of DVC file." + REMOVE_HELP = "Remove outputs of DVC-file." remove_parser = subparsers.add_parser( "remove", parents=[parent_parser], @@ -57,14 +57,14 @@ def add_parser(subparsers, parent_parser): "--outs", action="store_true", default=True, - help="Only remove DVC file outputs.(default)", + help="Only remove DVC-file outputs. (Default)", ) remove_parser_group.add_argument( "-p", "--purge", action="store_true", default=False, - help="Remove DVC file and all its outputs.", + help="Remove DVC-file and all its outputs.", ) remove_parser.add_argument( "-f", @@ -73,5 +73,5 @@ def add_parser(subparsers, parent_parser): default=False, help="Force purge.", ) - remove_parser.add_argument("targets", nargs="+", help="DVC files.") + remove_parser.add_argument("targets", nargs="+", help="DVC-files.") remove_parser.set_defaults(func=CmdRemove) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 271d6908d..3c7ee2f21 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -58,7 +58,7 @@ class CmdRepro(CmdBase): def add_parser(subparsers, parent_parser): - REPRO_HELP = "Check for changes and reproduce DVC file and dependencies." + REPRO_HELP = "Check for changes and reproduce DVC-file and dependencies." repro_parser = subparsers.add_parser( "repro", parents=[parent_parser], @@ -69,7 +69,7 @@ def add_parser(subparsers, parent_parser): repro_parser.add_argument( "targets", nargs="*", - help="DVC file to reproduce (default - 'Dvcfile').", + help="DVC-file to reproduce (default - 'Dvcfile').", ) repro_parser.add_argument( "-f", diff --git a/dvc/command/run.py b/dvc/command/run.py index e711744ee..6afbc5073 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -127,7 +127,7 @@ def add_parser(subparsers, parent_parser): "(do not put into DVC cache).", ) run_parser.add_argument( - "-f", "--file", help="Specify name of the DVC file it generates." + "-f", "--file", help="Specify name of the DVC-file it generates." ) run_parser.add_argument( "-c", "--cwd", default=None, help="Deprecated, use -w and -f instead." @@ -155,7 +155,7 @@ def add_parser(subparsers, parent_parser): "--overwrite-dvcfile", action="store_true", default=False, - help="Overwrite existing dvc file without asking for confirmation.", + help="Overwrite existing DVC-file without asking for confirmation.", ) run_parser.add_argument( "--ignore-build-cache", diff --git a/dvc/command/tag.py b/dvc/command/tag.py index 5701db2e5..679ccbb92 100644 --- a/dvc/command/tag.py +++ b/dvc/command/tag.py @@ -82,14 +82,14 @@ def add_parser(subparsers, parent_parser): ) tag_add_parser.add_argument("tag", help="Dvc tag.") tag_add_parser.add_argument( - "targets", nargs="*", default=[None], help="Dvc files." + "targets", nargs="*", default=[None], help="DVC-files." ) tag_add_parser.add_argument( "-d", "--with-deps", action="store_true", default=False, - help="Add tag for all dependencies of the specified DVC file.", + help="Add tag for all dependencies of the specified DVC-file.", ) tag_add_parser.add_argument( "-R", @@ -109,14 +109,14 @@ def add_parser(subparsers, parent_parser): ) tag_remove_parser.add_argument("tag", help="Dvc tag.") tag_remove_parser.add_argument( - "targets", nargs="*", default=[None], help="Dvc files." + "targets", nargs="*", default=[None], help="DVC-files." ) tag_remove_parser.add_argument( "-d", "--with-deps", action="store_true", default=False, - help="Remove tag for all dependencies of the specified DVC file.", + help="Remove tag for all dependencies of the specified DVC-file.", ) tag_remove_parser.add_argument( "-R", @@ -135,14 +135,14 @@ def add_parser(subparsers, parent_parser): help=TAG_LIST_HELP, ) tag_list_parser.add_argument( - "targets", nargs="*", default=[None], help="Dvc files." + "targets", nargs="*", default=[None], help="DVC-files." ) tag_list_parser.add_argument( "-d", "--with-deps", action="store_true", default=False, - help="List tags for all dependencies of the specified DVC file.", + help="List tags for all dependencies of the specified DVC-file.", ) tag_list_parser.add_argument( "-R", diff --git a/dvc/command/version.py b/dvc/command/version.py index 7b17c8f53..34be52980 100644 --- a/dvc/command/version.py +++ b/dvc/command/version.py @@ -1,11 +1,18 @@ from __future__ import unicode_literals +import os +import psutil import platform import argparse import logging +import uuid +from dvc.utils.compat import Path +from dvc.repo import Repo from dvc.command.base import CmdBaseNoRepo, append_doc_link from dvc.version import __version__ +from dvc.exceptions import DvcException, NotDvcRepoError +from dvc.system import System logger = logging.getLogger(__name__) @@ -13,18 +20,83 @@ logger = logging.getLogger(__name__) class CmdVersion(CmdBaseNoRepo): def run(self): + dvc_version = __version__ + python_version = platform.python_version() + platform_type = platform.platform() + info = ( "DVC version: {dvc_version}\n" "Python version: {python_version}\n" - "Platform: {platform_type}" + "Platform: {platform_type}\n" ).format( - dvc_version=__version__, - python_version=platform.python_version(), - platform_type=platform.platform(), + dvc_version=dvc_version, + python_version=python_version, + platform_type=platform_type, + ) + + try: + repo = Repo() + root_directory = repo.root_dir + + info += ( + "Cache: {cache}\n" + "Filesystem type (cache directory): {fs_cache}\n" + ).format( + cache=self.get_linktype_support_info(repo), + fs_cache=self.get_fs_type(repo.cache.local.cache_dir), + ) + + except NotDvcRepoError: + root_directory = os.getcwd() + + info += ("Filesystem type (workspace): {fs_root}").format( + fs_root=self.get_fs_type(os.path.abspath(root_directory)) ) logger.info(info) return 0 + @staticmethod + def get_fs_type(path): + partition = { + Path(part.mountpoint): (part.fstype, part.device) + for part in psutil.disk_partitions() + } + for parent in list(Path(path).parents): + if parent in partition: + return partition[parent] + return ("unkown", "none") + + @staticmethod + def get_linktype_support_info(repo): + links = { + "reflink": System.reflink, + "hardlink": System.hardlink, + "symlink": System.symlink, + } + + fname = "." + str(uuid.uuid4()) + src = os.path.join(repo.cache.local.cache_dir, fname) + open(src, "w").close() + dst = os.path.join(repo.root_dir, fname) + + cache = [] + + for name, link in links.items(): + try: + link(src, dst) + os.unlink(dst) + supported = True + except DvcException: + supported = False + cache.append( + "{name} - {supported}".format( + name=name, supported=True if supported else False + ) + ) + os.remove(src) + + return ", ".join(cache) + def add_parser(subparsers, parent_parser): VERSION_HELP = "Show DVC version and system/environment informaion." diff --git a/dvc/config.py b/dvc/config.py index 40e705e27..b6f68759f 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -2,15 +2,17 @@ from __future__ import unicode_literals -from dvc.utils.compat import str, open, urlparse +from dvc.utils.compat import str, open import os +import re +import copy import errno import configobj import logging -from schema import Schema, Optional, And, Use, Regex -from dvc.exceptions import DvcException +from schema import Schema, Optional, And, Use, Regex, SchemaError +from dvc.exceptions import DvcException, NotDvcRepoError logger = logging.getLogger(__name__) @@ -20,12 +22,12 @@ class ConfigError(DvcException): Args: msg (str): error message. - ex (Exception): optional exception that has caused this error. + cause (Exception): optional exception that has caused this error. """ - def __init__(self, msg, ex=None): + def __init__(self, msg, cause=None): super(ConfigError, self).__init__( - "config file error: {}".format(msg), ex + "config file error: {}".format(msg), cause=cause ) @@ -43,24 +45,6 @@ def supported_cache_type(types): return True -def supported_loglevel(level): - """Checks if log level config option has a valid value. - - Args: - level (str): log level name. - """ - return level in ["info", "debug", "warning", "error"] - - -def supported_cloud(cloud): - """Checks if obsoleted cloud option has a valid value. - - Args: - cloud (str): cloud type name. - """ - return cloud in ["aws", "gcp", "local", ""] - - def is_bool(val): """Checks that value is a boolean. @@ -109,6 +93,23 @@ def is_percent(val): return int(val) >= 0 and int(val) <= 100 +class Choices(object): + """Checks that value belongs to the specified set of values + + Args: + *choices: pass allowed values as arguments, or pass a list or + tuple as a single argument + """ + + def __init__(self, *choices): + if len(choices) == 1 and isinstance(choices[0], (list, tuple)): + choices = choices[0] + self.choices = choices + + def __call__(self, value): + return value in self.choices + + class Config(object): # pylint: disable=too-many-instance-attributes """Class that manages configuration files for a dvc repo. @@ -134,11 +135,18 @@ class Config(object): # pylint: disable=too-many-instance-attributes CONFIG = "config" CONFIG_LOCAL = "config.local" + LEVEL_LOCAL = 0 + LEVEL_REPO = 1 + LEVEL_GLOBAL = 2 + LEVEL_SYSTEM = 3 + BOOL_SCHEMA = And(str, is_bool, Use(to_bool)) SECTION_CORE = "core" SECTION_CORE_LOGLEVEL = "loglevel" - SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel) + SECTION_CORE_LOGLEVEL_SCHEMA = And( + Use(str.lower), Choices("info", "debug", "warning", "error") + ) SECTION_CORE_REMOTE = "remote" SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA SECTION_CORE_INTERACTIVE = "interactive" @@ -173,7 +181,11 @@ class Config(object): # pylint: disable=too-many-instance-attributes # backward compatibility SECTION_CORE_CLOUD = "cloud" - SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud) + SECTION_CORE_CLOUD_SCHEMA = And( + Use(str.lower), + # only the clouds which were supported by obsoleted cloud config + Choices("aws", "gcp", "local", ""), + ) SECTION_CORE_STORAGEPATH = "storagepath" SECTION_CORE_SCHEMA = { @@ -201,6 +213,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes SECTION_AWS_REGION = "region" SECTION_AWS_PROFILE = "profile" SECTION_AWS_USE_SSL = "use_ssl" + SECTION_AWS_SSE = "sse" SECTION_AWS_SCHEMA = { SECTION_AWS_STORAGEPATH: str, Optional(SECTION_AWS_REGION): str, @@ -209,6 +222,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional(SECTION_AWS_ENDPOINT_URL): str, Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA, Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA, + Optional(SECTION_AWS_SSE): str, } # backward compatibility @@ -249,6 +263,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional(SECTION_AWS_ENDPOINT_URL): str, Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA, Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA, + Optional(SECTION_AWS_SSE): str, Optional(SECTION_GCP_PROJECTNAME): str, Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA, Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA, @@ -285,30 +300,20 @@ class Config(object): # pylint: disable=too-many-instance-attributes } def __init__(self, dvc_dir=None, validate=True): - self.system_config_file = os.path.join( - self.get_system_config_dir(), self.CONFIG - ) - self.global_config_file = os.path.join( - self.get_global_config_dir(), self.CONFIG - ) + self.dvc_dir = dvc_dir + self.validate = validate - if dvc_dir is not None: - self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) - self.config_file = os.path.join(dvc_dir, self.CONFIG) - self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL) + if not dvc_dir: + try: + from dvc.repo import Repo + + self.dvc_dir = os.path.join(Repo.find_dvc_dir()) + except NotDvcRepoError: + self.dvc_dir = None else: - self.dvc_dir = None - self.config_file = None - self.config_local_file = None + self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) - self._system_config = None - self._global_config = None - self._repo_config = None - self._local_config = None - - self.config = None - - self.load(validate=validate) + self.load() @staticmethod def get_global_config_dir(): @@ -350,35 +355,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes open(config_file, "w+").close() return Config(dvc_dir) - def _load(self): - self._system_config = configobj.ConfigObj(self.system_config_file) - self._global_config = configobj.ConfigObj(self.global_config_file) - - if self.config_file is not None: - self._repo_config = configobj.ConfigObj(self.config_file) - else: - self._repo_config = configobj.ConfigObj() - - if self.config_local_file is not None: - self._local_config = configobj.ConfigObj(self.config_local_file) - else: - self._local_config = configobj.ConfigObj() - - self.config = None - - def _load_config(self, path): - config = configobj.ConfigObj(path) - config = self._lower(config) - self._resolve_paths(config, path) - return config - - @staticmethod - def _resolve_path(path, config_file): - assert os.path.isabs(config_file) - config_dir = os.path.dirname(config_file) - return os.path.abspath(os.path.join(config_dir, path)) - - def _resolve_cache_path(self, config, fname): + def _resolve_cache_path(self, config): cache = config.get(self.SECTION_CACHE) if cache is None: return @@ -387,72 +364,86 @@ class Config(object): # pylint: disable=too-many-instance-attributes if cache_dir is None: return - cache[self.PRIVATE_CWD] = os.path.dirname(fname) + cache[self.PRIVATE_CWD] = os.path.dirname(config.filename) - def _resolve_paths(self, config, fname): - if fname is None: - return + def _resolve_paths(self, config): + if config.filename is None: + return config - self._resolve_cache_path(config, fname) - for section in config.values(): + ret = copy.deepcopy(config) + self._resolve_cache_path(ret) + + for section in ret.values(): if self.SECTION_REMOTE_URL not in section.keys(): continue - section[self.PRIVATE_CWD] = os.path.dirname(fname) + section[self.PRIVATE_CWD] = os.path.dirname(ret.filename) - def load(self, validate=True): + return ret + + def _load_configs(self): + system_config_file = os.path.join( + self.get_system_config_dir(), self.CONFIG + ) + + global_config_file = os.path.join( + self.get_global_config_dir(), self.CONFIG + ) + + self._system_config = configobj.ConfigObj(system_config_file) + self._global_config = configobj.ConfigObj(global_config_file) + self._repo_config = configobj.ConfigObj() + self._local_config = configobj.ConfigObj() + + if not self.dvc_dir: + return + + config_file = os.path.join(self.dvc_dir, self.CONFIG) + config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL) + + self._repo_config = configobj.ConfigObj(config_file) + self._local_config = configobj.ConfigObj(config_local_file) + + @property + def config_local_file(self): + return self._local_config.filename + + @property + def config_file(self): + return self._repo_config.filename + + def load(self): """Loads config from all the config files. - Args: - validate (bool): optional flag to tell dvc if it should validate - the config or just load it as is. 'True' by default. - - Raises: dvc.config.ConfigError: thrown if config has invalid format. """ - self._load() + self._load_configs() + + self.config = configobj.ConfigObj() + for c in [ + self._system_config, + self._global_config, + self._repo_config, + self._local_config, + ]: + c = self._resolve_paths(c) + c = self._lower(c) + self.config.merge(c) + + if not self.validate: + return + + d = self.config.dict() try: - self.config = self._load_config(self.system_config_file) - user = self._load_config(self.global_config_file) - config = self._load_config(self.config_file) - local = self._load_config(self.config_local_file) - - # NOTE: schema doesn't support ConfigObj.Section validation, so we - # need to convert our config to dict before passing it to - for conf in [user, config, local]: - self.config = self._merge(self.config, conf) - - if validate: - self.config = Schema(self.SCHEMA).validate(self.config) - - # NOTE: now converting back to ConfigObj - self.config = configobj.ConfigObj( - self.config, write_empty_values=True - ) - self.config.filename = self.config_file - self._resolve_paths(self.config, self.config_file) - except Exception as ex: - raise ConfigError(ex) - - @staticmethod - def _get_key(conf, name, add=False): - for k in conf.keys(): - if k.lower() == name.lower(): - return k - - if add: - conf[name] = {} - return name - - return None + d = Schema(self.SCHEMA).validate(d) + except SchemaError as exc: + raise ConfigError("config format error", cause=exc) + self.config = configobj.ConfigObj(d, write_empty_values=True) def save(self, config=None): """Saves config to config files. - Args: - config (configobj.ConfigObj): optional config object to save. - Raises: dvc.config.ConfigError: thrown if failed to write config file. """ @@ -467,129 +458,103 @@ class Config(object): # pylint: disable=too-many-instance-attributes ] for conf in clist: - if conf.filename is None: - continue + self._save(conf) - try: - logger.debug("Writing '{}'.".format(conf.filename)) - dname = os.path.dirname(os.path.abspath(conf.filename)) - try: - os.makedirs(dname) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - conf.write() - except Exception as exc: - msg = "failed to write config '{}'".format(conf.filename) - raise ConfigError(msg, exc) - - def get_remote_settings(self, name): - import posixpath - - """ - Args: - name (str): The name of the remote that we want to retrieve - - Returns: - dict: The content beneath the given remote name. - - Example: - >>> config = {'remote "server"': {'url': 'ssh://localhost/'}} - >>> get_remote_settings("server") - {'url': 'ssh://localhost/'} - """ - settings = self.config.get( - self.SECTION_REMOTE_FMT.format(name.lower()) - ) - - if settings is None: - raise ConfigError( - "unable to find remote section '{}'".format(name) - ) - - parsed = urlparse(settings["url"]) - - # Support for cross referenced remotes. - # This will merge the settings, giving priority to the outer reference. - # For example, having: - # - # dvc remote add server ssh://localhost - # dvc remote modify server user root - # dvc remote modify server ask_password true - # - # dvc remote add images remote://server/tmp/pictures - # dvc remote modify images user alice - # dvc remote modify images ask_password false - # dvc remote modify images password asdf1234 - # - # Results on a config dictionary like: - # - # { - # "url": "ssh://localhost/tmp/pictures", - # "user": "alice", - # "password": "asdf1234", - # "ask_password": False, - # } - # - if parsed.scheme == "remote": - reference = self.get_remote_settings(parsed.netloc) - url = posixpath.join(reference["url"], parsed.path.lstrip("/")) - merged = reference.copy() - merged.update(settings) - merged["url"] = url - return merged - - return settings + self.load() @staticmethod - def unset(config, section, opt=None): + def _save(config): + if config.filename is None: + return + + logger.debug("Writing '{}'.".format(config.filename)) + dname = os.path.dirname(os.path.abspath(config.filename)) + try: + os.makedirs(dname) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + config.write() + + def unset(self, section, opt=None, level=None, force=False): """Unsets specified option and/or section in the config. Args: - config (configobj.ConfigObj): config to work on. section (str): section name. opt (str): optional option name. + level (int): config level to use. + force (bool): don't error-out even if section doesn't exist. False + by default. + + Raises: + dvc.config.ConfigError: thrown if section doesn't exist and + `force != True`. """ + config = self.get_configobj(level) + if section not in config.keys(): + if force: + return raise ConfigError("section '{}' doesn't exist".format(section)) - if opt is None: - del config[section] - return + if opt: + if opt not in config[section].keys(): + if force: + return + raise ConfigError( + "option '{}.{}' doesn't exist".format(section, opt) + ) + del config[section][opt] - if opt not in config[section].keys(): - raise ConfigError( - "option '{}.{}' doesn't exist".format(section, opt) - ) - del config[section][opt] - - if not config[section]: + if not config[section]: + del config[section] + else: del config[section] - @staticmethod - def set(config, section, opt, value): + self.save(config) + + def set(self, section, opt, value, level=None, force=True): """Sets specified option in the config. Args: - config (configobj.ConfigObj): config to work on. section (str): section name. opt (str): option name. value: value to set option to. + level (int): config level to use. + force (bool): set option even if section already exists. True by + default. + + Raises: + dvc.config.ConfigError: thrown if section already exists and + `force != True`. + """ + config = self.get_configobj(level) + if section not in config.keys(): config[section] = {} + elif not force: + raise ConfigError( + "Section '{}' already exists. Use `-f|--force` to overwrite " + "section with new value.".format(section) + ) config[section][opt] = value + self.save(config) - @staticmethod - def show(config, section, opt): - """Prints option value from the config. + def get(self, section, opt=None, level=None): + """Return option value from the config. Args: - config (configobj.ConfigObj): config to work on. section (str): section name. opt (str): option name. + level (int): config level to use. + + Returns: + value (str, int): option value. """ + config = self.get_configobj(level) + if section not in config.keys(): raise ConfigError("section '{}' doesn't exist".format(section)) @@ -598,25 +563,35 @@ class Config(object): # pylint: disable=too-many-instance-attributes "option '{}.{}' doesn't exist".format(section, opt) ) - logger.info(config[section][opt]) - - @staticmethod - def _merge(first, second): - res = {} - sections = list(first.keys()) + list(second.keys()) - for section in sections: - first_copy = first.get(section, {}).copy() - second_copy = second.get(section, {}).copy() - first_copy.update(second_copy) - res[section] = first_copy - return res + return config[section][opt] @staticmethod def _lower(config): - new_config = {} + new_config = configobj.ConfigObj() for s_key, s_value in config.items(): new_s = {} for key, value in s_value.items(): new_s[key.lower()] = str(value) new_config[s_key.lower()] = new_s return new_config + + def get_configobj(self, level): + configs = { + self.LEVEL_LOCAL: self._local_config, + self.LEVEL_REPO: self._repo_config, + self.LEVEL_GLOBAL: self._global_config, + self.LEVEL_SYSTEM: self._system_config, + } + + return configs.get(level, self._repo_config) + + def list_options(self, section_regex, option, level=None): + ret = {} + config = self.get_configobj(level) + for section in config.keys(): + r = re.match(section_regex, section) + if r: + name = r.group("name") + value = config[section].get(option, "") + ret[name] = value + return ret diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 5802826af..d55eb86fd 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -25,7 +25,6 @@ class DataCloud(object): Args: repo (dvc.repo.Repo): repo instance that belongs to the repo that we are working on. - config (dict): config of the repo. Raises: config.ConfigError: thrown when config has invalid format. @@ -75,8 +74,7 @@ class DataCloud(object): ) def _init_remote(self, remote): - config = self.repo.config.get_remote_settings(remote) - return Remote(self.repo, config) + return Remote(self.repo, name=remote) def _init_compat(self): name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower() diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 5c5ba67d4..78ac2da31 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -52,8 +52,7 @@ def _get(stage, p, info): parsed = urlparse(p) if parsed.scheme == "remote": - settings = stage.repo.config.get_remote_settings(parsed.netloc) - remote = Remote(stage.repo, settings) + remote = Remote(stage.repo, name=parsed.netloc) return DEP_MAP[remote.scheme](stage, p, info, remote=remote) for d in DEPS: diff --git a/dvc/main.py b/dvc/main.py index 12db9e112..f00dc0ced 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals import logging from dvc.cli import parse_args +from dvc.config import ConfigError from dvc.analytics import Analytics from dvc.exceptions import NotDvcRepoError, DvcParserError @@ -36,6 +37,9 @@ def main(argv=None): cmd = args.func(args) ret = cmd.run_cmd() + except ConfigError: + logger.exception("configuration error") + ret = 251 except KeyboardInterrupt: logger.exception("interrupted by the user") ret = 252 diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index 60db6a50c..ca8ad7777 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -61,8 +61,7 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None): parsed = urlparse(p) if parsed.scheme == "remote": - settings = stage.repo.config.get_remote_settings(parsed.netloc) - remote = Remote(stage.repo, settings) + remote = Remote(stage.repo, name=parsed.netloc) return OUTS_MAP[remote.scheme]( stage, p, diff --git a/dvc/progress.py b/dvc/progress.py index b25f07d43..31c882f52 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -111,7 +111,7 @@ class Progress(object): logger = logging.getLogger(__name__) - if logger.getEffectiveLevel() == "CRITICAL": + if logger.getEffectiveLevel() == logging.CRITICAL: return print(*args, **kwargs) diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index 4eb124199..e8ffe81f4 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -10,6 +10,8 @@ from dvc.remote.http import RemoteHTTP from dvc.remote.https import RemoteHTTPS from dvc.remote.oss import RemoteOSS +from .config import RemoteConfig + REMOTES = [ RemoteAZURE, @@ -31,5 +33,11 @@ def _get(config): return RemoteLOCAL -def Remote(repo, config): - return _get(config)(repo, config) +def Remote(repo, **kwargs): + name = kwargs.get("name") + if name: + remote_config = RemoteConfig(repo.config) + settings = remote_config.get_settings(name) + else: + settings = kwargs + return _get(settings)(repo, settings) diff --git a/dvc/remote/config.py b/dvc/remote/config.py new file mode 100644 index 000000000..54b33ed2b --- /dev/null +++ b/dvc/remote/config.py @@ -0,0 +1,151 @@ +import os +import logging +import posixpath + +from dvc.utils.compat import urlparse +from dvc.config import Config, ConfigError + + +logger = logging.getLogger(__name__) + + +class RemoteConfig(object): + def __init__(self, config): + self.config = config + + def get_settings(self, name): + """ + Args: + name (str): The name of the remote that we want to retrieve + + Returns: + dict: The content beneath the given remote name. + + Example: + >>> config = {'remote "server"': {'url': 'ssh://localhost/'}} + >>> get_settings("server") + {'url': 'ssh://localhost/'} + """ + settings = self.config.config.get( + Config.SECTION_REMOTE_FMT.format(name.lower()) + ) + + if settings is None: + raise ConfigError( + "unable to find remote section '{}'".format(name) + ) + + parsed = urlparse(settings["url"]) + + # Support for cross referenced remotes. + # This will merge the settings, giving priority to the outer reference. + # For example, having: + # + # dvc remote add server ssh://localhost + # dvc remote modify server user root + # dvc remote modify server ask_password true + # + # dvc remote add images remote://server/tmp/pictures + # dvc remote modify images user alice + # dvc remote modify images ask_password false + # dvc remote modify images password asdf1234 + # + # Results on a config dictionary like: + # + # { + # "url": "ssh://localhost/tmp/pictures", + # "user": "alice", + # "password": "asdf1234", + # "ask_password": False, + # } + # + if parsed.scheme == "remote": + reference = self.get_settings(parsed.netloc) + url = posixpath.join(reference["url"], parsed.path.lstrip("/")) + merged = reference.copy() + merged.update(settings) + merged["url"] = url + return merged + + return settings + + @staticmethod + def resolve_path(path, config_file): + """Resolve path relative to config file location. + + Args: + path: Path to be resolved. + config_file: Path to config file, which `path` is specified + relative to. + + Returns: + Path relative to the `config_file` location. If `path` is an + absolute path then it will be returned without change. + + """ + if os.path.isabs(path): + return path + return os.path.relpath(path, os.path.dirname(config_file)) + + def add(self, name, url, default=False, force=False, level=None): + from dvc.remote import _get, RemoteLOCAL + + configobj = self.config.get_configobj(level) + remote = _get({Config.SECTION_REMOTE_URL: url}) + if remote == RemoteLOCAL and not url.startswith("remote://"): + url = self.resolve_path(url, configobj.filename) + + self.config.set( + Config.SECTION_REMOTE_FMT.format(name), + Config.SECTION_REMOTE_URL, + url, + force=force, + level=level, + ) + if default: + logger.info("Setting '{}' as a default remote.".format(name)) + self.config.set( + Config.SECTION_CORE, + Config.SECTION_CORE_REMOTE, + name, + level=level, + ) + + def remove(self, name, level=None): + self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level) + + if level is None: + level = Config.LEVEL_REPO + + for lev in [ + Config.LEVEL_LOCAL, + Config.LEVEL_REPO, + Config.LEVEL_GLOBAL, + Config.LEVEL_SYSTEM, + ]: + self.config.unset( + Config.SECTION_CORE, + Config.SECTION_CORE_REMOTE, + level=lev, + force=True, + ) + if lev == level: + break + + def modify(self, name, option, value, level=None): + self.config.set( + Config.SECTION_REMOTE_FMT.format(name), option, value, level=level + ) + + def list(self, level=None): + return self.config.list_options( + Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level + ) + + def set_default(self, name, unset=False, level=None): + if unset: + self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE) + return + self.config.set( + Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level + ) diff --git a/dvc/remote/local/__init__.py b/dvc/remote/local/__init__.py index 25bf1d425..3e314da4f 100644 --- a/dvc/remote/local/__init__.py +++ b/dvc/remote/local/__init__.py @@ -464,14 +464,20 @@ class RemoteLOCAL(RemoteBASE): if len(chunks) == 0: return 0 - futures = [] - with ThreadPoolExecutor(max_workers=jobs) as executor: - for from_infos, to_infos, names in chunks: - res = executor.submit(func, from_infos, to_infos, names=names) - futures.append(res) + if jobs > 1: + futures = [] + with ThreadPoolExecutor(max_workers=jobs) as executor: + for from_infos, to_infos, names in chunks: + res = executor.submit( + func, from_infos, to_infos, names=names + ) + futures.append(res) - for f in futures: - f.result() + for f in futures: + f.result() + else: + for from_infos, to_infos, names in chunks: + func(from_infos, to_infos, names=names) return len(chunks) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 99ff523d2..900f8b7bb 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -64,6 +64,12 @@ class RemoteS3(RemoteBASE): self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True) + self.extra_args = {} + + self.sse = config.get(Config.SECTION_AWS_SSE, "") + if self.sse: + self.extra_args["ServerSideEncryption"] = self.sse + shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH) if shared_creds: os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds) @@ -111,17 +117,16 @@ class RemoteS3(RemoteBASE): ) return obj - @classmethod - def _copy_multipart(cls, s3, from_info, to_info, size, n_parts): + def _copy_multipart(self, s3, from_info, to_info, size, n_parts): mpu = s3.create_multipart_upload( - Bucket=to_info.bucket, Key=to_info.path + Bucket=to_info.bucket, Key=to_info.path, **self.extra_args ) mpu_id = mpu["UploadId"] parts = [] byte_position = 0 for i in range(1, n_parts + 1): - obj = cls.get_head_object( + obj = self.get_head_object( s3, from_info.bucket, from_info.path, PartNumber=i ) part_size = obj["ContentLength"] @@ -153,8 +158,7 @@ class RemoteS3(RemoteBASE): MultipartUpload={"Parts": parts}, ) - @classmethod - def _copy(cls, s3, from_info, to_info): + def _copy(self, s3, from_info, to_info): # NOTE: object's etag depends on the way it was uploaded to s3 or the # way it was copied within the s3. More specifically, it depends on # the chunk size that was used to transfer it, which would affect @@ -175,19 +179,21 @@ class RemoteS3(RemoteBASE): # preserve etag, we need to transfer each part separately, so the # object is transfered in the same chunks as it was originally. - obj = cls.get_head_object(s3, from_info.bucket, from_info.path) + obj = self.get_head_object(s3, from_info.bucket, from_info.path) etag = obj["ETag"].strip('"') size = obj["ContentLength"] _, _, parts_suffix = etag.partition("-") if parts_suffix: n_parts = int(parts_suffix) - cls._copy_multipart(s3, from_info, to_info, size, n_parts) + self._copy_multipart(s3, from_info, to_info, size, n_parts) else: source = {"Bucket": from_info.bucket, "Key": from_info.path} - s3.copy(source, to_info.bucket, to_info.path) + s3.copy( + source, to_info.bucket, to_info.path, ExtraArgs=self.extra_args + ) - cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path) + cached_etag = self.get_etag(s3, to_info.bucket, to_info.path) if etag != cached_etag: raise ETagMismatchError(etag, cached_etag) @@ -257,7 +263,11 @@ class RemoteS3(RemoteBASE): try: s3.upload_file( - from_info.path, to_info.bucket, to_info.path, Callback=cb + from_info.path, + to_info.bucket, + to_info.path, + Callback=cb, + ExtraArgs=self.extra_args, ) except Exception: msg = "failed to upload '{}'".format(from_info.path) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 0f43957c1..884852947 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -5,6 +5,7 @@ import logging import dvc.prompt as prompt +from dvc.config import Config from dvc.exceptions import ( DvcException, NotDvcRepoError, @@ -41,7 +42,6 @@ class Repo(object): from dvc.repo.brancher import brancher def __init__(self, root_dir=None): - from dvc.config import Config from dvc.state import State from dvc.lock import Lock from dvc.scm import SCM @@ -89,15 +89,15 @@ class Repo(object): def __repr__(self): return "Repo: '{root_dir}'".format(root_dir=self.root_dir) - @staticmethod - def find_root(root=None): + @classmethod + def find_root(cls, root=None): if root is None: root = os.getcwd() else: root = os.path.abspath(os.path.realpath(root)) while True: - dvc_dir = os.path.join(root, Repo.DVC_DIR) + dvc_dir = os.path.join(root, cls.DVC_DIR) if os.path.isdir(dvc_dir): return root if os.path.ismount(root): @@ -105,10 +105,10 @@ class Repo(object): root = os.path.dirname(root) raise NotDvcRepoError(root) - @staticmethod - def find_dvc_dir(root=None): - root_dir = Repo.find_root(root) - return os.path.join(root_dir, Repo.DVC_DIR) + @classmethod + def find_dvc_dir(cls, root=None): + root_dir = cls.find_root(root) + return os.path.join(root_dir, cls.DVC_DIR) @staticmethod def init(root_dir=os.curdir, no_scm=False, force=False): @@ -318,7 +318,7 @@ class Repo(object): for stage in stages: if active and not target and stage.locked: logger.warning( - "DVC file '{path}' is locked. Its dependencies are" + "DVC-file '{path}' is locked. Its dependencies are" " not going to be pushed/pulled/fetched.".format( path=stage.relpath ) diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 8083feabc..2e1f3dd9f 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -47,7 +47,7 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False): for stage in stages: if stage.locked: logger.warning( - "DVC file '{path}' is locked. Its dependencies are" + "DVC-file '{path}' is locked. Its dependencies are" " not going to be checked out.".format(path=stage.relpath) ) diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py index 4a7b51ce7..daa2e40da 100644 --- a/dvc/repo/reproduce.py +++ b/dvc/repo/reproduce.py @@ -15,7 +15,7 @@ def _reproduce_stage(stages, node, force, dry, interactive, no_commit): if stage.locked: logger.warning( - "DVC file '{path}' is locked. Its dependencies are" + "DVC-file '{path}' is locked. Its dependencies are" " not going to be reproduced.".format(path=stage.relpath) ) diff --git a/dvc/repo/status.py b/dvc/repo/status.py index e9425d4d8..a3a2c7e0f 100644 --- a/dvc/repo/status.py +++ b/dvc/repo/status.py @@ -14,7 +14,7 @@ def _local_status(self, target=None, with_deps=False): for stage in stages: if stage.locked: logger.warning( - "DVC file '{path}' is locked. Its dependencies are" + "DVC-file '{path}' is locked. Its dependencies are" " not going to be shown in the status output.".format( path=stage.relpath ) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 3522f74d0..dd3e1f314 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -242,7 +242,7 @@ class Git(Base): logger.info( "\n" - "To track the changes with git run:\n" + "To track the changes with git, run:\n" "\n" "\tgit add {files}".format(files=" ".join(self.files_to_track)) ) @@ -297,8 +297,8 @@ class Git(Base): b_ref (str): optional second git reference, default None Returns: - dict: dictionary with keys: (a_tree, b_tree, a_ref, b_ref, - equal) + dict: dictionary with keys: {a_ref, b_ref, equal} + or {a_ref, b_ref, a_tree, b_tree} """ diff_dct = {DIFF_EQUAL: False} trees, commits = self._get_diff_trees(a_ref, b_ref) diff --git a/dvc/stage.py b/dvc/stage.py index a5b018230..3fece0483 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -54,7 +54,7 @@ class StageFileAlreadyExistsError(DvcException): class StageFileIsNotDvcFileError(DvcException): def __init__(self, fname): - msg = "'{}' is not a dvc file".format(fname) + msg = "'{}' is not a DVC-file".format(fname) sname = fname + Stage.STAGE_FILE_SUFFIX if Stage.is_stage_file(sname): @@ -218,7 +218,7 @@ class Stage(object): if self.is_callback: logger.warning( - "Dvc file '{fname}' is a 'callback' stage " + "DVC-file '{fname}' is a 'callback' stage " "(has a command and no dependencies) and thus always " "considered as changed.".format(fname=self.relpath) ) @@ -253,7 +253,7 @@ class Stage(object): def _changed_md5(self): if self.changed_md5(): - logger.warning("Dvc file '{}' changed.".format(self.relpath)) + logger.warning("DVC-file '{}' changed.".format(self.relpath)) return True return False @@ -583,7 +583,7 @@ class Stage(object): # it raises the proper exceptions by priority: # 1. when the file doesn't exists - # 2. filename is not a dvc filename + # 2. filename is not a DVC-file # 3. path doesn't represent a regular file Stage._check_file_exists(repo, fname) Stage._check_dvc_filename(fname) diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index cecc0a207..9a7bd9e73 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -21,8 +21,9 @@ def get_inode(path): def get_mtime_and_size(path): - size = os.path.getsize(path) - mtime = os.path.getmtime(path) + stat = os.stat(path) + size = stat.st_size + mtime = stat.st_mtime if os.path.isdir(path): for root, dirs, files in dvc_walk(str(path)): diff --git a/scripts/build_posix.sh b/scripts/build_posix.sh index ec5564316..74b86933c 100755 --- a/scripts/build_posix.sh +++ b/scripts/build_posix.sh @@ -15,6 +15,7 @@ fi BUILD_DIR=build BIN_DIR=$BUILD_DIR/$INSTALL_DIR/bin +DESC='Data Version Control - datasets, models, and experiments versioning for ML or data science projects' LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib print_error() @@ -46,6 +47,7 @@ fpm_build() fpm -s dir \ -f \ -t $1 \ + --description $DESC \ $FPM_FLAGS \ -n dvc \ -v $VERSION \ diff --git a/scripts/completion/dvc.bash b/scripts/completion/dvc.bash index d31149459..adc389e0f 100644 --- a/scripts/completion/dvc.bash +++ b/scripts/completion/dvc.bash @@ -12,30 +12,31 @@ _dvc_commands='init destroy add import checkout run pull push fetch \ status repro remove move unprotect gc config remote metrics \ install root lock unlock pipeline commit' -_dvc_global_options="-h --help -q --quiet -V --verbose" _dvc_options="-h --help -v --version" -_dvc_init="--no-scm -f --force" -_dvc_destroy="-f --force" +_dvc_global_options="-h --help -q --quiet -V --verbose" + _dvc_add="-R --recursive -f --file --no-commit" -_dvc_run="--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache -M --metrics-no-cache -y --yes --overwrite-dvc-file --ignore-build-cache --remove-outs --no-commit -w --wdir" +_dvc_checkout="$(compgen -G '*.dvc')" +_dvc_config="-u --unset --local --system --global" +_dvc_destroy="-f --force" +_dvc_fetch="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive" +_dvc_gc="-a --all-branches -T --all-tags -c --cloud -r --remote -f --force -p --project" +_dvc_import="--resume" +_dvc_init="--no-scm -f --force" +_dvc_install="" +_dvc_lock="$(compgen -G '*.dvc')" +_dvc_metrics="" +_dvc_move="$(compgen -G '*')" +_dvc_pipeline="" _dvc_pull="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -f --force -R --recursive" _dvc_push="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive" -_dvc_fetch="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive" -_dvc_status="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud -q --quiet" -_dvc_repro="--dry -f --force -s --single-item -c --cwd -m --metrics -i --interactive -p --pipeline -P --all-pipelines --ignore-build-cache --no-commit -s --single-item" -_dvc_remove="--dry -o --outs -p --purge --force" -_dvc_gc="-a --all-branches -T --all-tags -c --cloud -r --remote -f --force -p --project" -_dvc_config="-u --unset --local --system --global" -_dvc_checkout="$(compgen -G '*.dvc')" -_dvc_move="$(compgen -G '*')" -_dvc_lock="$(compgen -G '*.dvc')" -_dvc_unlock="$(compgen -G '*.dvc')" -_dvc_import="--resume" _dvc_remote="" -_dvc_metrics="" -_dvc_install="" +_dvc_remove="--dry -o --outs -p --purge --force" +_dvc_repro="--dry -f --force -s --single-item -c --cwd -m --metrics -i --interactive -p --pipeline -P --all-pipelines --ignore-build-cache --no-commit -s --single-item" _dvc_root="" -_dvc_pipeline="" +_dvc_run="--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache -M --metrics-no-cache -y --yes --overwrite-dvc-file --ignore-build-cache --remove-outs --no-commit -w --wdir" +_dvc_status="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud -q --quiet" +_dvc_unlock="$(compgen -G '*.dvc')" # Notes: # diff --git a/scripts/completion/dvc.zsh b/scripts/completion/dvc.zsh index 4ceef21b9..0faddee63 100644 --- a/scripts/completion/dvc.zsh +++ b/scripts/completion/dvc.zsh @@ -15,28 +15,28 @@ _dvc_commands() { "add:Take data files or directories under DVC control." "cache:Manage cache settings." "checkout:Checkout data files from cache." - "commit:Save changed data to cache and update DVC files." + "commit:Save changed data to cache and update DVC-files." "config:Get or set config settings." - "destroy:Remove DVC files, local DVC config and data cache." + "destroy:Remove DVC-files, local DVC config and data cache." "diff:Show a diff of a DVC controlled data file or a directory." "fetch:Fetch data files from a DVC remote storage." "gc:Collect unused data from DVC cache or a remote storage." "import:Download or copy files from URL and take under DVC control." "init:Initialize DVC in the current directory." "install:Install DVC git hooks into the repository." - "lock:Lock DVC file." + "lock:Lock DVC-file." "metrics:Commands to add, manage, collect and display metrics." "move:Rename or move a DVC controlled data file or a directory." "pipeline:Manage pipelines." "pull:Pull data files from a DVC remote storage." "push:Push data files to a DVC remote storage." "remote:Manage remote storage configuration." - "remove:Remove outputs of DVC file." - "repro:Check for changes and reproduce DVC file and dependencies." + "remove:Remove outputs of DVC-file." + "repro:Check for changes and reproduce DVC-file and dependencies." "root:Relative path to project's directory." "run:Generate a stage file from a command and execute the command." "status:Show changed stages, compare local cache and a remote storage." - "unlock:Unlock DVC file." + "unlock:Unlock DVC-file." "unprotect:Unprotect data file/directory." "version:Show DVC version and system/environment informaion." ) @@ -44,21 +44,21 @@ _dvc_commands() { _describe 'dvc commands' _commands } -_dvc_global_options=( - "(-)"{-h,--help}"[Show help message related to the command]" - "(-)"{-q,--quiet}"[Be quiet.]" - "(-)"{-V,--verbose}"[Be verbose.]" +_dvc_options=( + "(-)"{-h,--help}"[Show help message.]" + "(-)"{-V,--version}"[Show program's version]" ) -_dvc_options=( - "(-)"{-h,--help}"[Show this help message and exit]" - "(-)"{-v,--version}"[Show program's version]" +_dvc_global_options=( + "(-)"{-h,--help}"[Show help message related to the command.]" + "(-)"{-q,--quiet}"[Be quiet.]" + "(-)"{-v,--verbose}"[Be verbose.]" ) _dvc_add=( {-R,--recursive}"[Recursively add each file under the directory.]" - {-f,--file}"[Specify name of the stage file.]:File:_files" "--no-commit[Don't put files/directories into cache.]" + {-f,--file}"[Specify name of the DVC-file it generates.]:File:_files" "1:File:_files" ) @@ -67,27 +67,28 @@ _dvc_cache=( ) _dvc_checkout=( - {-f,--force}"[Do not prompt when removing working directory files.]" {-d,--with-deps}"[Checkout all dependencies of the specified target.]" + {-f,--force}"[Do not prompt when removing working directory files.]" + {-R,--recursive}"[Checkout all subdirectories of the specified directory.]" "1:Stages:_files -g '(*.dvc|Dvcfile)'" ) _dvc_commit=( "*:Stages:_files -g '(*.dvc|Dvcfile)'" - {-d,--with-deps}"[Commit all dependencies of the specified target.]" {-f,--force}"[Commit even if checksums for dependencies/outputs changed.]" + {-d,--with-deps}"[Commit all dependencies of the specified target.]" {-R,--recursive}"[Commit cache for subdirectories of the specified directory.]" ) _dvc_config=( - {-u,--unset}"[Unset option.]" - "--local[Unset option.]" - "--system[Use system config.]" "--global[Use global config.]" + "--system[Use system config.]" + "--local[Use local config.]" + {-u,--unset}"[Unset option.]" ) _dvc_destroy=( - {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]" + {-f,--force}"[Force destruction.]" ) _dvc_diff=( @@ -95,35 +96,36 @@ _dvc_diff=( ) _dvc_fetch=( + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + "--show-checksums[Show checksums instead of file names.]" + {-r,--remote}"[Remote repository to fetch from.]:Remote repository:" + {-a,--all-branches}"[Fetch cache for all branches.]" + {-T,--all-tags}"[Fetch cache for all tags.]" + {-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]" + {-R,--recursive}"[Fetch cache for subdirectories of specified directory.]" "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--show-checksums[Show checksums instead of file names]" - {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:" - {-r,--remote}"[Remote repository to pull from]:Remote repository:" - {-a,--all-branches}"[Fetch cache for all branches]" - {-T,--all-tags}"[Fetch cache for all tags]" - {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]" - {-R,--recursive}"[Fetch cache for subdirectories of the specified directory.]" ) _dvc_gc=( - {-a,--all-branches}"[Collect garbage for all branches]" - {-T,--all-tags}"[Collect garbage for all tags]" - {-c,--cloud}"[Collect garbage in remote repository]" - {-r,--remote}"[Remote repository to collect garbage in]:Remote repository:" - {-f,--force}"[Force garbage collection - automatically agree to all prompts.]" - - {-p,--projects}"[Keep data files required by these projects in addition to the current one. Useful if you share a single cache across repos.]:Repos:_files" + {-a,--all-branches}"[Keep data files for the tips of all git branches.]" + {-T,--all-tags}"[Keep data files for all git tags.]" + {-c,--cloud}"[Collect garbage in remote repository.]" + {-r,--remote}"[Remote storage to collect garbage in.]:Remote repository:" + {-f,--force}"[Force garbage collection - automatically agree to all prompts.]:Repos:_files" + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + {-p,--projects}"[Keep data files required by these projects in addition to the current one.]:Repos:_files" ) _dvc_import=( "--resume[Resume previously started download.]" + {-f,--file}"[Specify name of the DVC-file it generates.]:File:_files" "1:URL:" "2:Output:" ) _dvc_init=( - "--no-scm[Initiate dvc in directory that is not tracket by any scm tool]" - {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]" + "--no-scm[Initiate dvc in directory that is not tracked by any scm tool.]" + {-f,--force}"[Overwrite existing '.dvc' directory. This operation removes local cache.]" ) _dvc_install=() @@ -146,26 +148,26 @@ _dvc_pipeline=( ) _dvc_pull=( - "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--show-checksums[Show checksums instead of file names]" - {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:" - {-r,--remote}"[Remote repository to pull from]:Remote repository:" - {-a,--all-branches}"[Fetch cache for all branches]" - {-T,--all-tags}"[Fetch cache for all tags]" - {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]" + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + "--show-checksums[Show checksums instead of file names.]" + {-r,--remote}"[Remote repository to pull from.]:Remote repository:" + {-a,--all-branches}"[Fetch cache for all branches.]" + {-T,--all-tags}"[Fetch cache for all tags.]" + {-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]" {-f,--force}"[Do not prompt when removing working directory files.]" {-R,--recursive}"[Pull cache for subdirectories of the specified directory.]" + "*:Stages:_files -g '(*.dvc|Dvcfile)'" ) _dvc_push=( + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + "--show-checksums[Show checksums instead of file names.]" + {-r,--remote}"[Remote repository to push to.]:Remote repository:" + {-a,--all-branches}"[Push cache for all branches.]" + {-T,--all-tags}"[Push cache for all tags.]" + {-d,--with-deps}"[Push cache for all dependencies of the specified target.]" + {-R,--recursive}"[Push cache from subdirectories of specified directory.]" "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--show-checksums[Show checksums instead of file names]" - {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:" - {-r,--remote}"[Remote repository to pull from]:Remote repository:" - {-a,--all-branches}"[Fetch cache for all branches]" - {-T,--all-tags}"[Fetch cache for all tags]" - {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]" - {-R,--recursive}"[Push cache for subdirectories of the specified directory.]" ) _dvc_remote=( @@ -175,53 +177,58 @@ _dvc_remote=( _dvc_remove=( "*:Stages:_files -g '(*.dvc|Dvcfile)'" "--dry[Only print the commands that would be executed without actually executing]" - {-o,--outs}"[Only remove DVC file outputs.(default)]" - {-p,--purge}"[Remove DVC file and all its outputs]" + {-o,--outs}"[Only remove DVC-file outputs. (Default)]" + {-p,--purge}"[Remove DVC-file and all its outputs.]" {-f,--force}"[Force purge.]" ) _dvc_repro=( - "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--dry[Only print the commands that would be executed without actually executing]" - "--ignore-build-cache[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]" - "--no-commit[Don't put files/directories into cache.]" {-f,--force}"[Reproduce even if dependencies were not changed.]" {-s,--single-item}"[Reproduce only single data item without recursive dependencies check.]" + {-c,--cwd}"[Directory within your repo to reproduce from.]:CWD:_files -/" {-m,--metrics}"[Show metrics after reproduction.]" + "--dry[Only print the commands that would be executed without actually executing]" {-i,--interactive}"[Ask for confirmation before reproducing each stage.]" {-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]" - {-P,--all-pipelines}"[Reproduce all pipelines in the project.]" - {-d,--downstream}"[Reproduce the pipeline starting from the specified stage.]" + {-P,--all-pipelines}"[Reproduce all pipelines in the repo.]" + {-R,--recursive}"[Reproduce all stages in the specified directory.]" + "--ignore-build-cache[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]" + "--no-commit[Don't put files/directories into cache.]" + "--downstream[Reproduce the pipeline starting from the specified stage.]" + "*:Stages:_files -g '(*.dvc|Dvcfile)'" ) _dvc_root=() _dvc_run=( - "--no-exec[Only create stage file without actually running it.]" - "--overwrite-dvcfile[Overwrite existing dvc file without asking for confirmation.]" - "--ignore-build-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]" - "--no-commit[Don't put files/directories into cache.]" - {-f,--file}"[Specify name of the stage file.]:File:_files" - {-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/" - {-w,--wdir}"[Directory to run your command and place state file in.]:WDIR:_files -/" "*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files" - "*"{-o,--outs}"[Declare output data file or data directory.]:Output data:_files" - "*"{-O,--outs-no-cache}"[Declare output regular file or directory.]:Output regular:_files" - "*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache)]:Metrics (no cache):_files" - "*"{-m,--metrics}"[Declare output metric file or directory]:Metrics:_files" - {-y,--yes}"[Automatic 'yes' answer to all prompts.]" + "*"{-o,--outs}"[Declare output file or directory.]:Output data:_files" + "*"{-O,--outs-no-cache}"[Declare output file or directory (do not put into DVC cache).]:Output regular:_files" + "*"{-m,--metrics}"[Declare output metric file or directory.]:Metrics:_files" + "*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache).]:Metrics (no cache):_files" + {-f,--file}"[Specify name of the DVC-file it generates.]:File:_files" + {-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/" + {-w,--wdir}"[Directory within your repo to run your command in.]:WDIR:_files -/" + "--no-exec[Only create stage file without actually running it.]" + {-y,--yes}"[Deprecated, use --overwrite-dvcfile instead]" + "--overwrite-dvcfile[Overwrite existing DVC-file without asking for confirmation.]" + "--ignore-build-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]" + "--remove-outs[Deprecated, this is now the default behavior]" + "--no-commit[Don't put files/directories into cache.]" + "--outs-persist[Declare output file or directory that will not be removed upon repro.]:Output persistent:_files" + "--outs-persist-no-cache[Declare output file or directory that will not be removed upon repro (do not put into DVC cache).]:Output persistent regular:_files" ) _dvc_status=( + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + "--show-checksums[Show checksums instead of file names.]" + {-q,--quiet}"[Suppresses all output. Exit with 0 if pipeline is up to date, otherwise 1.]" + {-c,--cloud}"[Show status of a local cache compared to a remote repository.]" + {-r,--remote}"[Remote repository to compare local cache to.]:Remote repository:" + {-a,--all-branches}"[Show status of a local cache compared to a remote repository for all branches.]" + {-T,--all-tags}"[Show status of a local cache compared to a remote repository for all tags.]" + {-d,--with-deps}"[Show status for all dependencies of the specified target.]" "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--show-checksums[Show checksums instead of file names]" - {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:" - {-r,--remote}"[Remote repository to pull from]:Remote repository:" - {-a,--all-branches}"[Fetch cache for all branches]" - {-T,--all-tags}"[Fetch cache for all tags]" - {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]" - {-c,--cloud}"[Show status of a local cache compared to a remote repository]" - {-q,--quiet}"[Suppresses all output. Exit with 0 if pipeline is up]" ) _dvc_unlock=( diff --git a/setup.py b/setup.py index 6eda216c7..d8a0c4972 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ install_requires = [ "humanize>=0.5.1", "dulwich>=0.19.11", "ruamel.yaml>=0.15.91", + "psutil==5.6.2", ] # Extra dependencies for remote integrations @@ -76,7 +77,7 @@ tests_requirements = [ "wheel>=0.31.1", "pydot>=1.2.4", # Test requirements: - "pytest>=4.4.0", + "pytest>=4.6.0", "pytest-timeout>=1.3.3", "pytest-cov>=2.6.1", "pytest-xdist>=1.26.1", diff --git a/tests/__main__.py b/tests/__main__.py index 6a638ae93..2068d7f7d 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -15,6 +15,6 @@ params = " ".join(sys.argv[1:]) cmd = ( "py.test -v -n=4 --timeout=600 --timeout_method=thread --log-level=debug" - " --cov=dvc {params} ".format(params=params) + " --cov=dvc {params} --durations=0".format(params=params) ) check_call(cmd, shell=True) diff --git a/tests/basic_env.py b/tests/basic_env.py index 0c0b7ab70..8fc20e08c 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -57,6 +57,10 @@ class TestDirFixture(object): root_dir = self.mkdtemp() self._root_dir = os.path.realpath(root_dir) + @property + def root_dir(self): + return self._root_dir + def _pushd(self, d): if not hasattr(self, "_saved_dir"): self._saved_dir = os.path.realpath(os.curdir) @@ -193,9 +197,8 @@ class TestDvcDataFileFixture(TestDvcGitInitializedFixture): shutil.copytree(self.dvc.cache.local.cache_dir, self.cache_dir) class MockConfig: - system = None - glob = None - local = None + level = 1 + force = False default = True name = self.REMOTE url = self.cache_dir diff --git a/tests/conftest.py b/tests/conftest.py index 9de032b64..7e7d1208e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -68,12 +68,14 @@ def git(repo_dir): @pytest.fixture def dvc_repo(repo_dir, git): + dvc = None try: dvc = DvcRepo.init(repo_dir._root_dir) dvc.scm.commit("init dvc") yield dvc finally: - dvc.scm.git.close() + if dvc: + dvc.scm.git.close() here = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 0105c93da..9cef92c8c 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -114,7 +114,7 @@ class TestAddCmdDirectoryRecursive(TestDvc): with self._caplog.at_level(logging.WARNING, logger="dvc"): assert main(["add", "--recursive", "large-dir"]) == 0 - assert warning in self._caplog.text + assert warning in self._caplog.messages class TestAddDirectoryWithForwardSlash(TestDvc): diff --git a/tests/func/test_config.py b/tests/func/test_config.py index 0de3bafd9..f6cae78d3 100644 --- a/tests/func/test_config.py +++ b/tests/func/test_config.py @@ -1,7 +1,6 @@ import configobj from dvc.main import main -from dvc.command.config import CmdConfig from tests.basic_env import TestDvc @@ -67,38 +66,20 @@ class TestConfigCLI(TestDvc): self._do_test(True) def test_non_existing(self): - # FIXME check stdout/err - ret = main(["config", "non_existing_section.field"]) - self.assertEqual(ret, 1) + self.assertEqual(ret, 251) ret = main(["config", "global.non_existing_field"]) - self.assertEqual(ret, 1) + self.assertEqual(ret, 251) ret = main(["config", "non_existing_section.field", "-u"]) - self.assertEqual(ret, 1) + self.assertEqual(ret, 251) ret = main(["config", "global.non_existing_field", "-u"]) - self.assertEqual(ret, 1) + self.assertEqual(ret, 251) ret = main(["config", "core.remote", "myremote"]) self.assertEqual(ret, 0) ret = main(["config", "core.non_existing_field", "-u"]) - self.assertEqual(ret, 1) - - def test_failed_write(self): - class A(object): - system = False - glob = False - local = False - name = "core.remote" - value = "myremote" - unset = False - - args = A() - cmd = CmdConfig(args) - - cmd.configobj.write = None - ret = cmd.run() - self.assertNotEqual(ret, 0) + self.assertEqual(ret, 251) diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index d0e20bbed..97daa1a91 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -39,6 +39,7 @@ from tests.utils import spy TEST_REMOTE = "upstream" TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE) TEST_CONFIG = { + Config.SECTION_CACHE: {}, Config.SECTION_CORE: {Config.SECTION_CORE_REMOTE: TEST_REMOTE}, TEST_SECTION: {Config.SECTION_REMOTE_URL: ""}, } @@ -870,12 +871,12 @@ class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc): checksum_bar = file_md5(self.BAR)[0] self.message_header = ( "Some of the cache files do not exist neither locally " - "nor on remote. Missing cache files: \n" + "nor on remote. Missing cache files: " ) - self.message_bar_part = "\nname: {}, md5: {}\n".format( + self.message_bar_part = "name: {}, md5: {}".format( self.BAR, checksum_bar ) - self.message_foo_part = "\nname: {}, md5: {}\n".format( + self.message_foo_part = "name: {}, md5: {}".format( self.FOO, checksum_foo ) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index fd403cc1b..9f1b07d59 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -5,7 +5,6 @@ from dvc.remote.base import RemoteBASE from mock import patch from dvc.main import main -from dvc.command.remote import CmdRemoteAdd from dvc.config import Config from tests.basic_env import TestDvc @@ -33,22 +32,6 @@ class TestRemote(TestDvc): self.assertEqual(main(["remote", "list"]), 0) - def test_failed_write(self): - class A(object): - system = False - glob = False - local = False - name = "myremote" - url = "s3://remote" - unset = False - - args = A() - cmd = CmdRemoteAdd(args) - - cmd.configobj.write = None - ret = cmd.run() - self.assertNotEqual(ret, 0) - def test_relative_path(self): dname = os.path.join("..", "path", "to", "dir") ret = main(["remote", "add", "mylocal", dname]) @@ -64,7 +47,7 @@ class TestRemote(TestDvc): remote_name = "a" remote_url = "s3://bucket/name" self.assertEqual(main(["remote", "add", remote_name, remote_url]), 0) - self.assertEqual(main(["remote", "add", remote_name, remote_url]), 1) + self.assertEqual(main(["remote", "add", remote_name, remote_url]), 251) self.assertEqual( main(["remote", "add", "-f", remote_name, remote_url]), 0 ) diff --git a/tests/func/test_version.py b/tests/func/test_version.py new file mode 100644 index 000000000..a9089c502 --- /dev/null +++ b/tests/func/test_version.py @@ -0,0 +1,37 @@ +import re + +from dvc.main import main + + +def test_info_in_repo(dvc_repo, caplog): + assert main(["version"]) == 0 + + assert re.search(re.compile(r"DVC version: \d+\.\d+\.\d+"), caplog.text) + assert re.search(re.compile(r"Python version: \d\.\d\.\d"), caplog.text) + assert re.search(re.compile(r"Platform: .*"), caplog.text) + assert re.search( + re.compile(r"Filesystem type \(cache directory\): .*"), caplog.text + ) + assert re.search( + re.compile(r"Filesystem type \(workspace\): .*"), caplog.text + ) + assert re.search( + re.compile(r"(Cache: (.*link - (True|False)(,\s)?){3})"), caplog.text + ) + + +def test_info_outside_of_repo(repo_dir, caplog): + assert main(["version"]) == 0 + + assert re.search(re.compile(r"DVC version: \d+\.\d+\.\d+"), caplog.text) + assert re.search(re.compile(r"Python version: \d\.\d\.\d"), caplog.text) + assert re.search(re.compile(r"Platform: .*"), caplog.text) + assert re.search( + re.compile(r"Filesystem type \(workspace\): .*"), caplog.text + ) + assert not re.search( + re.compile(r"Filesystem type \(cache directory\): .*"), caplog.text + ) + assert not re.search( + re.compile(r"(Cache: (.*link - (True|False)(,\s)?){3})"), caplog.text + ) diff --git a/tests/unit/command/test_version.py b/tests/unit/command/test_version.py deleted file mode 100644 index 57c9184b5..000000000 --- a/tests/unit/command/test_version.py +++ /dev/null @@ -1,25 +0,0 @@ -import re - -from dvc.command.version import CmdVersion -from dvc.cli import parse_args - - -def test_run_in_repo(repo_dir, dvc_repo): - cmd = CmdVersion(parse_args(["version"])) - ret = cmd.run_cmd() - assert ret == 0 - - -def test_run_outside_of_repo(repo_dir): - cmd = CmdVersion(parse_args(["version"])) - ret = cmd.run_cmd() - assert ret == 0 - - -def test_info(caplog): - cmd = CmdVersion(parse_args(["version"])) - cmd.run() - - assert re.search(re.compile(r"DVC version: \d+\.\d+\.\d+"), caplog.text) - assert re.search(re.compile(r"Python version: \d\.\d\.\d"), caplog.text) - assert re.search(re.compile(r"Platform: .*"), caplog.text) diff --git a/tests/unit/test_progress.py b/tests/unit/test_progress.py index 8bcee4167..e617c1775 100644 --- a/tests/unit/test_progress.py +++ b/tests/unit/test_progress.py @@ -1,14 +1,19 @@ +import logging import mock -from unittest import TestCase - -from dvc.progress import ProgressCallback +from dvc.progress import progress, ProgressCallback -class TestProgressCallback(TestCase): +def test_quiet(caplog, capsys): + with caplog.at_level(logging.CRITICAL, logger="dvc"): + progress._print("something") + assert capsys.readouterr().out == "" + + +class TestProgressCallback: @mock.patch("dvc.progress.progress") def test_should_init_reset_progress(self, progress_mock): total_files_num = 1 ProgressCallback(total_files_num) - self.assertEqual([mock.call.reset()], progress_mock.method_calls) + assert [mock.call.reset()] == progress_mock.method_calls