Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

fixed: merge conflict

Browse Source
This commit is contained in:
Sanidhya Mangal
2019-06-11 00:19:47 +05:30
parent 1c14b8c579 c587c59dc6
commit e44e3ad127
52 changed files with 813 additions and 674 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.appveyor.yml
+2 -7
View File
@@ -1,8 +1,3 @@
branches:
only:
- master
- devel
cache:
- C:\ProgramData\chocolatey\bin -> appveyor.yml
- C:\ProgramData\chocolatey\lib -> appveyor.yml
@@ -32,8 +27,8 @@ environment:
RUBY: C:\Ruby25\bin
install:
- SET PATH=%RUBY%;%PATH%
- git clone -q https://github.com/nanaya-tachibana/oss-emulator.git
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%RUBY%;%PATH%"
- git clone -q https://github.com/iterative/oss-emulator.git
- gem install thor builder
- ps: cd oss-emulator
- ps: $OSSProcess = Start-Process -FilePath ruby -ArgumentList "bin\emulator -r store -p 50004" -PassThru -NoNewWindow
.deepsource.toml
+15
View File
@@ -0,0 +1,15 @@
version = 1
test_patterns = [
'*/tests/*'
]
exclude_patterns = [
]
[[analyzers]]
name = 'python'
enabled = true
runtime_version = '3.x.x'
README.rst
+2 -2
View File
@@ -31,8 +31,8 @@
|
**Data Science Version Control** or **DVC** is an **open-source** tool for data science and
machine learning projects. Key features:
**Data Version Control** or **DVC** is an **open-source** tool for data science and machine
learning projects. Key features:
#. simple **command line** Git-like experience. Does not require installing and maintaining
any databases. Does not depend on any proprietary online services;
dvc/cache.py
+16 -3
View File
@@ -7,6 +7,20 @@ import os
from dvc.config import Config
class CacheConfig(object):
def __init__(self, config):
self.config = config
def set_dir(self, dname, level=None):
from dvc.remote.config import RemoteConfig
configobj = self.config.get_configobj(level)
path = RemoteConfig.resolve_path(dname, configobj.filename)
self.config.set(
Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level
)
class Cache(object):
"""Class that manages cache locations of a dvc repo.
@@ -42,7 +56,7 @@ class Cache(object):
Config.SECTION_CACHE_PROTECTED: protected,
}
self.local = Remote(repo, settings)
self.local = Remote(repo, **settings)
self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
@@ -87,5 +101,4 @@ class Cache(object):
if not remote:
return None
settings = self.repo.config.get_remote_settings(remote)
return Remote(self.repo, settings)
return Remote(self.repo, name=remote)
dvc/command/add.py
+1 -1
View File
@@ -54,7 +54,7 @@ def add_parser(subparsers, parent_parser):
help="Don't put files/directories into cache.",
)
add_parser.add_argument(
"-f", "--file", help="Specify name of the DVC file it generates."
"-f", "--file", help="Specify name of the DVC-file it generates."
)
add_parser.add_argument(
"targets", nargs="+", help="Input files/directories."
dvc/command/cache.py
+7 -7
View File
@@ -3,18 +3,18 @@ from __future__ import unicode_literals
import argparse
from dvc.command.base import fix_subparsers, append_doc_link
from dvc.command.remote import CmdRemoteAdd
from dvc.command.config import CmdConfig
from dvc.cache import CacheConfig
class CmdCacheDir(CmdConfig):
def run(self):
self.args.name = "cache.dir"
self.args.value = CmdRemoteAdd.resolve_path(
self.args.value, self.configobj.filename
)
def __init__(self, args):
super(CmdCacheDir, self).__init__(args)
self.config = CacheConfig(self.config)
return super(CmdCacheDir, self).run()
def run(self):
self.config.set_dir(self.args.value, level=self.args.level)
return 0
def add_parser(subparsers, parent_parser):
dvc/command/checkout.py
+1 -1
View File
@@ -51,5 +51,5 @@ def add_parser(subparsers, parent_parser):
default=False,
help="Checkout all subdirectories of the specified directory.",
)
checkout_parser.add_argument("targets", nargs="*", help="DVC files.")
checkout_parser.add_argument("targets", nargs="*", help="DVC-files.")
checkout_parser.set_defaults(func=CmdCheckout)
dvc/command/commit.py
+2 -2
View File
@@ -34,7 +34,7 @@ class CmdCommit(CmdBase):
def add_parser(subparsers, parent_parser):
COMMIT_HELP = "Save changed data to cache and update DVC files."
COMMIT_HELP = "Save changed data to cache and update DVC-files."
commit_parser = subparsers.add_parser(
"commit",
@@ -65,6 +65,6 @@ def add_parser(subparsers, parent_parser):
help="Commit cache for subdirectories of the specified directory.",
)
commit_parser.add_argument(
"targets", nargs="*", default=None, help="DVC files."
"targets", nargs="*", default=None, help="DVC-files."
)
commit_parser.set_defaults(func=CmdCommit)
dvc/command/config.py
+22 -63
View File
@@ -1,12 +1,10 @@
from __future__ import unicode_literals
import os
import argparse
import logging
from dvc.command.base import CmdBaseNoRepo, append_doc_link
from dvc.config import Config
from dvc.exceptions import DvcException
logger = logging.getLogger(__name__)
@@ -14,87 +12,48 @@ logger = logging.getLogger(__name__)
class CmdConfig(CmdBaseNoRepo):
def __init__(self, args):
from dvc.repo import Repo, NotDvcRepoError
super(CmdConfig, self).__init__(args)
try:
dvc_dir = os.path.join(Repo.find_root(), Repo.DVC_DIR)
saved_exc = None
except NotDvcRepoError as exc:
dvc_dir = None
saved_exc = exc
self.config = Config(dvc_dir, validate=False)
if self.args.system:
self.configobj = self.config._system_config
elif self.args.glob:
self.configobj = self.config._global_config
elif self.args.local:
if dvc_dir is None:
raise saved_exc
self.configobj = self.config._local_config
else:
if dvc_dir is None:
raise saved_exc
self.configobj = self.config._repo_config
def _unset(self, section, opt=None, configobj=None):
if configobj is None:
configobj = self.configobj
try:
self.config.unset(configobj, section, opt)
self.config.save(configobj)
except DvcException:
logger.exception("failed to unset '{}'".format(self.args.name))
return 1
return 0
def _show(self, section, opt):
try:
self.config.show(self.configobj, section, opt)
except DvcException:
logger.exception("failed to show '{}'".format(self.args.name))
return 1
return 0
def _set(self, section, opt, value):
try:
self.config.set(self.configobj, section, opt, value)
self.config.save(self.configobj)
except DvcException:
logger.exception(
"failed to set '{}.{}' to '{}'".format(section, opt, value)
)
return 1
return 0
self.config = Config(validate=False)
def run(self):
section, opt = self.args.name.lower().strip().split(".", 1)
if self.args.unset:
return self._unset(section, opt)
self.config.unset(section, opt, level=self.args.level)
elif self.args.value is None:
return self._show(section, opt)
logger.info(self.config.get(section, opt, level=self.args.level))
else:
return self._set(section, opt, self.args.value)
self.config.set(
section, opt, self.args.value, level=self.args.level
)
return 0
parent_config_parser = argparse.ArgumentParser(add_help=False)
parent_config_parser.add_argument(
"--global",
dest="glob",
action="store_true",
default=False,
dest="level",
action="store_const",
const=Config.LEVEL_GLOBAL,
help="Use global config.",
)
parent_config_parser.add_argument(
"--system", action="store_true", default=False, help="Use system config."
"--system",
dest="level",
action="store_const",
const=Config.LEVEL_SYSTEM,
help="Use system config.",
)
parent_config_parser.add_argument(
"--local", action="store_true", default=False, help="Use local config."
"--local",
dest="level",
action="store_const",
const=Config.LEVEL_LOCAL,
help="Use local config.",
)
parent_config_parser.set_defaults(level=Config.LEVEL_REPO)
def add_parser(subparsers, parent_parser):
dvc/command/data_sync.py
+4
View File
@@ -113,7 +113,11 @@ def shared_parent_parser():
help="Show checksums instead of file names.",
)
shared_parent_parser.add_argument(
<<<<<<< HEAD
"targets", nargs="*", default=None, help="Limit the scope to these stage files."
=======
"targets", nargs="*", default=None, help="DVC-files."
>>>>>>> upstream/master
)
return shared_parent_parser
dvc/command/destroy.py
+1 -1
View File
@@ -35,7 +35,7 @@ class CmdDestroy(CmdBase):
def add_parser(subparsers, parent_parser):
DESTROY_HELP = "Remove DVC files, local DVC config and data cache."
DESTROY_HELP = "Remove DVC-files, local DVC config and data cache."
destroy_parser = subparsers.add_parser(
"destroy",
dvc/command/imp.py
+1 -1
View File
@@ -65,6 +65,6 @@ def add_parser(subparsers, parent_parser):
"out", nargs="?", help="Destination path to put files to."
)
import_parser.add_argument(
"-f", "--file", help="Specify name of the DVC file it generates."
"-f", "--file", help="Specify name of the DVC-file it generates."
)
import_parser.set_defaults(func=CmdImport)
dvc/command/lock.py
+4 -4
View File
@@ -37,7 +37,7 @@ class CmdUnlock(CmdLockBase):
def add_parser(subparsers, parent_parser):
LOCK_HELP = "Lock DVC file."
LOCK_HELP = "Lock DVC-file."
lock_parser = subparsers.add_parser(
"lock",
parents=[parent_parser],
@@ -45,10 +45,10 @@ def add_parser(subparsers, parent_parser):
help=LOCK_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
lock_parser.add_argument("targets", nargs="+", help="DVC files.")
lock_parser.add_argument("targets", nargs="+", help="DVC-files.")
lock_parser.set_defaults(func=CmdLock)
UNLOCK_HELP = "Unlock DVC file."
UNLOCK_HELP = "Unlock DVC-file."
unlock_parser = subparsers.add_parser(
"unlock",
parents=[parent_parser],
@@ -56,5 +56,5 @@ def add_parser(subparsers, parent_parser):
help=UNLOCK_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
unlock_parser.add_argument("targets", nargs="+", help="DVC files.")
unlock_parser.add_argument("targets", nargs="+", help="DVC-files.")
unlock_parser.set_defaults(func=CmdUnlock)
dvc/command/move.py
+1 -1
View File
@@ -27,7 +27,7 @@ def add_parser(subparsers, parent_parser):
MOVE_HELP = "Rename or move a DVC controlled data file or a directory."
MOVE_DESCRIPTION = (
"Rename or move a DVC controlled data file or a directory.\n"
"It renames and modifies the corresponding DVC file to reflect the"
"It renames and modifies the corresponding DVC-file to reflect the"
" changes."
)
dvc/command/pipeline.py
+3 -3
View File
@@ -209,14 +209,14 @@ def add_parser(subparsers, parent_parser):
"--commands",
action="store_true",
default=False,
help="Print commands instead of paths to DVC files.",
help="Print commands instead of paths to DVC-files.",
)
pipeline_show_group.add_argument(
"-o",
"--outs",
action="store_true",
default=False,
help="Print output files instead of paths to DVC files.",
help="Print output files instead of paths to DVC-files.",
)
pipeline_show_parser.add_argument(
"-l",
@@ -244,7 +244,7 @@ def add_parser(subparsers, parent_parser):
help="Output DAG as Dependencies Tree.",
)
pipeline_show_parser.add_argument(
"targets", nargs="*", help="DVC files. 'Dvcfile' by default."
"targets", nargs="*", help="DVC-files. 'Dvcfile' by default."
)
pipeline_show_parser.set_defaults(func=CmdPipelineShow)
dvc/command/remote.py
+35 -103
View File
@@ -1,130 +1,62 @@
from __future__ import unicode_literals
import argparse
import os
import re
import logging
from dvc.command.base import append_doc_link, fix_subparsers
from dvc.command.config import CmdConfig
from dvc.config import Config
from dvc.remote.config import RemoteConfig
logger = logging.getLogger(__name__)
class CmdRemoteAdd(CmdConfig):
@staticmethod
def resolve_path(path, config_file):
"""Resolve path relative to config file location.
class CmdRemoteConfig(CmdConfig):
def __init__(self, args):
super(CmdRemoteConfig, self).__init__(args)
self.config = RemoteConfig(self.config)
Args:
path: Path to be resolved.
config_file: Path to config file, which `path` is specified
relative to.
Returns:
Path relative to the `config_file` location. If `path` is an
absolute path then it will be returned without change.
"""
if os.path.isabs(path):
return path
return os.path.relpath(path, os.path.dirname(config_file))
class CmdRemoteAdd(CmdRemoteConfig):
def run(self):
from dvc.remote import _get, RemoteLOCAL
remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
if remote == RemoteLOCAL and not self.args.url.startswith("remote://"):
self.args.url = self.resolve_path(
self.args.url, self.configobj.filename
)
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
if (section in self.configobj.keys()) and not self.args.force:
logger.error(
"Remote with name {} already exists. "
"Use -f (--force) to overwrite remote "
"with new value".format(self.args.name)
)
return 1
ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
if ret != 0:
return ret
if self.args.default:
msg = "Setting '{}' as a default remote.".format(self.args.name)
logger.info(msg)
ret = self._set(
Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, self.args.name
)
return ret
class CmdRemoteRemove(CmdConfig):
def _remove_default(self, config):
core = config.get(Config.SECTION_CORE, None)
if core is None:
return 0
default = core.get(Config.SECTION_CORE_REMOTE, None)
if default is None:
return 0
if default == self.args.name:
return self._unset(
Config.SECTION_CORE,
opt=Config.SECTION_CORE_REMOTE,
configobj=config,
)
def run(self):
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
ret = self._unset(section)
if ret != 0:
return ret
for configobj in [
self.config._local_config,
self.config._repo_config,
self.config._global_config,
self.config._system_config,
]:
self._remove_default(configobj)
self.config.save(configobj)
if configobj == self.configobj:
break
self.config.add(
self.args.name,
self.args.url,
force=self.args.force,
default=self.args.default,
level=self.args.level,
)
return 0
class CmdRemoteModify(CmdConfig):
class CmdRemoteRemove(CmdRemoteConfig):
def run(self):
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
self.args.name = "{}.{}".format(section, self.args.option)
return super(CmdRemoteModify, self).run()
self.config.remove(self.args.name, level=self.args.level)
return 0
class CmdRemoteDefault(CmdConfig):
class CmdRemoteModify(CmdRemoteConfig):
def run(self):
self.args.value = self.args.name
self.args.name = "core.remote"
return super(CmdRemoteDefault, self).run()
self.config.modify(
self.args.name,
self.args.option,
self.args.value,
level=self.args.level,
)
return 0
class CmdRemoteList(CmdConfig):
class CmdRemoteDefault(CmdRemoteConfig):
def run(self):
for section in self.configobj.keys():
r = re.match(Config.SECTION_REMOTE_REGEX, section)
if r:
name = r.group("name")
url = self.configobj[section].get(
Config.SECTION_REMOTE_URL, ""
)
logger.info("{}\t{}".format(name, url))
self.config.set_default(
self.args.name, unset=self.args.unset, level=self.args.level
)
return 0
class CmdRemoteList(CmdRemoteConfig):
def run(self):
for name, url in self.config.list(level=self.args.level).items():
logger.info("{}\t{}".format(name, url))
return 0
dvc/command/remove.py
+4 -4
View File
@@ -43,7 +43,7 @@ class CmdRemove(CmdBase):
def add_parser(subparsers, parent_parser):
REMOVE_HELP = "Remove outputs of DVC file."
REMOVE_HELP = "Remove outputs of DVC-file."
remove_parser = subparsers.add_parser(
"remove",
parents=[parent_parser],
@@ -57,14 +57,14 @@ def add_parser(subparsers, parent_parser):
"--outs",
action="store_true",
default=True,
help="Only remove DVC file outputs.(default)",
help="Only remove DVC-file outputs. (Default)",
)
remove_parser_group.add_argument(
"-p",
"--purge",
action="store_true",
default=False,
help="Remove DVC file and all its outputs.",
help="Remove DVC-file and all its outputs.",
)
remove_parser.add_argument(
"-f",
@@ -73,5 +73,5 @@ def add_parser(subparsers, parent_parser):
default=False,
help="Force purge.",
)
remove_parser.add_argument("targets", nargs="+", help="DVC files.")
remove_parser.add_argument("targets", nargs="+", help="DVC-files.")
remove_parser.set_defaults(func=CmdRemove)
dvc/command/repro.py
+2 -2
View File
@@ -58,7 +58,7 @@ class CmdRepro(CmdBase):
def add_parser(subparsers, parent_parser):
REPRO_HELP = "Check for changes and reproduce DVC file and dependencies."
REPRO_HELP = "Check for changes and reproduce DVC-file and dependencies."
repro_parser = subparsers.add_parser(
"repro",
parents=[parent_parser],
@@ -69,7 +69,7 @@ def add_parser(subparsers, parent_parser):
repro_parser.add_argument(
"targets",
nargs="*",
help="DVC file to reproduce (default - 'Dvcfile').",
help="DVC-file to reproduce (default - 'Dvcfile').",
)
repro_parser.add_argument(
"-f",
dvc/command/run.py
+2 -2
View File
@@ -127,7 +127,7 @@ def add_parser(subparsers, parent_parser):
"(do not put into DVC cache).",
)
run_parser.add_argument(
"-f", "--file", help="Specify name of the DVC file it generates."
"-f", "--file", help="Specify name of the DVC-file it generates."
)
run_parser.add_argument(
"-c", "--cwd", default=None, help="Deprecated, use -w and -f instead."
@@ -155,7 +155,7 @@ def add_parser(subparsers, parent_parser):
"--overwrite-dvcfile",
action="store_true",
default=False,
help="Overwrite existing dvc file without asking for confirmation.",
help="Overwrite existing DVC-file without asking for confirmation.",
)
run_parser.add_argument(
"--ignore-build-cache",
dvc/command/tag.py
+6 -6
View File
@@ -82,14 +82,14 @@ def add_parser(subparsers, parent_parser):
)
tag_add_parser.add_argument("tag", help="Dvc tag.")
tag_add_parser.add_argument(
"targets", nargs="*", default=[None], help="Dvc files."
"targets", nargs="*", default=[None], help="DVC-files."
)
tag_add_parser.add_argument(
"-d",
"--with-deps",
action="store_true",
default=False,
help="Add tag for all dependencies of the specified DVC file.",
help="Add tag for all dependencies of the specified DVC-file.",
)
tag_add_parser.add_argument(
"-R",
@@ -109,14 +109,14 @@ def add_parser(subparsers, parent_parser):
)
tag_remove_parser.add_argument("tag", help="Dvc tag.")
tag_remove_parser.add_argument(
"targets", nargs="*", default=[None], help="Dvc files."
"targets", nargs="*", default=[None], help="DVC-files."
)
tag_remove_parser.add_argument(
"-d",
"--with-deps",
action="store_true",
default=False,
help="Remove tag for all dependencies of the specified DVC file.",
help="Remove tag for all dependencies of the specified DVC-file.",
)
tag_remove_parser.add_argument(
"-R",
@@ -135,14 +135,14 @@ def add_parser(subparsers, parent_parser):
help=TAG_LIST_HELP,
)
tag_list_parser.add_argument(
"targets", nargs="*", default=[None], help="Dvc files."
"targets", nargs="*", default=[None], help="DVC-files."
)
tag_list_parser.add_argument(
"-d",
"--with-deps",
action="store_true",
default=False,
help="List tags for all dependencies of the specified DVC file.",
help="List tags for all dependencies of the specified DVC-file.",
)
tag_list_parser.add_argument(
"-R",
dvc/command/version.py
+76 -4
View File
@@ -1,11 +1,18 @@
from __future__ import unicode_literals
import os
import psutil
import platform
import argparse
import logging
import uuid
from dvc.utils.compat import Path
from dvc.repo import Repo
from dvc.command.base import CmdBaseNoRepo, append_doc_link
from dvc.version import __version__
from dvc.exceptions import DvcException, NotDvcRepoError
from dvc.system import System
logger = logging.getLogger(__name__)
@@ -13,18 +20,83 @@ logger = logging.getLogger(__name__)
class CmdVersion(CmdBaseNoRepo):
def run(self):
dvc_version = __version__
python_version = platform.python_version()
platform_type = platform.platform()
info = (
"DVC version: {dvc_version}\n"
"Python version: {python_version}\n"
"Platform: {platform_type}"
"Platform: {platform_type}\n"
).format(
dvc_version=__version__,
python_version=platform.python_version(),
platform_type=platform.platform(),
dvc_version=dvc_version,
python_version=python_version,
platform_type=platform_type,
)
try:
repo = Repo()
root_directory = repo.root_dir
info += (
"Cache: {cache}\n"
"Filesystem type (cache directory): {fs_cache}\n"
).format(
cache=self.get_linktype_support_info(repo),
fs_cache=self.get_fs_type(repo.cache.local.cache_dir),
)
except NotDvcRepoError:
root_directory = os.getcwd()
info += ("Filesystem type (workspace): {fs_root}").format(
fs_root=self.get_fs_type(os.path.abspath(root_directory))
)
logger.info(info)
return 0
@staticmethod
def get_fs_type(path):
partition = {
Path(part.mountpoint): (part.fstype, part.device)
for part in psutil.disk_partitions()
}
for parent in list(Path(path).parents):
if parent in partition:
return partition[parent]
return ("unkown", "none")
@staticmethod
def get_linktype_support_info(repo):
links = {
"reflink": System.reflink,
"hardlink": System.hardlink,
"symlink": System.symlink,
}
fname = "." + str(uuid.uuid4())
src = os.path.join(repo.cache.local.cache_dir, fname)
open(src, "w").close()
dst = os.path.join(repo.root_dir, fname)
cache = []
for name, link in links.items():
try:
link(src, dst)
os.unlink(dst)
supported = True
except DvcException:
supported = False
cache.append(
"{name} - {supported}".format(
name=name, supported=True if supported else False
)
)
os.remove(src)
return ", ".join(cache)
def add_parser(subparsers, parent_parser):
VERSION_HELP = "Show DVC version and system/environment informaion."
dvc/config.py
+205 -230
View File
@@ -2,15 +2,17 @@
from __future__ import unicode_literals
from dvc.utils.compat import str, open, urlparse
from dvc.utils.compat import str, open
import os
import re
import copy
import errno
import configobj
import logging
from schema import Schema, Optional, And, Use, Regex
from dvc.exceptions import DvcException
from schema import Schema, Optional, And, Use, Regex, SchemaError
from dvc.exceptions import DvcException, NotDvcRepoError
logger = logging.getLogger(__name__)
@@ -20,12 +22,12 @@ class ConfigError(DvcException):
Args:
msg (str): error message.
ex (Exception): optional exception that has caused this error.
cause (Exception): optional exception that has caused this error.
"""
def __init__(self, msg, ex=None):
def __init__(self, msg, cause=None):
super(ConfigError, self).__init__(
"config file error: {}".format(msg), ex
"config file error: {}".format(msg), cause=cause
)
@@ -43,24 +45,6 @@ def supported_cache_type(types):
return True
def supported_loglevel(level):
"""Checks if log level config option has a valid value.
Args:
level (str): log level name.
"""
return level in ["info", "debug", "warning", "error"]
def supported_cloud(cloud):
"""Checks if obsoleted cloud option has a valid value.
Args:
cloud (str): cloud type name.
"""
return cloud in ["aws", "gcp", "local", ""]
def is_bool(val):
"""Checks that value is a boolean.
@@ -109,6 +93,23 @@ def is_percent(val):
return int(val) >= 0 and int(val) <= 100
class Choices(object):
"""Checks that value belongs to the specified set of values
Args:
*choices: pass allowed values as arguments, or pass a list or
tuple as a single argument
"""
def __init__(self, *choices):
if len(choices) == 1 and isinstance(choices[0], (list, tuple)):
choices = choices[0]
self.choices = choices
def __call__(self, value):
return value in self.choices
class Config(object): # pylint: disable=too-many-instance-attributes
"""Class that manages configuration files for a dvc repo.
@@ -134,11 +135,18 @@ class Config(object): # pylint: disable=too-many-instance-attributes
CONFIG = "config"
CONFIG_LOCAL = "config.local"
LEVEL_LOCAL = 0
LEVEL_REPO = 1
LEVEL_GLOBAL = 2
LEVEL_SYSTEM = 3
BOOL_SCHEMA = And(str, is_bool, Use(to_bool))
SECTION_CORE = "core"
SECTION_CORE_LOGLEVEL = "loglevel"
SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
SECTION_CORE_LOGLEVEL_SCHEMA = And(
Use(str.lower), Choices("info", "debug", "warning", "error")
)
SECTION_CORE_REMOTE = "remote"
SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA
SECTION_CORE_INTERACTIVE = "interactive"
@@ -173,7 +181,11 @@ class Config(object): # pylint: disable=too-many-instance-attributes
# backward compatibility
SECTION_CORE_CLOUD = "cloud"
SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud)
SECTION_CORE_CLOUD_SCHEMA = And(
Use(str.lower),
# only the clouds which were supported by obsoleted cloud config
Choices("aws", "gcp", "local", ""),
)
SECTION_CORE_STORAGEPATH = "storagepath"
SECTION_CORE_SCHEMA = {
@@ -201,6 +213,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes
SECTION_AWS_REGION = "region"
SECTION_AWS_PROFILE = "profile"
SECTION_AWS_USE_SSL = "use_ssl"
SECTION_AWS_SSE = "sse"
SECTION_AWS_SCHEMA = {
SECTION_AWS_STORAGEPATH: str,
Optional(SECTION_AWS_REGION): str,
@@ -209,6 +222,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes
Optional(SECTION_AWS_ENDPOINT_URL): str,
Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
Optional(SECTION_AWS_SSE): str,
}
# backward compatibility
@@ -249,6 +263,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes
Optional(SECTION_AWS_ENDPOINT_URL): str,
Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
Optional(SECTION_AWS_SSE): str,
Optional(SECTION_GCP_PROJECTNAME): str,
Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
@@ -285,30 +300,20 @@ class Config(object): # pylint: disable=too-many-instance-attributes
}
def __init__(self, dvc_dir=None, validate=True):
self.system_config_file = os.path.join(
self.get_system_config_dir(), self.CONFIG
)
self.global_config_file = os.path.join(
self.get_global_config_dir(), self.CONFIG
)
self.dvc_dir = dvc_dir
self.validate = validate
if dvc_dir is not None:
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self.config_file = os.path.join(dvc_dir, self.CONFIG)
self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
if not dvc_dir:
try:
from dvc.repo import Repo
self.dvc_dir = os.path.join(Repo.find_dvc_dir())
except NotDvcRepoError:
self.dvc_dir = None
else:
self.dvc_dir = None
self.config_file = None
self.config_local_file = None
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self._system_config = None
self._global_config = None
self._repo_config = None
self._local_config = None
self.config = None
self.load(validate=validate)
self.load()
@staticmethod
def get_global_config_dir():
@@ -350,35 +355,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes
open(config_file, "w+").close()
return Config(dvc_dir)
def _load(self):
self._system_config = configobj.ConfigObj(self.system_config_file)
self._global_config = configobj.ConfigObj(self.global_config_file)
if self.config_file is not None:
self._repo_config = configobj.ConfigObj(self.config_file)
else:
self._repo_config = configobj.ConfigObj()
if self.config_local_file is not None:
self._local_config = configobj.ConfigObj(self.config_local_file)
else:
self._local_config = configobj.ConfigObj()
self.config = None
def _load_config(self, path):
config = configobj.ConfigObj(path)
config = self._lower(config)
self._resolve_paths(config, path)
return config
@staticmethod
def _resolve_path(path, config_file):
assert os.path.isabs(config_file)
config_dir = os.path.dirname(config_file)
return os.path.abspath(os.path.join(config_dir, path))
def _resolve_cache_path(self, config, fname):
def _resolve_cache_path(self, config):
cache = config.get(self.SECTION_CACHE)
if cache is None:
return
@@ -387,72 +364,86 @@ class Config(object): # pylint: disable=too-many-instance-attributes
if cache_dir is None:
return
cache[self.PRIVATE_CWD] = os.path.dirname(fname)
cache[self.PRIVATE_CWD] = os.path.dirname(config.filename)
def _resolve_paths(self, config, fname):
if fname is None:
return
def _resolve_paths(self, config):
if config.filename is None:
return config
self._resolve_cache_path(config, fname)
for section in config.values():
ret = copy.deepcopy(config)
self._resolve_cache_path(ret)
for section in ret.values():
if self.SECTION_REMOTE_URL not in section.keys():
continue
section[self.PRIVATE_CWD] = os.path.dirname(fname)
section[self.PRIVATE_CWD] = os.path.dirname(ret.filename)
def load(self, validate=True):
return ret
def _load_configs(self):
system_config_file = os.path.join(
self.get_system_config_dir(), self.CONFIG
)
global_config_file = os.path.join(
self.get_global_config_dir(), self.CONFIG
)
self._system_config = configobj.ConfigObj(system_config_file)
self._global_config = configobj.ConfigObj(global_config_file)
self._repo_config = configobj.ConfigObj()
self._local_config = configobj.ConfigObj()
if not self.dvc_dir:
return
config_file = os.path.join(self.dvc_dir, self.CONFIG)
config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
self._repo_config = configobj.ConfigObj(config_file)
self._local_config = configobj.ConfigObj(config_local_file)
@property
def config_local_file(self):
return self._local_config.filename
@property
def config_file(self):
return self._repo_config.filename
def load(self):
"""Loads config from all the config files.
Args:
validate (bool): optional flag to tell dvc if it should validate
the config or just load it as is. 'True' by default.
Raises:
dvc.config.ConfigError: thrown if config has invalid format.
"""
self._load()
self._load_configs()
self.config = configobj.ConfigObj()
for c in [
self._system_config,
self._global_config,
self._repo_config,
self._local_config,
]:
c = self._resolve_paths(c)
c = self._lower(c)
self.config.merge(c)
if not self.validate:
return
d = self.config.dict()
try:
self.config = self._load_config(self.system_config_file)
user = self._load_config(self.global_config_file)
config = self._load_config(self.config_file)
local = self._load_config(self.config_local_file)
# NOTE: schema doesn't support ConfigObj.Section validation, so we
# need to convert our config to dict before passing it to
for conf in [user, config, local]:
self.config = self._merge(self.config, conf)
if validate:
self.config = Schema(self.SCHEMA).validate(self.config)
# NOTE: now converting back to ConfigObj
self.config = configobj.ConfigObj(
self.config, write_empty_values=True
)
self.config.filename = self.config_file
self._resolve_paths(self.config, self.config_file)
except Exception as ex:
raise ConfigError(ex)
@staticmethod
def _get_key(conf, name, add=False):
for k in conf.keys():
if k.lower() == name.lower():
return k
if add:
conf[name] = {}
return name
return None
d = Schema(self.SCHEMA).validate(d)
except SchemaError as exc:
raise ConfigError("config format error", cause=exc)
self.config = configobj.ConfigObj(d, write_empty_values=True)
def save(self, config=None):
"""Saves config to config files.
Args:
config (configobj.ConfigObj): optional config object to save.
Raises:
dvc.config.ConfigError: thrown if failed to write config file.
"""
@@ -467,129 +458,103 @@ class Config(object): # pylint: disable=too-many-instance-attributes
]
for conf in clist:
if conf.filename is None:
continue
self._save(conf)
try:
logger.debug("Writing '{}'.".format(conf.filename))
dname = os.path.dirname(os.path.abspath(conf.filename))
try:
os.makedirs(dname)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
conf.write()
except Exception as exc:
msg = "failed to write config '{}'".format(conf.filename)
raise ConfigError(msg, exc)
def get_remote_settings(self, name):
import posixpath
"""
Args:
name (str): The name of the remote that we want to retrieve
Returns:
dict: The content beneath the given remote name.
Example:
>>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
>>> get_remote_settings("server")
{'url': 'ssh://localhost/'}
"""
settings = self.config.get(
self.SECTION_REMOTE_FMT.format(name.lower())
)
if settings is None:
raise ConfigError(
"unable to find remote section '{}'".format(name)
)
parsed = urlparse(settings["url"])
# Support for cross referenced remotes.
# This will merge the settings, giving priority to the outer reference.
# For example, having:
#
# dvc remote add server ssh://localhost
# dvc remote modify server user root
# dvc remote modify server ask_password true
#
# dvc remote add images remote://server/tmp/pictures
# dvc remote modify images user alice
# dvc remote modify images ask_password false
# dvc remote modify images password asdf1234
#
# Results on a config dictionary like:
#
# {
# "url": "ssh://localhost/tmp/pictures",
# "user": "alice",
# "password": "asdf1234",
# "ask_password": False,
# }
#
if parsed.scheme == "remote":
reference = self.get_remote_settings(parsed.netloc)
url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
merged = reference.copy()
merged.update(settings)
merged["url"] = url
return merged
return settings
self.load()
@staticmethod
def unset(config, section, opt=None):
def _save(config):
if config.filename is None:
return
logger.debug("Writing '{}'.".format(config.filename))
dname = os.path.dirname(os.path.abspath(config.filename))
try:
os.makedirs(dname)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
config.write()
def unset(self, section, opt=None, level=None, force=False):
"""Unsets specified option and/or section in the config.
Args:
config (configobj.ConfigObj): config to work on.
section (str): section name.
opt (str): optional option name.
level (int): config level to use.
force (bool): don't error-out even if section doesn't exist. False
by default.
Raises:
dvc.config.ConfigError: thrown if section doesn't exist and
`force != True`.
"""
config = self.get_configobj(level)
if section not in config.keys():
if force:
return
raise ConfigError("section '{}' doesn't exist".format(section))
if opt is None:
del config[section]
return
if opt:
if opt not in config[section].keys():
if force:
return
raise ConfigError(
"option '{}.{}' doesn't exist".format(section, opt)
)
del config[section][opt]
if opt not in config[section].keys():
raise ConfigError(
"option '{}.{}' doesn't exist".format(section, opt)
)
del config[section][opt]
if not config[section]:
if not config[section]:
del config[section]
else:
del config[section]
@staticmethod
def set(config, section, opt, value):
self.save(config)
def set(self, section, opt, value, level=None, force=True):
"""Sets specified option in the config.
Args:
config (configobj.ConfigObj): config to work on.
section (str): section name.
opt (str): option name.
value: value to set option to.
level (int): config level to use.
force (bool): set option even if section already exists. True by
default.
Raises:
dvc.config.ConfigError: thrown if section already exists and
`force != True`.
"""
config = self.get_configobj(level)
if section not in config.keys():
config[section] = {}
elif not force:
raise ConfigError(
"Section '{}' already exists. Use `-f|--force` to overwrite "
"section with new value.".format(section)
)
config[section][opt] = value
self.save(config)
@staticmethod
def show(config, section, opt):
"""Prints option value from the config.
def get(self, section, opt=None, level=None):
"""Return option value from the config.
Args:
config (configobj.ConfigObj): config to work on.
section (str): section name.
opt (str): option name.
level (int): config level to use.
Returns:
value (str, int): option value.
"""
config = self.get_configobj(level)
if section not in config.keys():
raise ConfigError("section '{}' doesn't exist".format(section))
@@ -598,25 +563,35 @@ class Config(object): # pylint: disable=too-many-instance-attributes
"option '{}.{}' doesn't exist".format(section, opt)
)
logger.info(config[section][opt])
@staticmethod
def _merge(first, second):
res = {}
sections = list(first.keys()) + list(second.keys())
for section in sections:
first_copy = first.get(section, {}).copy()
second_copy = second.get(section, {}).copy()
first_copy.update(second_copy)
res[section] = first_copy
return res
return config[section][opt]
@staticmethod
def _lower(config):
new_config = {}
new_config = configobj.ConfigObj()
for s_key, s_value in config.items():
new_s = {}
for key, value in s_value.items():
new_s[key.lower()] = str(value)
new_config[s_key.lower()] = new_s
return new_config
def get_configobj(self, level):
configs = {
self.LEVEL_LOCAL: self._local_config,
self.LEVEL_REPO: self._repo_config,
self.LEVEL_GLOBAL: self._global_config,
self.LEVEL_SYSTEM: self._system_config,
}
return configs.get(level, self._repo_config)
def list_options(self, section_regex, option, level=None):
ret = {}
config = self.get_configobj(level)
for section in config.keys():
r = re.match(section_regex, section)
if r:
name = r.group("name")
value = config[section].get(option, "")
ret[name] = value
return ret
dvc/data_cloud.py
+1 -3
View File
@@ -25,7 +25,6 @@ class DataCloud(object):
Args:
repo (dvc.repo.Repo): repo instance that belongs to the repo that
we are working on.
config (dict): config of the repo.
Raises:
config.ConfigError: thrown when config has invalid format.
@@ -75,8 +74,7 @@ class DataCloud(object):
)
def _init_remote(self, remote):
config = self.repo.config.get_remote_settings(remote)
return Remote(self.repo, config)
return Remote(self.repo, name=remote)
def _init_compat(self):
name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower()
dvc/dependency/__init__.py
+1 -2
View File
@@ -52,8 +52,7 @@ def _get(stage, p, info):
parsed = urlparse(p)
if parsed.scheme == "remote":
settings = stage.repo.config.get_remote_settings(parsed.netloc)
remote = Remote(stage.repo, settings)
remote = Remote(stage.repo, name=parsed.netloc)
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
for d in DEPS:
dvc/main.py
+4
View File
@@ -5,6 +5,7 @@ from __future__ import unicode_literals
import logging
from dvc.cli import parse_args
from dvc.config import ConfigError
from dvc.analytics import Analytics
from dvc.exceptions import NotDvcRepoError, DvcParserError
@@ -36,6 +37,9 @@ def main(argv=None):
cmd = args.func(args)
ret = cmd.run_cmd()
except ConfigError:
logger.exception("configuration error")
ret = 251
except KeyboardInterrupt:
logger.exception("interrupted by the user")
ret = 252
dvc/output/__init__.py
+1 -2
View File
@@ -61,8 +61,7 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
parsed = urlparse(p)
if parsed.scheme == "remote":
settings = stage.repo.config.get_remote_settings(parsed.netloc)
remote = Remote(stage.repo, settings)
remote = Remote(stage.repo, name=parsed.netloc)
return OUTS_MAP[remote.scheme](
stage,
p,
dvc/progress.py
+1 -1
View File
@@ -111,7 +111,7 @@ class Progress(object):
logger = logging.getLogger(__name__)
if logger.getEffectiveLevel() == "CRITICAL":
if logger.getEffectiveLevel() == logging.CRITICAL:
return
print(*args, **kwargs)
dvc/remote/__init__.py
+10 -2
View File
@@ -10,6 +10,8 @@ from dvc.remote.http import RemoteHTTP
from dvc.remote.https import RemoteHTTPS
from dvc.remote.oss import RemoteOSS
from .config import RemoteConfig
REMOTES = [
RemoteAZURE,
@@ -31,5 +33,11 @@ def _get(config):
return RemoteLOCAL
def Remote(repo, config):
return _get(config)(repo, config)
def Remote(repo, **kwargs):
name = kwargs.get("name")
if name:
remote_config = RemoteConfig(repo.config)
settings = remote_config.get_settings(name)
else:
settings = kwargs
return _get(settings)(repo, settings)
dvc/remote/config.py
+151
View File
@@ -0,0 +1,151 @@
import os
import logging
import posixpath
from dvc.utils.compat import urlparse
from dvc.config import Config, ConfigError
logger = logging.getLogger(__name__)
class RemoteConfig(object):
def __init__(self, config):
self.config = config
def get_settings(self, name):
"""
Args:
name (str): The name of the remote that we want to retrieve
Returns:
dict: The content beneath the given remote name.
Example:
>>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
>>> get_settings("server")
{'url': 'ssh://localhost/'}
"""
settings = self.config.config.get(
Config.SECTION_REMOTE_FMT.format(name.lower())
)
if settings is None:
raise ConfigError(
"unable to find remote section '{}'".format(name)
)
parsed = urlparse(settings["url"])
# Support for cross referenced remotes.
# This will merge the settings, giving priority to the outer reference.
# For example, having:
#
# dvc remote add server ssh://localhost
# dvc remote modify server user root
# dvc remote modify server ask_password true
#
# dvc remote add images remote://server/tmp/pictures
# dvc remote modify images user alice
# dvc remote modify images ask_password false
# dvc remote modify images password asdf1234
#
# Results on a config dictionary like:
#
# {
# "url": "ssh://localhost/tmp/pictures",
# "user": "alice",
# "password": "asdf1234",
# "ask_password": False,
# }
#
if parsed.scheme == "remote":
reference = self.get_settings(parsed.netloc)
url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
merged = reference.copy()
merged.update(settings)
merged["url"] = url
return merged
return settings
@staticmethod
def resolve_path(path, config_file):
"""Resolve path relative to config file location.
Args:
path: Path to be resolved.
config_file: Path to config file, which `path` is specified
relative to.
Returns:
Path relative to the `config_file` location. If `path` is an
absolute path then it will be returned without change.
"""
if os.path.isabs(path):
return path
return os.path.relpath(path, os.path.dirname(config_file))
def add(self, name, url, default=False, force=False, level=None):
from dvc.remote import _get, RemoteLOCAL
configobj = self.config.get_configobj(level)
remote = _get({Config.SECTION_REMOTE_URL: url})
if remote == RemoteLOCAL and not url.startswith("remote://"):
url = self.resolve_path(url, configobj.filename)
self.config.set(
Config.SECTION_REMOTE_FMT.format(name),
Config.SECTION_REMOTE_URL,
url,
force=force,
level=level,
)
if default:
logger.info("Setting '{}' as a default remote.".format(name))
self.config.set(
Config.SECTION_CORE,
Config.SECTION_CORE_REMOTE,
name,
level=level,
)
def remove(self, name, level=None):
self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level)
if level is None:
level = Config.LEVEL_REPO
for lev in [
Config.LEVEL_LOCAL,
Config.LEVEL_REPO,
Config.LEVEL_GLOBAL,
Config.LEVEL_SYSTEM,
]:
self.config.unset(
Config.SECTION_CORE,
Config.SECTION_CORE_REMOTE,
level=lev,
force=True,
)
if lev == level:
break
def modify(self, name, option, value, level=None):
self.config.set(
Config.SECTION_REMOTE_FMT.format(name), option, value, level=level
)
def list(self, level=None):
return self.config.list_options(
Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level
)
def set_default(self, name, unset=False, level=None):
if unset:
self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE)
return
self.config.set(
Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level
)
dvc/remote/local/__init__.py
+13 -7
View File
@@ -464,14 +464,20 @@ class RemoteLOCAL(RemoteBASE):
if len(chunks) == 0:
return 0
futures = []
with ThreadPoolExecutor(max_workers=jobs) as executor:
for from_infos, to_infos, names in chunks:
res = executor.submit(func, from_infos, to_infos, names=names)
futures.append(res)
if jobs > 1:
futures = []
with ThreadPoolExecutor(max_workers=jobs) as executor:
for from_infos, to_infos, names in chunks:
res = executor.submit(
func, from_infos, to_infos, names=names
)
futures.append(res)
for f in futures:
f.result()
for f in futures:
f.result()
else:
for from_infos, to_infos, names in chunks:
func(from_infos, to_infos, names=names)
return len(chunks)
dvc/remote/s3.py
+21 -11
View File
@@ -64,6 +64,12 @@ class RemoteS3(RemoteBASE):
self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
self.extra_args = {}
self.sse = config.get(Config.SECTION_AWS_SSE, "")
if self.sse:
self.extra_args["ServerSideEncryption"] = self.sse
shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
if shared_creds:
os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
@@ -111,17 +117,16 @@ class RemoteS3(RemoteBASE):
)
return obj
@classmethod
def _copy_multipart(cls, s3, from_info, to_info, size, n_parts):
def _copy_multipart(self, s3, from_info, to_info, size, n_parts):
mpu = s3.create_multipart_upload(
Bucket=to_info.bucket, Key=to_info.path
Bucket=to_info.bucket, Key=to_info.path, **self.extra_args
)
mpu_id = mpu["UploadId"]
parts = []
byte_position = 0
for i in range(1, n_parts + 1):
obj = cls.get_head_object(
obj = self.get_head_object(
s3, from_info.bucket, from_info.path, PartNumber=i
)
part_size = obj["ContentLength"]
@@ -153,8 +158,7 @@ class RemoteS3(RemoteBASE):
MultipartUpload={"Parts": parts},
)
@classmethod
def _copy(cls, s3, from_info, to_info):
def _copy(self, s3, from_info, to_info):
# NOTE: object's etag depends on the way it was uploaded to s3 or the
# way it was copied within the s3. More specifically, it depends on
# the chunk size that was used to transfer it, which would affect
@@ -175,19 +179,21 @@ class RemoteS3(RemoteBASE):
# preserve etag, we need to transfer each part separately, so the
# object is transfered in the same chunks as it was originally.
obj = cls.get_head_object(s3, from_info.bucket, from_info.path)
obj = self.get_head_object(s3, from_info.bucket, from_info.path)
etag = obj["ETag"].strip('"')
size = obj["ContentLength"]
_, _, parts_suffix = etag.partition("-")
if parts_suffix:
n_parts = int(parts_suffix)
cls._copy_multipart(s3, from_info, to_info, size, n_parts)
self._copy_multipart(s3, from_info, to_info, size, n_parts)
else:
source = {"Bucket": from_info.bucket, "Key": from_info.path}
s3.copy(source, to_info.bucket, to_info.path)
s3.copy(
source, to_info.bucket, to_info.path, ExtraArgs=self.extra_args
)
cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
cached_etag = self.get_etag(s3, to_info.bucket, to_info.path)
if etag != cached_etag:
raise ETagMismatchError(etag, cached_etag)
@@ -257,7 +263,11 @@ class RemoteS3(RemoteBASE):
try:
s3.upload_file(
from_info.path, to_info.bucket, to_info.path, Callback=cb
from_info.path,
to_info.bucket,
to_info.path,
Callback=cb,
ExtraArgs=self.extra_args,
)
except Exception:
msg = "failed to upload '{}'".format(from_info.path)
dvc/repo/__init__.py
+9 -9
View File
@@ -5,6 +5,7 @@ import logging
import dvc.prompt as prompt
from dvc.config import Config
from dvc.exceptions import (
DvcException,
NotDvcRepoError,
@@ -41,7 +42,6 @@ class Repo(object):
from dvc.repo.brancher import brancher
def __init__(self, root_dir=None):
from dvc.config import Config
from dvc.state import State
from dvc.lock import Lock
from dvc.scm import SCM
@@ -89,15 +89,15 @@ class Repo(object):
def __repr__(self):
return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
@staticmethod
def find_root(root=None):
@classmethod
def find_root(cls, root=None):
if root is None:
root = os.getcwd()
else:
root = os.path.abspath(os.path.realpath(root))
while True:
dvc_dir = os.path.join(root, Repo.DVC_DIR)
dvc_dir = os.path.join(root, cls.DVC_DIR)
if os.path.isdir(dvc_dir):
return root
if os.path.ismount(root):
@@ -105,10 +105,10 @@ class Repo(object):
root = os.path.dirname(root)
raise NotDvcRepoError(root)
@staticmethod
def find_dvc_dir(root=None):
root_dir = Repo.find_root(root)
return os.path.join(root_dir, Repo.DVC_DIR)
@classmethod
def find_dvc_dir(cls, root=None):
root_dir = cls.find_root(root)
return os.path.join(root_dir, cls.DVC_DIR)
@staticmethod
def init(root_dir=os.curdir, no_scm=False, force=False):
@@ -318,7 +318,7 @@ class Repo(object):
for stage in stages:
if active and not target and stage.locked:
logger.warning(
"DVC file '{path}' is locked. Its dependencies are"
"DVC-file '{path}' is locked. Its dependencies are"
" not going to be pushed/pulled/fetched.".format(
path=stage.relpath
)
dvc/repo/checkout.py
+1 -1
View File
@@ -47,7 +47,7 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False):
for stage in stages:
if stage.locked:
logger.warning(
"DVC file '{path}' is locked. Its dependencies are"
"DVC-file '{path}' is locked. Its dependencies are"
" not going to be checked out.".format(path=stage.relpath)
)
dvc/repo/reproduce.py
+1 -1
View File
@@ -15,7 +15,7 @@ def _reproduce_stage(stages, node, force, dry, interactive, no_commit):
if stage.locked:
logger.warning(
"DVC file '{path}' is locked. Its dependencies are"
"DVC-file '{path}' is locked. Its dependencies are"
" not going to be reproduced.".format(path=stage.relpath)
)
dvc/repo/status.py
+1 -1
View File
@@ -14,7 +14,7 @@ def _local_status(self, target=None, with_deps=False):
for stage in stages:
if stage.locked:
logger.warning(
"DVC file '{path}' is locked. Its dependencies are"
"DVC-file '{path}' is locked. Its dependencies are"
" not going to be shown in the status output.".format(
path=stage.relpath
)
dvc/scm/git/__init__.py
+3 -3
View File
@@ -242,7 +242,7 @@ class Git(Base):
logger.info(
"\n"
"To track the changes with git run:\n"
"To track the changes with git, run:\n"
"\n"
"\tgit add {files}".format(files=" ".join(self.files_to_track))
)
@@ -297,8 +297,8 @@ class Git(Base):
b_ref (str): optional second git reference, default None
Returns:
dict: dictionary with keys: (a_tree, b_tree, a_ref, b_ref,
equal)
dict: dictionary with keys: {a_ref, b_ref, equal}
or {a_ref, b_ref, a_tree, b_tree}
"""
diff_dct = {DIFF_EQUAL: False}
trees, commits = self._get_diff_trees(a_ref, b_ref)
dvc/stage.py
+4 -4
View File
@@ -54,7 +54,7 @@ class StageFileAlreadyExistsError(DvcException):
class StageFileIsNotDvcFileError(DvcException):
def __init__(self, fname):
msg = "'{}' is not a dvc file".format(fname)
msg = "'{}' is not a DVC-file".format(fname)
sname = fname + Stage.STAGE_FILE_SUFFIX
if Stage.is_stage_file(sname):
@@ -218,7 +218,7 @@ class Stage(object):
if self.is_callback:
logger.warning(
"Dvc file '{fname}' is a 'callback' stage "
"DVC-file '{fname}' is a 'callback' stage "
"(has a command and no dependencies) and thus always "
"considered as changed.".format(fname=self.relpath)
)
@@ -253,7 +253,7 @@ class Stage(object):
def _changed_md5(self):
if self.changed_md5():
logger.warning("Dvc file '{}' changed.".format(self.relpath))
logger.warning("DVC-file '{}' changed.".format(self.relpath))
return True
return False
@@ -583,7 +583,7 @@ class Stage(object):
# it raises the proper exceptions by priority:
# 1. when the file doesn't exists
# 2. filename is not a dvc filename
# 2. filename is not a DVC-file
# 3. path doesn't represent a regular file
Stage._check_file_exists(repo, fname)
Stage._check_dvc_filename(fname)
dvc/utils/fs.py
+3 -2
View File
@@ -21,8 +21,9 @@ def get_inode(path):
def get_mtime_and_size(path):
size = os.path.getsize(path)
mtime = os.path.getmtime(path)
stat = os.stat(path)
size = stat.st_size
mtime = stat.st_mtime
if os.path.isdir(path):
for root, dirs, files in dvc_walk(str(path)):
scripts/build_posix.sh
+2
View File
@@ -15,6 +15,7 @@ fi
BUILD_DIR=build
BIN_DIR=$BUILD_DIR/$INSTALL_DIR/bin
DESC='Data Version Control - datasets, models, and experiments versioning for ML or data science projects'
LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib
print_error()
@@ -46,6 +47,7 @@ fpm_build()
fpm -s dir \
-f \
-t $1 \
--description $DESC \
$FPM_FLAGS \
-n dvc \
-v $VERSION \
scripts/completion/dvc.bash
+19 -18
View File
@@ -12,30 +12,31 @@ _dvc_commands='init destroy add import checkout run pull push fetch \
status repro remove move unprotect gc config remote metrics \
install root lock unlock pipeline commit'
_dvc_global_options="-h --help -q --quiet -V --verbose"
_dvc_options="-h --help -v --version"
_dvc_init="--no-scm -f --force"
_dvc_destroy="-f --force"
_dvc_global_options="-h --help -q --quiet -V --verbose"
_dvc_add="-R --recursive -f --file --no-commit"
_dvc_run="--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache -M --metrics-no-cache -y --yes --overwrite-dvc-file --ignore-build-cache --remove-outs --no-commit -w --wdir"
_dvc_checkout="$(compgen -G '*.dvc')"
_dvc_config="-u --unset --local --system --global"
_dvc_destroy="-f --force"
_dvc_fetch="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive"
_dvc_gc="-a --all-branches -T --all-tags -c --cloud -r --remote -f --force -p --project"
_dvc_import="--resume"
_dvc_init="--no-scm -f --force"
_dvc_install=""
_dvc_lock="$(compgen -G '*.dvc')"
_dvc_metrics=""
_dvc_move="$(compgen -G '*')"
_dvc_pipeline=""
_dvc_pull="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -f --force -R --recursive"
_dvc_push="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive"
_dvc_fetch="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive"
_dvc_status="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud -q --quiet"
_dvc_repro="--dry -f --force -s --single-item -c --cwd -m --metrics -i --interactive -p --pipeline -P --all-pipelines --ignore-build-cache --no-commit -s --single-item"
_dvc_remove="--dry -o --outs -p --purge --force"
_dvc_gc="-a --all-branches -T --all-tags -c --cloud -r --remote -f --force -p --project"
_dvc_config="-u --unset --local --system --global"
_dvc_checkout="$(compgen -G '*.dvc')"
_dvc_move="$(compgen -G '*')"
_dvc_lock="$(compgen -G '*.dvc')"
_dvc_unlock="$(compgen -G '*.dvc')"
_dvc_import="--resume"
_dvc_remote=""
_dvc_metrics=""
_dvc_install=""
_dvc_remove="--dry -o --outs -p --purge --force"
_dvc_repro="--dry -f --force -s --single-item -c --cwd -m --metrics -i --interactive -p --pipeline -P --all-pipelines --ignore-build-cache --no-commit -s --single-item"
_dvc_root=""
_dvc_pipeline=""
_dvc_run="--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache -M --metrics-no-cache -y --yes --overwrite-dvc-file --ignore-build-cache --remove-outs --no-commit -w --wdir"
_dvc_status="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud -q --quiet"
_dvc_unlock="$(compgen -G '*.dvc')"
# Notes:
#
scripts/completion/dvc.zsh
+85 -78
View File
@@ -15,28 +15,28 @@ _dvc_commands() {
"add:Take data files or directories under DVC control."
"cache:Manage cache settings."
"checkout:Checkout data files from cache."
"commit:Save changed data to cache and update DVC files."
"commit:Save changed data to cache and update DVC-files."
"config:Get or set config settings."
"destroy:Remove DVC files, local DVC config and data cache."
"destroy:Remove DVC-files, local DVC config and data cache."
"diff:Show a diff of a DVC controlled data file or a directory."
"fetch:Fetch data files from a DVC remote storage."
"gc:Collect unused data from DVC cache or a remote storage."
"import:Download or copy files from URL and take under DVC control."
"init:Initialize DVC in the current directory."
"install:Install DVC git hooks into the repository."
"lock:Lock DVC file."
"lock:Lock DVC-file."
"metrics:Commands to add, manage, collect and display metrics."
"move:Rename or move a DVC controlled data file or a directory."
"pipeline:Manage pipelines."
"pull:Pull data files from a DVC remote storage."
"push:Push data files to a DVC remote storage."
"remote:Manage remote storage configuration."
"remove:Remove outputs of DVC file."
"repro:Check for changes and reproduce DVC file and dependencies."
"remove:Remove outputs of DVC-file."
"repro:Check for changes and reproduce DVC-file and dependencies."
"root:Relative path to project's directory."
"run:Generate a stage file from a command and execute the command."
"status:Show changed stages, compare local cache and a remote storage."
"unlock:Unlock DVC file."
"unlock:Unlock DVC-file."
"unprotect:Unprotect data file/directory."
"version:Show DVC version and system/environment informaion."
)
@@ -44,21 +44,21 @@ _dvc_commands() {
_describe 'dvc commands' _commands
}
_dvc_global_options=(
"(-)"{-h,--help}"[Show help message related to the command]"
"(-)"{-q,--quiet}"[Be quiet.]"
"(-)"{-V,--verbose}"[Be verbose.]"
_dvc_options=(
"(-)"{-h,--help}"[Show help message.]"
"(-)"{-V,--version}"[Show program's version]"
)
_dvc_options=(
"(-)"{-h,--help}"[Show this help message and exit]"
"(-)"{-v,--version}"[Show program's version]"
_dvc_global_options=(
"(-)"{-h,--help}"[Show help message related to the command.]"
"(-)"{-q,--quiet}"[Be quiet.]"
"(-)"{-v,--verbose}"[Be verbose.]"
)
_dvc_add=(
{-R,--recursive}"[Recursively add each file under the directory.]"
{-f,--file}"[Specify name of the stage file.]:File:_files"
"--no-commit[Don't put files/directories into cache.]"
{-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
"1:File:_files"
)
@@ -67,27 +67,28 @@ _dvc_cache=(
)
_dvc_checkout=(
{-f,--force}"[Do not prompt when removing working directory files.]"
{-d,--with-deps}"[Checkout all dependencies of the specified target.]"
{-f,--force}"[Do not prompt when removing working directory files.]"
{-R,--recursive}"[Checkout all subdirectories of the specified directory.]"
"1:Stages:_files -g '(*.dvc|Dvcfile)'"
)
_dvc_commit=(
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
{-d,--with-deps}"[Commit all dependencies of the specified target.]"
{-f,--force}"[Commit even if checksums for dependencies/outputs changed.]"
{-d,--with-deps}"[Commit all dependencies of the specified target.]"
{-R,--recursive}"[Commit cache for subdirectories of the specified directory.]"
)
_dvc_config=(
{-u,--unset}"[Unset option.]"
"--local[Unset option.]"
"--system[Use system config.]"
"--global[Use global config.]"
"--system[Use system config.]"
"--local[Use local config.]"
{-u,--unset}"[Unset option.]"
)
_dvc_destroy=(
{-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
{-f,--force}"[Force destruction.]"
)
_dvc_diff=(
@@ -95,35 +96,36 @@ _dvc_diff=(
)
_dvc_fetch=(
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
"--show-checksums[Show checksums instead of file names.]"
{-r,--remote}"[Remote repository to fetch from.]:Remote repository:"
{-a,--all-branches}"[Fetch cache for all branches.]"
{-T,--all-tags}"[Fetch cache for all tags.]"
{-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]"
{-R,--recursive}"[Fetch cache for subdirectories of specified directory.]"
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
"--show-checksums[Show checksums instead of file names]"
{-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
{-r,--remote}"[Remote repository to pull from]:Remote repository:"
{-a,--all-branches}"[Fetch cache for all branches]"
{-T,--all-tags}"[Fetch cache for all tags]"
{-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
{-R,--recursive}"[Fetch cache for subdirectories of the specified directory.]"
)
_dvc_gc=(
{-a,--all-branches}"[Collect garbage for all branches]"
{-T,--all-tags}"[Collect garbage for all tags]"
{-c,--cloud}"[Collect garbage in remote repository]"
{-r,--remote}"[Remote repository to collect garbage in]:Remote repository:"
{-f,--force}"[Force garbage collection - automatically agree to all prompts.]"
{-p,--projects}"[Keep data files required by these projects in addition to the current one. Useful if you share a single cache across repos.]:Repos:_files"
{-a,--all-branches}"[Keep data files for the tips of all git branches.]"
{-T,--all-tags}"[Keep data files for all git tags.]"
{-c,--cloud}"[Collect garbage in remote repository.]"
{-r,--remote}"[Remote storage to collect garbage in.]:Remote repository:"
{-f,--force}"[Force garbage collection - automatically agree to all prompts.]:Repos:_files"
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
{-p,--projects}"[Keep data files required by these projects in addition to the current one.]:Repos:_files"
)
_dvc_import=(
"--resume[Resume previously started download.]"
{-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
"1:URL:"
"2:Output:"
)
_dvc_init=(
"--no-scm[Initiate dvc in directory that is not tracket by any scm tool]"
{-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
"--no-scm[Initiate dvc in directory that is not tracked by any scm tool.]"
{-f,--force}"[Overwrite existing '.dvc' directory. This operation removes local cache.]"
)
_dvc_install=()
@@ -146,26 +148,26 @@ _dvc_pipeline=(
)
_dvc_pull=(
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
"--show-checksums[Show checksums instead of file names]"
{-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
{-r,--remote}"[Remote repository to pull from]:Remote repository:"
{-a,--all-branches}"[Fetch cache for all branches]"
{-T,--all-tags}"[Fetch cache for all tags]"
{-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
"--show-checksums[Show checksums instead of file names.]"
{-r,--remote}"[Remote repository to pull from.]:Remote repository:"
{-a,--all-branches}"[Fetch cache for all branches.]"
{-T,--all-tags}"[Fetch cache for all tags.]"
{-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]"
{-f,--force}"[Do not prompt when removing working directory files.]"
{-R,--recursive}"[Pull cache for subdirectories of the specified directory.]"
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
)
_dvc_push=(
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
"--show-checksums[Show checksums instead of file names.]"
{-r,--remote}"[Remote repository to push to.]:Remote repository:"
{-a,--all-branches}"[Push cache for all branches.]"
{-T,--all-tags}"[Push cache for all tags.]"
{-d,--with-deps}"[Push cache for all dependencies of the specified target.]"
{-R,--recursive}"[Push cache from subdirectories of specified directory.]"
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
"--show-checksums[Show checksums instead of file names]"
{-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
{-r,--remote}"[Remote repository to pull from]:Remote repository:"
{-a,--all-branches}"[Fetch cache for all branches]"
{-T,--all-tags}"[Fetch cache for all tags]"
{-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
{-R,--recursive}"[Push cache for subdirectories of the specified directory.]"
)
_dvc_remote=(
@@ -175,53 +177,58 @@ _dvc_remote=(
_dvc_remove=(
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
"--dry[Only print the commands that would be executed without actually executing]"
{-o,--outs}"[Only remove DVC file outputs.(default)]"
{-p,--purge}"[Remove DVC file and all its outputs]"
{-o,--outs}"[Only remove DVC-file outputs. (Default)]"
{-p,--purge}"[Remove DVC-file and all its outputs.]"
{-f,--force}"[Force purge.]"
)
_dvc_repro=(
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
"--dry[Only print the commands that would be executed without actually executing]"
"--ignore-build-cache[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]"
"--no-commit[Don't put files/directories into cache.]"
{-f,--force}"[Reproduce even if dependencies were not changed.]"
{-s,--single-item}"[Reproduce only single data item without recursive dependencies check.]"
{-c,--cwd}"[Directory within your repo to reproduce from.]:CWD:_files -/"
{-m,--metrics}"[Show metrics after reproduction.]"
"--dry[Only print the commands that would be executed without actually executing]"
{-i,--interactive}"[Ask for confirmation before reproducing each stage.]"
{-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]"
{-P,--all-pipelines}"[Reproduce all pipelines in the project.]"
{-d,--downstream}"[Reproduce the pipeline starting from the specified stage.]"
{-P,--all-pipelines}"[Reproduce all pipelines in the repo.]"
{-R,--recursive}"[Reproduce all stages in the specified directory.]"
"--ignore-build-cache[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]"
"--no-commit[Don't put files/directories into cache.]"
"--downstream[Reproduce the pipeline starting from the specified stage.]"
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
)
_dvc_root=()
_dvc_run=(
"--no-exec[Only create stage file without actually running it.]"
"--overwrite-dvcfile[Overwrite existing dvc file without asking for confirmation.]"
"--ignore-build-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
"--no-commit[Don't put files/directories into cache.]"
{-f,--file}"[Specify name of the stage file.]:File:_files"
{-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/"
{-w,--wdir}"[Directory to run your command and place state file in.]:WDIR:_files -/"
"*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files"
"*"{-o,--outs}"[Declare output data file or data directory.]:Output data:_files"
"*"{-O,--outs-no-cache}"[Declare output regular file or directory.]:Output regular:_files"
"*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache)]:Metrics (no cache):_files"
"*"{-m,--metrics}"[Declare output metric file or directory]:Metrics:_files"
{-y,--yes}"[Automatic 'yes' answer to all prompts.]"
"*"{-o,--outs}"[Declare output file or directory.]:Output data:_files"
"*"{-O,--outs-no-cache}"[Declare output file or directory (do not put into DVC cache).]:Output regular:_files"
"*"{-m,--metrics}"[Declare output metric file or directory.]:Metrics:_files"
"*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache).]:Metrics (no cache):_files"
{-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
{-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/"
{-w,--wdir}"[Directory within your repo to run your command in.]:WDIR:_files -/"
"--no-exec[Only create stage file without actually running it.]"
{-y,--yes}"[Deprecated, use --overwrite-dvcfile instead]"
"--overwrite-dvcfile[Overwrite existing DVC-file without asking for confirmation.]"
"--ignore-build-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
"--remove-outs[Deprecated, this is now the default behavior]"
"--no-commit[Don't put files/directories into cache.]"
"--outs-persist[Declare output file or directory that will not be removed upon repro.]:Output persistent:_files"
"--outs-persist-no-cache[Declare output file or directory that will not be removed upon repro (do not put into DVC cache).]:Output persistent regular:_files"
)
_dvc_status=(
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
"--show-checksums[Show checksums instead of file names.]"
{-q,--quiet}"[Suppresses all output. Exit with 0 if pipeline is up to date, otherwise 1.]"
{-c,--cloud}"[Show status of a local cache compared to a remote repository.]"
{-r,--remote}"[Remote repository to compare local cache to.]:Remote repository:"
{-a,--all-branches}"[Show status of a local cache compared to a remote repository for all branches.]"
{-T,--all-tags}"[Show status of a local cache compared to a remote repository for all tags.]"
{-d,--with-deps}"[Show status for all dependencies of the specified target.]"
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
"--show-checksums[Show checksums instead of file names]"
{-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
{-r,--remote}"[Remote repository to pull from]:Remote repository:"
{-a,--all-branches}"[Fetch cache for all branches]"
{-T,--all-tags}"[Fetch cache for all tags]"
{-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
{-c,--cloud}"[Show status of a local cache compared to a remote repository]"
{-q,--quiet}"[Suppresses all output. Exit with 0 if pipeline is up]"
)
_dvc_unlock=(
setup.py
+2 -1
View File
@@ -60,6 +60,7 @@ install_requires = [
"humanize>=0.5.1",
"dulwich>=0.19.11",
"ruamel.yaml>=0.15.91",
"psutil==5.6.2",
]
# Extra dependencies for remote integrations
@@ -76,7 +77,7 @@ tests_requirements = [
"wheel>=0.31.1",
"pydot>=1.2.4",
# Test requirements:
"pytest>=4.4.0",
"pytest>=4.6.0",
"pytest-timeout>=1.3.3",
"pytest-cov>=2.6.1",
"pytest-xdist>=1.26.1",
tests/__main__.py
+1 -1
View File
@@ -15,6 +15,6 @@ params = " ".join(sys.argv[1:])
cmd = (
"py.test -v -n=4 --timeout=600 --timeout_method=thread --log-level=debug"
" --cov=dvc {params} ".format(params=params)
" --cov=dvc {params} --durations=0".format(params=params)
)
check_call(cmd, shell=True)
tests/basic_env.py
+6 -3
View File
@@ -57,6 +57,10 @@ class TestDirFixture(object):
root_dir = self.mkdtemp()
self._root_dir = os.path.realpath(root_dir)
@property
def root_dir(self):
return self._root_dir
def _pushd(self, d):
if not hasattr(self, "_saved_dir"):
self._saved_dir = os.path.realpath(os.curdir)
@@ -193,9 +197,8 @@ class TestDvcDataFileFixture(TestDvcGitInitializedFixture):
shutil.copytree(self.dvc.cache.local.cache_dir, self.cache_dir)
class MockConfig:
system = None
glob = None
local = None
level = 1
force = False
default = True
name = self.REMOTE
url = self.cache_dir
tests/conftest.py
+3 -1
View File
@@ -68,12 +68,14 @@ def git(repo_dir):
@pytest.fixture
def dvc_repo(repo_dir, git):
dvc = None
try:
dvc = DvcRepo.init(repo_dir._root_dir)
dvc.scm.commit("init dvc")
yield dvc
finally:
dvc.scm.git.close()
if dvc:
dvc.scm.git.close()
here = os.path.abspath(os.path.dirname(__file__))
tests/func/test_add.py
+1 -1
View File
@@ -114,7 +114,7 @@ class TestAddCmdDirectoryRecursive(TestDvc):
with self._caplog.at_level(logging.WARNING, logger="dvc"):
assert main(["add", "--recursive", "large-dir"]) == 0
assert warning in self._caplog.text
assert warning in self._caplog.messages
class TestAddDirectoryWithForwardSlash(TestDvc):
tests/func/test_config.py
+5 -24
View File
@@ -1,7 +1,6 @@
import configobj
from dvc.main import main
from dvc.command.config import CmdConfig
from tests.basic_env import TestDvc
@@ -67,38 +66,20 @@ class TestConfigCLI(TestDvc):
self._do_test(True)
def test_non_existing(self):
# FIXME check stdout/err
ret = main(["config", "non_existing_section.field"])
self.assertEqual(ret, 1)
self.assertEqual(ret, 251)
ret = main(["config", "global.non_existing_field"])
self.assertEqual(ret, 1)
self.assertEqual(ret, 251)
ret = main(["config", "non_existing_section.field", "-u"])
self.assertEqual(ret, 1)
self.assertEqual(ret, 251)
ret = main(["config", "global.non_existing_field", "-u"])
self.assertEqual(ret, 1)
self.assertEqual(ret, 251)
ret = main(["config", "core.remote", "myremote"])
self.assertEqual(ret, 0)
ret = main(["config", "core.non_existing_field", "-u"])
self.assertEqual(ret, 1)
def test_failed_write(self):
class A(object):
system = False
glob = False
local = False
name = "core.remote"
value = "myremote"
unset = False
args = A()
cmd = CmdConfig(args)
cmd.configobj.write = None
ret = cmd.run()
self.assertNotEqual(ret, 0)
self.assertEqual(ret, 251)
tests/func/test_data_cloud.py
+4 -3
View File
@@ -39,6 +39,7 @@ from tests.utils import spy
TEST_REMOTE = "upstream"
TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE)
TEST_CONFIG = {
Config.SECTION_CACHE: {},
Config.SECTION_CORE: {Config.SECTION_CORE_REMOTE: TEST_REMOTE},
TEST_SECTION: {Config.SECTION_REMOTE_URL: ""},
}
@@ -870,12 +871,12 @@ class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc):
checksum_bar = file_md5(self.BAR)[0]
self.message_header = (
"Some of the cache files do not exist neither locally "
"nor on remote. Missing cache files: \n"
"nor on remote. Missing cache files: "
)
self.message_bar_part = "\nname: {}, md5: {}\n".format(
self.message_bar_part = "name: {}, md5: {}".format(
self.BAR, checksum_bar
)
self.message_foo_part = "\nname: {}, md5: {}\n".format(
self.message_foo_part = "name: {}, md5: {}".format(
self.FOO, checksum_foo
)
tests/func/test_remote.py
+1 -18
View File
@@ -5,7 +5,6 @@ from dvc.remote.base import RemoteBASE
from mock import patch
from dvc.main import main
from dvc.command.remote import CmdRemoteAdd
from dvc.config import Config
from tests.basic_env import TestDvc
@@ -33,22 +32,6 @@ class TestRemote(TestDvc):
self.assertEqual(main(["remote", "list"]), 0)
def test_failed_write(self):
class A(object):
system = False
glob = False
local = False
name = "myremote"
url = "s3://remote"
unset = False
args = A()
cmd = CmdRemoteAdd(args)
cmd.configobj.write = None
ret = cmd.run()
self.assertNotEqual(ret, 0)
def test_relative_path(self):
dname = os.path.join("..", "path", "to", "dir")
ret = main(["remote", "add", "mylocal", dname])
@@ -64,7 +47,7 @@ class TestRemote(TestDvc):
remote_name = "a"
remote_url = "s3://bucket/name"
self.assertEqual(main(["remote", "add", remote_name, remote_url]), 0)
self.assertEqual(main(["remote", "add", remote_name, remote_url]), 1)
self.assertEqual(main(["remote", "add", remote_name, remote_url]), 251)
self.assertEqual(
main(["remote", "add", "-f", remote_name, remote_url]), 0
)
tests/func/test_version.py
+37
View File
@@ -0,0 +1,37 @@
import re
from dvc.main import main
def test_info_in_repo(dvc_repo, caplog):
assert main(["version"]) == 0
assert re.search(re.compile(r"DVC version: \d+\.\d+\.\d+"), caplog.text)
assert re.search(re.compile(r"Python version: \d\.\d\.\d"), caplog.text)
assert re.search(re.compile(r"Platform: .*"), caplog.text)
assert re.search(
re.compile(r"Filesystem type \(cache directory\): .*"), caplog.text
)
assert re.search(
re.compile(r"Filesystem type \(workspace\): .*"), caplog.text
)
assert re.search(
re.compile(r"(Cache: (.*link - (True|False)(,\s)?){3})"), caplog.text
)
def test_info_outside_of_repo(repo_dir, caplog):
assert main(["version"]) == 0
assert re.search(re.compile(r"DVC version: \d+\.\d+\.\d+"), caplog.text)
assert re.search(re.compile(r"Python version: \d\.\d\.\d"), caplog.text)
assert re.search(re.compile(r"Platform: .*"), caplog.text)
assert re.search(
re.compile(r"Filesystem type \(workspace\): .*"), caplog.text
)
assert not re.search(
re.compile(r"Filesystem type \(cache directory\): .*"), caplog.text
)
assert not re.search(
re.compile(r"(Cache: (.*link - (True|False)(,\s)?){3})"), caplog.text
)
tests/unit/command/test_version.py
-25
View File
@@ -1,25 +0,0 @@
import re
from dvc.command.version import CmdVersion
from dvc.cli import parse_args
def test_run_in_repo(repo_dir, dvc_repo):
cmd = CmdVersion(parse_args(["version"]))
ret = cmd.run_cmd()
assert ret == 0
def test_run_outside_of_repo(repo_dir):
cmd = CmdVersion(parse_args(["version"]))
ret = cmd.run_cmd()
assert ret == 0
def test_info(caplog):
cmd = CmdVersion(parse_args(["version"]))
cmd.run()
assert re.search(re.compile(r"DVC version: \d+\.\d+\.\d+"), caplog.text)
assert re.search(re.compile(r"Python version: \d\.\d\.\d"), caplog.text)
assert re.search(re.compile(r"Platform: .*"), caplog.text)
tests/unit/test_progress.py
+10 -5
View File
@@ -1,14 +1,19 @@
import logging
import mock
from unittest import TestCase
from dvc.progress import ProgressCallback
from dvc.progress import progress, ProgressCallback
class TestProgressCallback(TestCase):
def test_quiet(caplog, capsys):
with caplog.at_level(logging.CRITICAL, logger="dvc"):
progress._print("something")
assert capsys.readouterr().out == ""
class TestProgressCallback:
@mock.patch("dvc.progress.progress")
def test_should_init_reset_progress(self, progress_mock):
total_files_num = 1
ProgressCallback(total_files_num)
self.assertEqual([mock.call.reset()], progress_mock.method_calls)
assert [mock.call.reset()] == progress_mock.method_calls