Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into relink

Browse Source
This commit is contained in:
Ruslan Kuprieiev
2019-12-13 20:35:02 +02:00
committed by GitHub
parent 5d1f01d58f 75ecc9960d
commit da49badbfe
46 changed files with 1314 additions and 695 deletions
Download Patch File Download Diff File Expand all files Collapse all files
debian/changelog
Vendored
+5
View File
@@ -0,0 +1,5 @@
dvc (0.75.b1) experimental; urgency=low
* First native Debian package version.
-- Dmitry Petrov <[email protected]> Sat, 07 Dec 2019 09:49:28 +0300
debian/compat
Vendored
+1
View File
@@ -0,0 +1 @@
10
debian/control
Vendored
+33
View File
@@ -0,0 +1,33 @@
Source: dvc
Maintainer: Dmitry Petrov <[email protected]>
Section: python
Priority: optional
Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
Standards-Version: 4.0.0
Homepage: http://dvc.org
Vcs-Git: https://github.com/iterative/dvc.git
Vcs-browser: https://github.com/iterative/dvc
Package: dvc
Architecture: all
Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
Recommends: python3-arrow (=0.14.0),
python3-paramiko (>=2.5.0),
python3-gssapi (>= 2.5.0),
python3-boto3 (>=1.9.86),
python3-pydrive (=1.3.1),
python3-azure-storage-blob (=12.1.0),
python3-oss2 (=2.6.1),
python3-google-cloud-storage (=1.19.0)
Description: Git for data scientists - manage your code and data together
Data Version Control or DVC is an open-source tool for data science and machine learning projects.
.
Key features:
.
1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
.
2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
.
3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
.
4. Helps manage experiments with Git tags/branches and metrics tracking.
debian/copyright
Vendored
+11
View File
@@ -0,0 +1,11 @@
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: DVC
Upstream-Contact: Dmitry Petrov <[email protected]>
Source: https://github.com/iterative/dvc
Files: *
Copyright: Dmitry Petrov <[email protected]>
License: Apache 2.0
On Debian systems, the full text of the Apache 2.0
License can be found in the file
`/usr/share/common-licenses/Apache-2.0`.
debian/rules
Vendored Executable
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/make -f
export PYBUILD_NAME=dvc
%:
dh $@ --with python3 --buildsystem=pybuild
debian/source/format
Vendored
+1
View File
@@ -0,0 +1 @@
3.0 (native)
debian/watch
Vendored
+4
View File
@@ -0,0 +1,4 @@
# please also check http://pypi.debian.net/dvc/watch
version=3
opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))
dvc/analytics.py
+137 -226
View File
@@ -1,256 +1,167 @@
"""Collect and send usage analytics"""
from __future__ import unicode_literals
import errno
import json
import logging
import os
import platform
import requests
import sys
import tempfile
import uuid
import distro
from dvc import __version__
from dvc.utils import env2bool
from dvc.utils.compat import str
from dvc.config import Config, to_bool
from dvc.daemon import daemon
from dvc.exceptions import NotDvcRepoError
from dvc.lock import Lock, LockError
from dvc.repo import Repo
from dvc.scm import SCM
from dvc.utils import env2bool, is_binary, makedirs
from dvc.utils.compat import str, FileNotFoundError
logger = logging.getLogger(__name__)
class Analytics(object):
"""Class for collecting and sending usage analytics.
Args:
info (dict): optional existing analytics report.
def collect_and_send_report(args=None, return_code=None):
"""
Collect information from the runtime/environment and the command
being executed into a report and send it over the network.
URL = "https://analytics.dvc.org"
TIMEOUT_POST = 5
To prevent analytics from blocking the execution of the main thread,
sending the report is done in a separate process.
USER_ID_FILE = "user_id"
The inter-process communication happens through a file containing the
report as a JSON, where the _collector_ generates it and the _sender_
removes it after sending it.
"""
report = _runtime_info()
PARAM_DVC_VERSION = "dvc_version"
PARAM_USER_ID = "user_id"
PARAM_SYSTEM_INFO = "system_info"
# Include command execution information on the report only when available.
if args and hasattr(args, "func"):
report.update({"cmd_class": args.func.__name__})
PARAM_OS = "os"
if return_code is not None:
report.update({"cmd_return_code": return_code})
PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major"
PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor"
PARAM_WINDOWS_VERSION_BUILD = "windows_version_build"
PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack"
with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
json.dump(report, fobj)
daemon(["analytics", fobj.name])
PARAM_MAC_VERSION = "mac_version"
PARAM_LINUX_DISTRO = "linux_distro"
PARAM_LINUX_DISTRO_VERSION = "linux_distro_version"
PARAM_LINUX_DISTRO_LIKE = "linux_distro_like"
def is_enabled():
if env2bool("DVC_TEST"):
return False
PARAM_SCM_CLASS = "scm_class"
PARAM_IS_BINARY = "is_binary"
PARAM_CMD_CLASS = "cmd_class"
PARAM_CMD_RETURN_CODE = "cmd_return_code"
enabled = to_bool(
Config(validate=False)
.config.get(Config.SECTION_CORE, {})
.get(Config.SECTION_CORE_ANALYTICS, "true")
)
def __init__(self, info=None):
from dvc.config import Config
from dvc.lock import Lock
logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
if info is None:
info = {}
return enabled
self.info = info
cdir = Config.get_global_config_dir()
try:
os.makedirs(cdir)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
def send(report):
"""
Side effect: Removes the report after sending it.
self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
self.user_id_file_lock = Lock(self.user_id_file + ".lock")
The report is generated and stored in a temporary file, see:
`collect_and_send_report`. Sending happens on another process,
thus, the need of removing such file afterwards.
"""
url = "https://analytics.dvc.org"
headers = {"content-type": "application/json"}
@staticmethod
def load(path):
"""Loads analytics report from json file specified by path.
with open(report, "rb") as fobj:
requests.post(url, data=fobj, headers=headers, timeout=5)
Args:
path (str): path to json file with analytics report.
"""
with open(path, "r") as fobj:
analytics = Analytics(info=json.load(fobj))
os.unlink(path)
return analytics
os.remove(report)
def _write_user_id(self):
import uuid
with open(self.user_id_file, "w+") as fobj:
user_id = str(uuid.uuid4())
info = {self.PARAM_USER_ID: user_id}
json.dump(info, fobj)
def _scm_in_use():
try:
scm = SCM(root_dir=Repo.find_root())
return type(scm).__name__
except NotDvcRepoError:
pass
def _runtime_info():
"""
Gather information from the environment where DVC runs to fill a report.
"""
return {
"dvc_version": __version__,
"is_binary": is_binary(),
"scm_class": _scm_in_use(),
"system_info": _system_info(),
"user_id": _find_or_create_user_id(),
}
def _system_info():
system = platform.system()
if system == "Windows":
version = sys.getwindowsversion()
return {
"os": "windows",
"windows_version_build": version.build,
"windows_version_major": version.major,
"windows_version_minor": version.minor,
"windows_version_service_pack": version.service_pack,
}
if system == "Darwin":
return {"os": "mac", "mac_version": platform.mac_ver()[0]}
if system == "Linux":
return {
"os": "linux",
"linux_distro": distro.id(),
"linux_distro_like": distro.like(),
"linux_distro_version": distro.version(),
}
# We don't collect data for any other system.
raise NotImplementedError
def _find_or_create_user_id():
"""
The user's ID is stored on a file under the global config directory.
The file should contain a JSON with a "user_id" key:
{"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"}
IDs are generated randomly with UUID.
"""
config_dir = Config.get_global_config_dir()
fname = os.path.join(config_dir, "user_id")
lockfile = os.path.join(config_dir, "user_id.lock")
# Since the `fname` and `lockfile` are under the global config,
# we need to make sure such directory exist already.
makedirs(config_dir, exist_ok=True)
try:
with Lock(lockfile):
try:
with open(fname, "r") as fobj:
user_id = json.load(fobj)["user_id"]
except (FileNotFoundError, ValueError, KeyError):
user_id = str(uuid.uuid4())
with open(fname, "w") as fobj:
json.dump({"user_id": user_id}, fobj)
return user_id
def _read_user_id(self):
if not os.path.exists(self.user_id_file):
return None
with open(self.user_id_file, "r") as fobj:
try:
info = json.load(fobj)
except ValueError as exc:
logger.debug("Failed to load user_id: {}".format(exc))
return None
return info[self.PARAM_USER_ID]
def _get_user_id(self):
from dvc.lock import LockError
try:
with self.user_id_file_lock:
user_id = self._read_user_id()
if user_id is None:
user_id = self._write_user_id()
return user_id
except LockError:
msg = "Failed to acquire '{}'"
logger.debug(msg.format(self.user_id_file_lock.lockfile))
def _collect_windows(self):
import sys
version = sys.getwindowsversion() # pylint: disable=no-member
info = {}
info[self.PARAM_OS] = "windows"
info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack
return info
def _collect_darwin(self):
import platform
info = {}
info[self.PARAM_OS] = "mac"
info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
return info
def _collect_linux(self):
import distro
info = {}
info[self.PARAM_OS] = "linux"
info[self.PARAM_LINUX_DISTRO] = distro.id()
info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
return info
def _collect_system_info(self):
import platform
system = platform.system()
if system == "Windows":
return self._collect_windows()
if system == "Darwin":
return self._collect_darwin()
if system == "Linux":
return self._collect_linux()
raise NotImplementedError
def collect(self):
"""Collect analytics report."""
from dvc.scm import SCM
from dvc.utils import is_binary
from dvc.repo import Repo
from dvc.exceptions import NotDvcRepoError
self.info[self.PARAM_DVC_VERSION] = __version__
self.info[self.PARAM_IS_BINARY] = is_binary()
self.info[self.PARAM_USER_ID] = self._get_user_id()
self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
try:
scm = SCM(root_dir=Repo.find_root())
self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
except NotDvcRepoError:
pass
def collect_cmd(self, args, ret):
"""Collect analytics info from a CLI command."""
from dvc.command.daemon import CmdDaemonAnalytics
assert isinstance(ret, int) or ret is None
if ret is not None:
self.info[self.PARAM_CMD_RETURN_CODE] = ret
if args is not None and hasattr(args, "func"):
assert args.func != CmdDaemonAnalytics
self.info[self.PARAM_CMD_CLASS] = args.func.__name__
def dump(self):
"""Save analytics report to a temporary file.
Returns:
str: path to the temporary file that contains the analytics report.
"""
import tempfile
with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
json.dump(self.info, fobj)
return fobj.name
@staticmethod
def is_enabled(cmd=None):
from dvc.config import Config, to_bool
from dvc.command.daemon import CmdDaemonBase
if env2bool("DVC_TEST"):
return False
if isinstance(cmd, CmdDaemonBase):
return False
core = Config(validate=False).config.get(Config.SECTION_CORE, {})
enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true"))
logger.debug(
"Analytics is {}.".format("enabled" if enabled else "disabled")
)
return enabled
@staticmethod
def send_cmd(cmd, args, ret):
"""Collect and send analytics for CLI command.
Args:
args (list): parsed args for the CLI command.
ret (int): return value of the CLI command.
"""
from dvc.daemon import daemon
if not Analytics.is_enabled(cmd):
return
analytics = Analytics()
analytics.collect_cmd(args, ret)
daemon(["analytics", analytics.dump()])
def send(self):
"""Collect and send analytics."""
import requests
if not self.is_enabled():
return
self.collect()
logger.debug("Sending analytics: {}".format(self.info))
try:
requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
except requests.exceptions.RequestException as exc:
logger.debug("Failed to send analytics: {}".format(str(exc)))
except LockError:
logger.debug("Failed to acquire {lockfile}".format(lockfile=lockfile))
dvc/command/base.py
+6 -9
View File
@@ -2,8 +2,6 @@ from __future__ import unicode_literals
import logging
import colorama
logger = logging.getLogger(__name__)
@@ -24,15 +22,13 @@ def fix_subparsers(subparsers):
def append_doc_link(help_message, path):
from dvc.utils import format_link
if not path:
return help_message
doc_base = "https://man.dvc.org/"
return "{message}\nDocumentation: <{blue}{base}{path}{nc}>".format(
message=help_message,
base=doc_base,
path=path,
blue=colorama.Fore.CYAN,
nc=colorama.Fore.RESET,
return "{message}\nDocumentation: {link}".format(
message=help_message, link=format_link(doc_base + path)
)
@@ -44,7 +40,8 @@ class CmdBase(object):
self.repo = Repo()
self.config = self.repo.config
self.args = args
updater = Updater(self.repo.dvc_dir)
hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
updater.check()
@property
dvc/command/daemon.py
+8 -4
View File
@@ -12,11 +12,16 @@ class CmdDaemonUpdater(CmdDaemonBase):
def run(self):
import os
from dvc.repo import Repo
from dvc.config import Config
from dvc.updater import Updater
root_dir = Repo.find_root()
dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
updater = Updater(dvc_dir)
config = Config(dvc_dir, validate=False)
hardlink_lock = config.config.get("core", {}).get(
"hardlink_lock", False
)
updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
updater.fetch(detach=False)
return 0
@@ -24,10 +29,9 @@ class CmdDaemonUpdater(CmdDaemonBase):
class CmdDaemonAnalytics(CmdDaemonBase):
def run(self):
from dvc.analytics import Analytics
from dvc import analytics
analytics = Analytics.load(self.args.target)
analytics.send()
analytics.send(self.args.target)
return 0
dvc/command/get.py
+8 -3
View File
@@ -33,7 +33,7 @@ class CmdGet(CmdBaseNoRepo):
def add_parser(subparsers, parent_parser):
GET_HELP = "Download data from DVC repository."
GET_HELP = "Download/copy files or directories from DVC repository."
get_parser = subparsers.add_parser(
"get",
parents=[parent_parser],
@@ -44,9 +44,14 @@ def add_parser(subparsers, parent_parser):
get_parser.add_argument(
"url", help="URL of Git repository with DVC project to download from."
)
get_parser.add_argument("path", help="Path to data within DVC repository.")
get_parser.add_argument(
"-o", "--out", nargs="?", help="Destination path to put data to."
"path", help="Path to a file or directory within a DVC repository."
)
get_parser.add_argument(
"-o",
"--out",
nargs="?",
help="Destination path to copy/download files to.",
)
get_parser.add_argument(
"--rev", nargs="?", help="DVC repository git revision."
dvc/command/imp_url.py
+5
View File
@@ -44,12 +44,17 @@ def add_parser(subparsers, parent_parser):
"url",
help="Supported urls:\n"
"/path/to/file\n"
"/path/to/directory\n"
"C:\\\\path\\to\\file\n"
"C:\\\\path\\to\\directory\n"
"https://example.com/path/to/file\n"
"s3://bucket/path/to/file\n"
"s3://bucket/path/to/directory\n"
"gs://bucket/path/to/file\n"
"gs://bucket/path/to/directory\n"
"hdfs://example.com/path/to/file\n"
"ssh://example.com:/path/to/file\n"
"ssh://example.com:/path/to/directory\n"
"remote://myremote/path/to/file (see `dvc remote`)",
)
import_parser.add_argument(
dvc/command/remote.py
+4
View File
@@ -19,6 +19,10 @@ class CmdRemoteConfig(CmdConfig):
class CmdRemoteAdd(CmdRemoteConfig):
def run(self):
if self.args.default:
logger.info(
"Setting '{}' as a default remote.".format(self.args.name)
)
self.remote_config.add(
self.args.name,
self.args.url,
dvc/config.py
+11 -2
View File
@@ -13,8 +13,7 @@ from voluptuous import All, Any, Lower, Range, Coerce, Match
from dvc.exceptions import DvcException
from dvc.exceptions import NotDvcRepoError
from dvc.utils.compat import open
from dvc.utils.compat import str
from dvc.utils.compat import open, str
logger = logging.getLogger(__name__)
@@ -126,6 +125,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes
SECTION_CORE_INTERACTIVE = "interactive"
SECTION_CORE_ANALYTICS = "analytics"
SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
SECTION_CORE_HARDLINK_LOCK = "hardlink_lock"
SECTION_CACHE = "cache"
SECTION_CACHE_DIR = "dir"
@@ -161,6 +161,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes
Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool,
}
# aws specific options
@@ -172,6 +173,10 @@ class Config(object): # pylint: disable=too-many-instance-attributes
SECTION_AWS_USE_SSL = "use_ssl"
SECTION_AWS_SSE = "sse"
SECTION_AWS_ACL = "acl"
SECTION_AWS_GRANT_READ = "grant_read"
SECTION_AWS_GRANT_READ_ACP = "grant_read_acp"
SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp"
SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control"
# gcp specific options
SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH
@@ -211,6 +216,10 @@ class Config(object): # pylint: disable=too-many-instance-attributes
Optional(SECTION_AWS_USE_SSL, default=True): Bool,
SECTION_AWS_SSE: str,
SECTION_AWS_ACL: str,
SECTION_AWS_GRANT_READ: str,
SECTION_AWS_GRANT_READ_ACP: str,
SECTION_AWS_GRANT_WRITE_ACP: str,
SECTION_AWS_GRANT_FULL_CONTROL: str,
SECTION_GCP_PROJECTNAME: str,
SECTION_CACHE_TYPE: supported_cache_type,
Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
dvc/exceptions.py
+2 -9
View File
@@ -47,7 +47,8 @@ class OutputDuplicationError(DvcException):
)
msg = (
"file/directory '{}' is specified as an output in more than one "
"stage: {}"
"stage: {}\n"
"This is not allowed. Consider using a different output name."
).format(output, "\n ".join(stages))
super(OutputDuplicationError, self).__init__(msg)
@@ -283,14 +284,6 @@ class UrlNotDvcRepoError(DvcException):
)
class GetDVCFileError(DvcException):
def __init__(self):
super(GetDVCFileError, self).__init__(
"the given path is a DVC-file, you must specify a data file "
"or a directory"
)
class GitHookAlreadyExistsError(DvcException):
def __init__(self, hook_name):
super(GitHookAlreadyExistsError, self).__init__(
dvc/external_repo.py
+34 -1
View File
@@ -7,8 +7,9 @@ from distutils.dir_util import copy_tree
from funcy import retry
from dvc.config import NoRemoteError
from dvc.config import NoRemoteError, ConfigError
from dvc.exceptions import NoRemoteInExternalRepoError
from dvc.remote import RemoteConfig
from dvc.exceptions import NoOutputInExternalRepoError
from dvc.exceptions import OutputNotFoundError
from dvc.utils.fs import remove
@@ -69,6 +70,22 @@ def _external_repo(url=None, rev=None, cache_dir=None):
repo = Repo(new_path)
try:
# check if the URL is local and no default remote is present
# add default remote pointing to the original repo's cache location
if os.path.isdir(url):
rconfig = RemoteConfig(repo.config)
if not _default_remote_set(rconfig):
original_repo = Repo(url)
try:
rconfig.add(
"auto-generated-upstream",
original_repo.cache.local.cache_dir,
default=True,
level=Config.LEVEL_LOCAL,
)
finally:
original_repo.close()
if cache_dir is not None:
cache_config = CacheConfig(repo.config)
cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
@@ -113,3 +130,19 @@ def _clone_repo(url, path):
git = Git.clone(url, path)
git.close()
def _default_remote_set(rconfig):
"""
Checks if default remote config is present.
Args:
rconfig: a remote config
Returns:
True if the default remote config is set, else False
"""
try:
rconfig.get_default()
return True
except ConfigError:
return False
dvc/lock.py
+66 -52
View File
@@ -6,12 +6,12 @@ import os
import time
from datetime import timedelta
from funcy.py3 import lkeep
import zc.lockfile
from dvc.exceptions import DvcException
from dvc.utils import makedirs
from dvc.utils.compat import is_py3
from dvc.utils import format_link
from dvc.utils.compat import is_py3, is_py2
from dvc.progress import Tqdm
DEFAULT_TIMEOUT = 5
@@ -26,10 +26,60 @@ class LockError(DvcException):
"""Thrown when unable to acquire the lock for dvc repo."""
class Lock(object):
"""Class for dvc repo lock.
Uses zc.lockfile as backend.
"""
def __init__(self, lockfile, friendly=False, **kwargs):
self._friendly = friendly
self.lockfile = lockfile
self._lock = None
@property
def files(self):
return [self.lockfile]
def _do_lock(self):
try:
with Tqdm(
bar_format="{desc}",
disable=not self._friendly,
desc=(
"If DVC froze, see `hardlink_lock` in {}".format(
format_link("man.dvc.org/config#core")
)
),
):
self._lock = zc.lockfile.LockFile(self.lockfile)
except zc.lockfile.LockError:
raise LockError(FAILED_TO_LOCK_MESSAGE)
def lock(self):
try:
self._do_lock()
return
except LockError:
time.sleep(DEFAULT_TIMEOUT)
self._do_lock()
def unlock(self):
self._lock.close()
self._lock = None
def __enter__(self):
self.lock()
def __exit__(self, typ, value, tbck):
self.unlock()
if is_py3:
import flufl.lock
class Lock(flufl.lock.Lock):
class HardlinkLock(flufl.lock.Lock):
"""Class for dvc repo lock.
Args:
@@ -38,12 +88,10 @@ if is_py3:
tmp_dir (str): a directory to store claim files.
"""
def __init__(self, lockfile, tmp_dir=None):
def __init__(self, lockfile, tmp_dir=None, **kwargs):
import socket
self._tmp_dir = tmp_dir
if self._tmp_dir is not None:
makedirs(self._tmp_dir, exist_ok=True)
# NOTE: this is basically Lock.__init__ copy-paste, except that
# instead of using `socket.getfqdn()` we use `socket.gethostname()`
@@ -69,10 +117,6 @@ if is_py3:
def lockfile(self):
return self._lockfile
@property
def files(self):
return lkeep([self._lockfile, self._tmp_dir])
def lock(self):
try:
super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
@@ -101,44 +145,14 @@ if is_py3:
pass
else:
import zc.lockfile
def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False):
if hardlink_lock and is_py2:
raise DvcException(
"Hardlink locks are not supported on Python <3.5. "
"See `hardlink_lock` in {}".format(
format_link("man.dvc.org/config#core")
)
)
class Lock(object):
"""Class for dvc repo lock.
Uses zc.lockfile as backend.
"""
def __init__(self, lockfile, tmp_dir=None):
self.lockfile = lockfile
self._lock = None
@property
def files(self):
return [self.lockfile]
def _do_lock(self):
try:
self._lock = zc.lockfile.LockFile(self.lockfile)
except zc.lockfile.LockError:
raise LockError(FAILED_TO_LOCK_MESSAGE)
def lock(self):
try:
self._do_lock()
return
except LockError:
time.sleep(DEFAULT_TIMEOUT)
self._do_lock()
def unlock(self):
self._lock.close()
self._lock = None
def __enter__(self):
self.lock()
def __exit__(self, typ, value, tbck):
self.unlock()
cls = HardlinkLock if hardlink_lock else Lock
return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly)
dvc/main.py
+3 -2
View File
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
import logging
from dvc.analytics import Analytics
from dvc import analytics
from dvc.cli import parse_args
from dvc.config import ConfigError
from dvc.exceptions import DvcParserError
@@ -81,6 +81,7 @@ def main(argv=None):
if ret != 0:
logger.info(FOOTER)
Analytics().send_cmd(cmd, args, ret)
if analytics.is_enabled():
analytics.collect_and_send_report(args, ret)
return ret
dvc/path_info.py
+11 -2
View File
@@ -23,7 +23,16 @@ if is_py2:
fs_encoding = "utf-8"
class PathInfo(pathlib.PurePath):
class _BasePath(object):
def overlaps(self, other):
if isinstance(other, basestring):
other = self.__class__(other)
elif self.__class__ != other.__class__:
return False
return self == other or self.isin(other) or other.isin(self)
class PathInfo(pathlib.PurePath, _BasePath):
# Use __slots__ in PathInfo objects following PurePath implementation.
# This makes objects smaller and speeds up attribute access.
# We don't add any fields so it's empty.
@@ -138,7 +147,7 @@ class _URLPathParents(object):
return "<{}.parents>".format(self.src)
class URLInfo(object):
class URLInfo(_BasePath):
DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22, "hdfs": 0}
def __init__(self, url):
dvc/remote/config.py
-1
View File
@@ -105,7 +105,6 @@ class RemoteConfig(object):
level=level,
)
if default:
logger.info("Setting '{}' as a default remote.".format(name))
self.config.set(
Config.SECTION_CORE,
Config.SECTION_CORE_REMOTE,
dvc/remote/s3.py
+36 -2
View File
@@ -5,10 +5,9 @@ import logging
import os
import threading
from botocore.exceptions import ClientError
from funcy import cached_property, wrap_prop
from dvc.config import Config
from dvc.config import Config, ConfigError
from dvc.exceptions import DvcException
from dvc.exceptions import ETagMismatchError
from dvc.path_info import CloudURLInfo
@@ -54,6 +53,8 @@ class RemoteS3(RemoteBASE):
if self.acl:
self.extra_args["ACL"] = self.acl
self._append_aws_grants_to_extra_args(config)
shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
if shared_creds:
os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
@@ -209,6 +210,8 @@ class RemoteS3(RemoteBASE):
return self._list_paths(self.path_info)
def isfile(self, path_info):
from botocore.exceptions import ClientError
if path_info.path.endswith("/"):
return False
@@ -301,3 +304,34 @@ class RemoteS3(RemoteBASE):
continue
yield path_info.replace(path=fname)
def _append_aws_grants_to_extra_args(self, config):
# Keys for extra_args can be one of the following list:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS
"""
ALLOWED_UPLOAD_ARGS = [
'ACL', 'CacheControl', 'ContentDisposition', 'ContentEncoding',
'ContentLanguage', 'ContentType', 'Expires', 'GrantFullControl',
'GrantRead', 'GrantReadACP', 'GrantWriteACP', 'Metadata',
'RequestPayer', 'ServerSideEncryption', 'StorageClass',
'SSECustomerAlgorithm', 'SSECustomerKey', 'SSECustomerKeyMD5',
'SSEKMSKeyId', 'WebsiteRedirectLocation'
]
"""
grants = {
Config.SECTION_AWS_GRANT_FULL_CONTROL: "GrantFullControl",
Config.SECTION_AWS_GRANT_READ: "GrantRead",
Config.SECTION_AWS_GRANT_READ_ACP: "GrantReadACP",
Config.SECTION_AWS_GRANT_WRITE_ACP: "GrantWriteACP",
}
for grant_option, extra_args_key in grants.items():
if config.get(grant_option, ""):
if self.acl:
raise ConfigError(
"`acl` and `grant_*` AWS S3 config options "
"are mutually exclusive"
)
self.extra_args[extra_args_key] = config.get(grant_option)
dvc/repo/__init__.py
+12 -9
View File
@@ -69,13 +69,14 @@ class Repo(object):
def __init__(self, root_dir=None):
from dvc.state import State
from dvc.lock import Lock
from dvc.lock import make_lock
from dvc.scm import SCM
from dvc.cache import Cache
from dvc.data_cloud import DataCloud
from dvc.repo.metrics import Metrics
from dvc.scm.tree import WorkingTree
from dvc.repo.tag import Tag
from dvc.utils import makedirs
root_dir = self.find_root(root_dir)
@@ -88,10 +89,17 @@ class Repo(object):
self.tree = WorkingTree(self.root_dir)
self.lock = Lock(
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
makedirs(self.tmp_dir, exist_ok=True)
hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
self.lock = make_lock(
os.path.join(self.dvc_dir, "lock"),
tmp_dir=os.path.join(self.dvc_dir, "tmp"),
hardlink_lock=hardlink_lock,
friendly=True,
)
# NOTE: storing state and link_state in the repository itself to avoid
# any possible state corruption in 'shared cache dir' scenario.
self.state = State(self, self.config.config)
@@ -162,9 +170,8 @@ class Repo(object):
flist = (
[self.config.config_local_file, updater.updater_file]
+ [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir]
+ self.state.files
+ self.lock.files
+ updater.lock.files
)
if path_isin(self.cache.local.cache_dir, self.root_dir):
@@ -349,11 +356,7 @@ class Repo(object):
continue
for out in outs:
if (
out == dep.path_info
or dep.path_info.isin(out)
or out.isin(dep.path_info)
):
if out.overlaps(dep.path_info):
dep_stage = outs[out].stage
dep_node = relpath(dep_stage.path, self.root_dir)
G.add_node(dep_node, stage=dep_stage)
dvc/repo/get.py
+57 -18
View File
@@ -1,22 +1,42 @@
import logging
import os
import shutil
from dvc.utils.compat import FileNotFoundError
import shortuuid
from dvc.exceptions import GetDVCFileError
from dvc.exceptions import NotDvcRepoError
from dvc.exceptions import OutputNotFoundError
from dvc.exceptions import UrlNotDvcRepoError
from dvc.exceptions import (
DvcException,
NotDvcRepoError,
OutputNotFoundError,
UrlNotDvcRepoError,
)
from dvc.external_repo import external_repo
from dvc.path_info import PathInfo
from dvc.stage import Stage
from dvc.state import StateNoop
from dvc.utils import resolve_output
from dvc.utils.fs import remove
logger = logging.getLogger(__name__)
class GetDVCFileError(DvcException):
def __init__(self):
super(GetDVCFileError, self).__init__(
"the given path is a DVC-file, you must specify a data file "
"or a directory"
)
class PathMissingError(DvcException):
def __init__(self, path, repo):
msg = (
"The path '{}' does not exist in the target repository '{}'"
" neighther as an output nor a git-handled file."
)
super(PathMissingError, self).__init__(msg.format(path, repo))
@staticmethod
def get(url, path, out=None, rev=None):
out = resolve_output(path, out)
@@ -33,11 +53,6 @@ def get(url, path, out=None, rev=None):
tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid()))
try:
with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo:
# Note: we need to replace state, because in case of getting DVC
# dependency on CIFS or NFS filesystems, sqlite-based state
# will be unable to obtain lock
repo.state = StateNoop()
# Try any links possible to avoid data duplication.
#
# Not using symlink, because we need to remove cache after we are
@@ -49,16 +64,40 @@ def get(url, path, out=None, rev=None):
# the same cache file might be used a few times in a directory.
repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]
o = repo.find_out_by_relpath(path)
with repo.state:
repo.cloud.pull(o.get_used_cache())
o.path_info = PathInfo(os.path.abspath(out))
with o.repo.state:
o.checkout()
try:
output = repo.find_out_by_relpath(path)
except OutputNotFoundError:
output = None
if output and output.use_cache:
_get_cached(repo, output, out)
else:
# Either an uncached out with absolute path or a user error
if os.path.isabs(path):
raise FileNotFoundError
_copy(os.path.join(repo.root_dir, path), out)
except (OutputNotFoundError, FileNotFoundError):
raise PathMissingError(path, url)
except NotDvcRepoError:
raise UrlNotDvcRepoError(url)
except OutputNotFoundError:
raise OutputNotFoundError(path)
finally:
remove(tmp_dir)
def _get_cached(repo, output, out):
with repo.state:
repo.cloud.pull(output.get_used_cache())
output.path_info = PathInfo(os.path.abspath(out))
failed = output.checkout()
# This might happen when pull haven't really pulled all the files
if failed:
raise FileNotFoundError
def _copy(src, dst):
if os.path.isdir(src):
shutil.copytree(src, dst)
else:
shutil.copy2(src, dst)
dvc/repo/init.py
+2 -2
View File
@@ -3,7 +3,7 @@ import os
import colorama
from dvc.analytics import Analytics
from dvc import analytics
from dvc.config import Config
from dvc.exceptions import InitError
from dvc.repo import Repo
@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
def _welcome_message():
if Analytics.is_enabled():
if analytics.is_enabled():
logger.info(
boxify(
"DVC has enabled anonymous aggregate usage analytics.\n"
dvc/rwlock.py
+191
View File
@@ -0,0 +1,191 @@
from __future__ import unicode_literals
import os
import json
from collections import defaultdict
from contextlib import contextmanager
from voluptuous import Schema, Invalid
from funcy.py3 import lmap, lfilter, lmapcat
from .exceptions import DvcException
from .lock import LockError
from .utils.compat import (
convert_to_unicode,
FileNotFoundError,
JSONDecodeError,
str,
)
from .utils.fs import relpath
INFO_SCHEMA = {"pid": int, "cmd": str}
SCHEMA = Schema({"write": {str: INFO_SCHEMA}, "read": {str: [INFO_SCHEMA]}})
class RWLockFileCorruptedError(DvcException):
def __init__(self, path, cause):
super(RWLockFileCorruptedError, self).__init__(
"Unable to read RWLock-file '{}'. JSON structure is "
"corrupted".format(relpath(path)),
cause=cause,
)
class RWLockFileFormatError(DvcException):
def __init__(self, path, cause):
super(RWLockFileFormatError, self).__init__(
"RWLock-file '{}' format error.".format(relpath(path)), cause=cause
)
@contextmanager
def _edit_rwlock(lock_dir):
path = os.path.join(lock_dir, "rwlock")
try:
with open(path, "r") as fobj:
lock = json.load(fobj)
lock = SCHEMA(convert_to_unicode(lock))
except FileNotFoundError:
lock = SCHEMA({})
except JSONDecodeError as exc:
raise RWLockFileCorruptedError(path, cause=exc)
except Invalid as exc:
raise RWLockFileFormatError(path, cause=exc)
lock = defaultdict(dict, lock)
lock["read"] = defaultdict(list, lock["read"])
lock["write"] = defaultdict(dict, lock["write"])
yield lock
with open(path, "w+") as fobj:
json.dump(lock, fobj)
def _infos_to_str(infos):
return "\n".join(
" (PID {}): {}".format(info["pid"], info["cmd"]) for info in infos
)
def _check_no_writers(lock, info, path_infos):
for path_info in path_infos:
blocking_urls = lfilter(path_info.overlaps, lock["write"])
if not blocking_urls:
continue
writers = lmap(lock["write"].get, blocking_urls)
writers = lfilter(lambda i: info != i, writers)
if not writers:
continue
raise LockError(
"'{}' is busy, it is being written to by:\n{}".format(
str(path_info), _infos_to_str(writers)
)
)
def _check_no_readers(lock, info, path_infos):
for path_info in path_infos:
blocking_urls = lfilter(path_info.overlaps, lock["read"])
if not blocking_urls:
continue
readers = lmapcat(lock["read"].get, blocking_urls)
readers = lfilter(lambda i: info != i, readers)
if not readers:
continue
raise LockError(
"'{}' is busy, it is being read by:\n{}".format(
str(path_info), _infos_to_str(readers)
)
)
def _acquire_read(lock, info, path_infos):
changes = []
for path_info in path_infos:
url = path_info.url
readers = lock["read"][url]
if info in readers:
continue
changes.append(url)
readers.append(info)
return changes
def _acquire_write(lock, info, path_infos):
changes = []
for path_info in path_infos:
url = path_info.url
if lock["write"][url] == info:
continue
changes.append(url)
lock["write"][url] = info
return changes
def _release_write(lock, info, changes):
for url in changes:
assert "write" in lock
assert url in lock["write"]
assert lock["write"][url] == info
del lock["write"][url]
if not lock["write"]:
del lock["write"]
def _release_read(lock, info, changes):
for url in changes:
assert "read" in lock
assert url in lock["read"]
assert info in lock["read"][url]
lock["read"][url].remove(info)
if not lock["read"][url]:
del lock["read"][url]
if not lock["read"]:
del lock["read"]
@contextmanager
def rwlock(tmp_dir, cmd, read, write):
"""Create non-thread-safe RWLock for PathInfos.
Args:
tmp_dir (str): existing directory where to create the rwlock file.
cmd (str): command that will be working on these PathInfos.
read ([PathInfo]): PathInfos that are going to be read.
write ([PathInfo]): PathInfos that are going to be written.
Raises:
LockError: raised if PathInfo we want to read is being written to by
another command or if PathInfo we want to write is being written
to or read from by another command.
RWLockFileCorruptedError: raised if rwlock file is not a valid JSON.
RWLockFileFormatError: raised if rwlock file is a valid JSON, but
has internal format that doesn't pass our schema validation.
"""
info = {"pid": os.getpid(), "cmd": cmd}
with _edit_rwlock(tmp_dir) as lock:
_check_no_writers(lock, info, read + write)
_check_no_readers(lock, info, write)
rchanges = _acquire_read(lock, info, read)
wchanges = _acquire_write(lock, info, write)
try:
yield
finally:
with _edit_rwlock(tmp_dir) as lock:
_release_write(lock, info, wchanges)
_release_read(lock, info, rchanges)
dvc/stage.py
+58 -1
View File
@@ -7,7 +7,10 @@ import re
import signal
import subprocess
import threading
from functools import wraps
from itertools import chain
from funcy import decorator
from voluptuous import Any, Schema, MultipleInvalid
@@ -129,6 +132,52 @@ class MissingDataSource(DvcException):
super(MissingDataSource, self).__init__(msg)
@decorator
def rwlocked(call, read=None, write=None):
import sys
from dvc.rwlock import rwlock
from dvc.dependency.repo import DependencyREPO
if read is None:
read = []
if write is None:
write = []
stage = call._args[0]
def _chain(names):
return [
item.path_info
for attr in names
for item in getattr(stage, attr)
# There is no need to lock DependencyREPO deps, as there is no
# corresponding OutputREPO, so we can't even write it.
if not isinstance(item, DependencyREPO)
]
cmd = " ".join(sys.argv)
with rwlock(stage.repo.tmp_dir, cmd, _chain(read), _chain(write)):
return call()
def unlocked_repo(f):
@wraps(f)
def wrapper(stage, *args, **kwargs):
stage.repo.state.dump()
stage.repo.lock.unlock()
stage.repo._reset()
try:
ret = f(stage, *args, **kwargs)
finally:
stage.repo.lock.lock()
stage.repo.state.load()
return ret
return wrapper
class Stage(object):
STAGE_FILE = "Dvcfile"
STAGE_FILE_SUFFIX = ".dvc"
@@ -286,6 +335,7 @@ class Stage(object):
return True
return False
@rwlocked(read=["deps", "outs"])
def changed(self):
# Short-circuit order: stage md5 is fast, deps are expected to change
ret = (
@@ -299,6 +349,7 @@ class Stage(object):
return ret
@rwlocked(write=["outs"])
def remove_outs(self, ignore_remove=False, force=False):
"""Used mainly for `dvc remove --outs` and :func:`Stage.reproduce`."""
for out in self.outs:
@@ -316,6 +367,7 @@ class Stage(object):
for out in self.outs:
out.unprotect()
@rwlocked(write=["outs"])
def remove(self, force=False, remove_outs=True):
if remove_outs:
self.remove_outs(ignore_remove=True, force=force)
@@ -323,6 +375,7 @@ class Stage(object):
self.unprotect_outs()
os.unlink(self.path)
@rwlocked(read=["deps"], write=["outs"])
def reproduce(self, interactive=False, **kwargs):
if not kwargs.get("force", False) and not self.changed():
@@ -755,6 +808,7 @@ class Stage(object):
)
self.save()
@rwlocked(write=["outs"])
def commit(self):
for out in self.outs:
out.commit()
@@ -801,6 +855,7 @@ class Stage(object):
if occurrence > 1:
raise ArgumentDuplicationError(str(path))
@unlocked_repo
def _run(self):
self._check_missing_deps()
@@ -850,6 +905,7 @@ class Stage(object):
if (p is None) or (p.returncode != 0):
raise StageCmdFailedError(self)
@rwlocked(read=["deps"], write=["outs"])
def run(self, dry=False, no_commit=False, force=False):
if (self.cmd or self.is_import) and not self.locked and not dry:
self.remove_outs(ignore_remove=False, force=False)
@@ -874,7 +930,6 @@ class Stage(object):
self.outs[0].checkout()
else:
self.deps[0].download(self.outs[0])
elif self.is_data_source:
msg = "Verifying data sources in '{}'".format(self.relpath)
logger.info(msg)
@@ -904,6 +959,7 @@ class Stage(object):
if paths:
raise MissingDataSource(paths)
@rwlocked(write=["outs"])
def checkout(self, force=False, progress_callback=None, relink=False):
failed_checkouts = []
for out in self.outs:
@@ -926,6 +982,7 @@ class Stage(object):
return ret
@rwlocked(read=["deps", "outs"])
def status(self):
ret = []
dvc/updater.py
+7 -5
View File
@@ -9,8 +9,7 @@ import colorama
from packaging import version
from dvc import __version__
from dvc.lock import Lock
from dvc.lock import LockError
from dvc.lock import make_lock, LockError
from dvc.utils import boxify
from dvc.utils import env2bool
from dvc.utils.pkg import PKG
@@ -24,11 +23,14 @@ class Updater(object): # pragma: no cover
TIMEOUT = 24 * 60 * 60 # every day
TIMEOUT_GET = 10
def __init__(self, dvc_dir):
def __init__(self, dvc_dir, friendly=False, hardlink_lock=False):
self.dvc_dir = dvc_dir
self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
self.lock = Lock(
self.updater_file + ".lock", tmp_dir=os.path.join(dvc_dir, "tmp")
self.lock = make_lock(
self.updater_file + ".lock",
tmp_dir=os.path.join(dvc_dir, "tmp"),
friendly=friendly,
hardlink_lock=hardlink_lock,
)
self.current = version.parse(__version__).base_version
dvc/utils/__init__.py
+8
View File
@@ -420,3 +420,11 @@ def resolve_output(inp, out):
if os.path.isdir(out):
return os.path.join(out, name)
return out
def format_link(link):
import colorama
return "<{blue}{link}{nc}>".format(
blue=colorama.Fore.CYAN, link=link, nc=colorama.Fore.RESET
)
dvc/utils/compat.py
+2
View File
@@ -123,6 +123,7 @@ if is_py2:
makedirs = _makedirs
range = xrange # noqa: F821
FileNotFoundError = IOError
JSONDecodeError = ValueError
import StringIO
import io
@@ -164,6 +165,7 @@ elif is_py3:
import configparser as ConfigParser # noqa: F401
from collections.abc import Mapping # noqa: F401
from contextlib import ExitStack # noqa: F401
from json.decoder import JSONDecodeError # noqa: F401
builtin_str = str # noqa: F821
str = str # noqa: F821
dvc/version.py
+1 -1
View File
@@ -7,7 +7,7 @@ import os
import subprocess
_BASE_VERSION = "0.75.0"
_BASE_VERSION = "0.77.1"
def _generate_version(base_version):
setup.py
+3 -7
View File
@@ -79,6 +79,7 @@ install_requires = [
"shortuuid>=0.5.0",
"tqdm>=4.40.0,<5",
"packaging>=19.0",
"zc.lockfile>=1.2.1",
"win-unicode-console>=0.5; sys_platform == 'win32'",
"pywin32>=225; sys_platform == 'win32'",
]
@@ -104,7 +105,7 @@ ssh = ["paramiko>=2.5.0"]
# requirements, including kerberos itself. Once all the wheels are available,
# we can start shipping it by default.
ssh_gssapi = ["paramiko[gssapi]>=2.5.0"]
hdfs = ["pyarrow==0.14.0"]
hdfs = ["pyarrow==0.15.1"]
all_remotes = gs + s3 + azure + ssh + oss + gdrive
if os.name != "nt" or sys.version_info[0] != 2:
@@ -163,12 +164,7 @@ setup(
"ssh_gssapi": ssh_gssapi,
"hdfs": hdfs,
# NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1
":python_version=='2.7'": [
"futures",
"pathlib2",
"contextlib2",
"zc.lockfile>=1.2.1",
],
":python_version=='2.7'": ["futures", "pathlib2", "contextlib2"],
":python_version>='3.0'": ["flufl.lock>=3.2"],
"tests": tests_requirements,
},
snap/snapcraft.yaml
-18
View File
@@ -6,13 +6,6 @@ grade: stable
confinement: classic # for same reasons as snap `git-ubuntu`
base: core18
license: Apache-2.0
plugs:
config-dvc:
interface: system-files
read:
- /etc/host.conf
- /etc/resolv.conf
- /etc/hosts
layout:
/etc/dvc:
bind: $SNAP_DATA/etc/dvc
@@ -48,18 +41,7 @@ apps:
environment:
GIT_PYTHON_GIT_EXECUTABLE: $SNAP/usr/bin/git
XDG_CONFIG_DIRS: /etc
plugs:
- config-dvc
- home
- mount-observe
- network-control
- removable-media
git:
command: usr/bin/git
environment:
GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
plugs:
- home
- mount-observe
- network-control
- removable-media
tests/conftest.py
-6
View File
@@ -7,7 +7,6 @@ import pytest
from git import Repo
from git.exc import GitCommandNotFound
from dvc.remote.config import RemoteConfig
from dvc.remote.ssh.connection import SSHConnection
from dvc.repo import Repo as DvcRepo
from dvc.utils.compat import cast_bytes_py2
@@ -148,11 +147,6 @@ def erepo(repo_dir):
repo.dvc.scm.add([stage_foo.path, stage_bar.path, stage_data_dir.path])
repo.dvc.scm.commit("init repo")
rconfig = RemoteConfig(repo.dvc.config)
rconfig.add("upstream", repo.dvc.cache.local.cache_dir, default=True)
repo.dvc.scm.add([repo.dvc.config.config_file])
repo.dvc.scm.commit("add remote")
repo.create("version", "master")
repo.dvc.add("version")
repo.dvc.scm.add([".gitignore", "version.dvc"])
tests/dir_helpers.py
+70 -23
View File
@@ -1,3 +1,47 @@
"""
The goal of this module is making dvc functional tests setup a breeze. This
includes a temporary dir, initializing git and dvc repos and bootstrapping some
file structure.
The cornerstone of these fixtures is `tmp_dir`, which creates a temporary dir
and changes path to it, it might be combined with `scm` and `dvc` to initialize
empty git and dvc repos. `tmp_dir` returns a Path instance, which should save
you from using `open()`, `os` and `os.path` utils many times:
(tmp_dir / "some_file").write_text("some text")
# ...
assert "some text" == (tmp_dir / "some_file").read_text()
assert (tmp_dir / "some_file").exists()
Additionally it provides `.gen()`, `.scm_gen()` and `.dvc_gen()` methods to
bootstrap a required file structure in a single call:
# Generate a dir with files
tmp_dir.gen({"dir": {"file": "file text", "second_file": "..."}})
# Generate a single file, dirs will be created along the way
tmp_dir.gen("dir/file", "file text")
# Generate + git add
tmp_dir.scm_gen({"file1": "...", ...})
# Generate + git add + git commit
tmp_dir.scm_gen({"file1": "...", ...}, commit="add files")
# Generate + dvc add
tmp_dir.dvc_gen({"file1": "...", ...})
# Generate + dvc add + git commit -am "..."
# This commits stages to git not the generated files.
tmp_dir.dvc_gen({"file1": "...", ...}, commit="add files")
Making it easier to bootstrap things has a supergoal of incentivizing a move
from global repo template to creating everything inplace, which:
- makes all path references local to test, enhancing readability
- allows using telling filenames, e.g. "git_tracked_file" instead of "foo"
- does not create unnecessary files
"""
from __future__ import unicode_literals
import os
@@ -6,7 +50,7 @@ import pytest
from funcy.py3 import lmap, retry
from dvc.utils import makedirs
from dvc.utils.compat import basestring, is_py2, pathlib, fspath, fspath_py35
from dvc.utils.compat import basestring, pathlib, fspath, fspath_py35, bytes
__all__ = ["tmp_dir", "scm", "dvc", "repo_template", "run_copy", "erepo_dir"]
@@ -44,7 +88,7 @@ class TmpDir(pathlib.Path):
)
def gen(self, struct, text=""):
if isinstance(struct, basestring):
if isinstance(struct, (basestring, pathlib.PurePath)):
struct = {struct: text}
self._gen(struct)
@@ -55,13 +99,16 @@ class TmpDir(pathlib.Path):
path = (prefix or self) / name
if isinstance(contents, dict):
self._gen(contents, prefix=path)
if not contents:
makedirs(path, exist_ok=True)
else:
self._gen(contents, prefix=path)
else:
makedirs(path.parent, exist_ok=True)
if is_py2 and isinstance(contents, str):
if isinstance(contents, bytes):
path.write_bytes(contents)
else:
path.write_text(contents)
path.write_text(contents, encoding="utf-8")
def dvc_gen(self, struct, text="", commit=None):
paths = self.gen(struct, text)
@@ -182,7 +229,7 @@ def run_copy(tmp_dir, dvc, request):
# Do we need this?
if "scm" in request.fixturenames:
request.getfixturevalue("scm")
tmp_dir.git_add("copy.py", commit="add copy.py")
tmp_dir.scm_add("copy.py", commit="add copy.py")
def run_copy(src, dst, **run_kwargs):
return dvc.run(
@@ -203,27 +250,27 @@ def erepo_dir(tmp_path_factory, monkeypatch):
path = TmpDir(fspath_py35(tmp_path_factory.mktemp("erepo")))
# Chdir for git and dvc to work locally
monkeypatch.chdir(fspath_py35(path))
with monkeypatch.context() as m:
m.chdir(fspath_py35(path))
_git_init()
path.dvc = Repo.init()
path.scm = path.dvc.scm
path.dvc_gen(REPO_TEMPLATE, commit="init repo")
_git_init()
path.dvc = Repo.init()
path.scm = path.dvc.scm
path.dvc_gen(REPO_TEMPLATE, commit="init repo")
rconfig = RemoteConfig(path.dvc.config)
rconfig.add("upstream", path.dvc.cache.local.cache_dir, default=True)
path.scm_add([path.dvc.config.config_file], commit="add remote")
rconfig = RemoteConfig(path.dvc.config)
rconfig.add("upstream", path.dvc.cache.local.cache_dir, default=True)
path.scm_add([path.dvc.config.config_file], commit="add remote")
path.dvc_gen("version", "master")
path.scm_add([".gitignore", "version.dvc"], commit="master")
path.dvc_gen("version", "master")
path.scm_add([".gitignore", "version.dvc"], commit="master")
path.scm.checkout("branch", create_new=True)
(path / "version").unlink() # For mac ???
path.dvc_gen("version", "branch")
path.scm_add([".gitignore", "version.dvc"], commit="branch")
path.scm.checkout("branch", create_new=True)
(path / "version").unlink() # For mac ???
path.dvc_gen("version", "branch")
path.scm_add([".gitignore", "version.dvc"], commit="branch")
path.scm.checkout("master")
path.dvc.close()
monkeypatch.undo() # Undo chdir
path.scm.checkout("master")
path.dvc.close()
return path
tests/func/test_add.py
+33 -43
View File
@@ -31,57 +31,47 @@ from tests.utils import get_gitignore_content
from tests.utils import spy
class TestAdd(TestDvc):
def test(self):
md5 = file_md5(self.FOO)[0]
def test_add(tmp_dir, dvc):
stage, = tmp_dir.dvc_gen({"foo": "foo"})
md5, _ = file_md5("foo")
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
assert stage is not None
self.assertIsInstance(stage, Stage)
self.assertTrue(os.path.isfile(stage.path))
self.assertEqual(len(stage.outs), 1)
self.assertEqual(len(stage.deps), 0)
self.assertEqual(stage.cmd, None)
self.assertEqual(stage.outs[0].info["md5"], md5)
self.assertEqual(stage.md5, "ee343f2482f53efffc109be83cc976ac")
def test_unicode(self):
fname = "\xe1"
with open(fname, "w") as fobj:
fobj.write("something")
stage = self.dvc.add(fname)[0]
self.assertTrue(os.path.isfile(stage.path))
assert isinstance(stage, Stage)
assert os.path.isfile(stage.path)
assert len(stage.outs) == 1
assert len(stage.deps) == 0
assert stage.cmd is None
assert stage.outs[0].info["md5"] == md5
assert stage.md5 == "ee343f2482f53efffc109be83cc976ac"
class TestAddUnsupportedFile(TestDvc):
def test(self):
with self.assertRaises(DvcException):
self.dvc.add("unsupported://unsupported")
def test_add_unicode(tmp_dir, dvc):
with open("\xe1", "wb") as fd:
fd.write("something".encode("utf-8"))
stage, = dvc.add("\xe1")
assert os.path.isfile(stage.path)
class TestAddDirectory(TestDvc):
def test(self):
dname = "directory"
os.mkdir(dname)
self.create(os.path.join(dname, "file"), "file")
stages = self.dvc.add(dname)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
self.assertEqual(len(stage.deps), 0)
self.assertEqual(len(stage.outs), 1)
def test_add_unsupported_file(dvc):
with pytest.raises(DvcException):
dvc.add("unsupported://unsupported")
md5 = stage.outs[0].info["md5"]
dir_info = self.dvc.cache.local.load_dir_cache(md5)
for info in dir_info:
self.assertTrue("\\" not in info["relpath"])
def test_add_directory(tmp_dir, dvc):
stage, = tmp_dir.dvc_gen({"dir": {"file": "file"}})
assert stage is not None
assert len(stage.deps) == 0
assert len(stage.outs) == 1
md5 = stage.outs[0].info["md5"]
dir_info = dvc.cache.local.load_dir_cache(md5)
for info in dir_info:
assert "\\" not in info["relpath"]
class TestAddDirectoryRecursive(TestDvc):
tests/func/test_analytics.py
+13 -58
View File
@@ -1,65 +1,20 @@
import os
import mock
import requests
from dvc.analytics import Analytics
from dvc.main import main
from tests.basic_env import TestDir
from tests.basic_env import TestDvc
from tests.basic_env import TestGit
from dvc.utils.compat import fspath
def _clean_getenv(key, default=None):
"""
Remove env vars that affect dvc behavior in tests
"""
if key in ["DVC_TEST", "CI"]:
return None
return os.environ.get(key, default)
@mock.patch("dvc.analytics.send")
def test_daemon_analytics(mock_send, tmp_path):
report = fspath(tmp_path)
assert 0 == main(["daemon", "analytics", report])
mock_send.assert_called_with(report)
class TestAnalytics(TestDir):
def test(self):
a = Analytics()
a.collect()
self.assertTrue(isinstance(a.info, dict))
self.assertNotEqual(a.info, {})
self.assertTrue(a.PARAM_USER_ID in a.info.keys())
self.assertTrue(a.PARAM_SYSTEM_INFO in a.info.keys())
self.assertNotEqual(a.info[a.PARAM_SYSTEM_INFO], {})
@mock.patch.object(os, "getenv", new=_clean_getenv)
@mock.patch("requests.post")
def test_send(self, mockpost):
ret = main(["daemon", "analytics", Analytics().dump(), "-v"])
self.assertEqual(ret, 0)
self.assertTrue(mockpost.called)
@mock.patch.object(os, "getenv", new=_clean_getenv)
@mock.patch.object(
requests, "post", side_effect=requests.exceptions.RequestException()
)
def test_send_failed(self, mockpost):
ret = main(["daemon", "analytics", Analytics().dump(), "-v"])
self.assertEqual(ret, 0)
self.assertTrue(mockpost.called)
class TestAnalyticsGit(TestAnalytics, TestGit):
pass
class TestAnalyticsDvc(TestAnalytics, TestDvc):
@mock.patch("requests.post")
def test_send_disabled(self, mockpost):
ret = main(["config", "core.analytics", "false"])
self.assertEqual(ret, 0)
with mock.patch.object(os, "getenv", new=_clean_getenv):
ret = main(["daemon", "analytics", Analytics().dump(), "-v"])
self.assertEqual(ret, 0)
self.assertFalse(mockpost.called)
@mock.patch("dvc.analytics.collect_and_send_report")
@mock.patch("dvc.analytics.is_enabled", return_value=True)
def test_main_analytics(mock_is_enabled, mock_report, dvc_repo):
assert 0 == main(["add", "foo"])
assert mock_is_enabled.called
assert mock_report.called
tests/func/test_checkout.py
+7 -4
View File
@@ -474,11 +474,14 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
)
def test_checkout_no_checksum(repo_dir, dvc_repo):
stage = dvc_repo.run(outs=[repo_dir.FOO], no_exec=True, cmd="somecmd")
def test_checkout_no_checksum(tmp_dir, dvc):
tmp_dir.gen("file", "file content")
stage = dvc.run(outs=["file"], no_exec=True, cmd="somecmd")
with pytest.raises(CheckoutError):
dvc_repo.checkout([stage.path], force=True)
assert not os.path.exists(repo_dir.FOO)
dvc.checkout([stage.path], force=True)
assert not os.path.exists("file")
@pytest.mark.parametrize(
tests/func/test_commit.py
+22 -26
View File
@@ -7,40 +7,37 @@ from dvc.utils.stage import dump_stage_file
from dvc.utils.stage import load_stage_file
def test_commit_recursive(dvc_repo, repo_dir):
stages = dvc_repo.add(repo_dir.DATA_DIR, recursive=True, no_commit=True)
def test_commit_recursive(tmp_dir, dvc):
tmp_dir.gen({"dir": {"file": "text1", "subdir": {"file2": "text2"}}})
stages = dvc.add("dir", recursive=True, no_commit=True)
assert len(stages) == 2
assert dvc.status() != {}
assert dvc_repo.status() != {}
dvc_repo.commit(repo_dir.DATA_DIR, recursive=True)
assert dvc_repo.status() == {}
dvc.commit("dir", recursive=True)
assert dvc.status() == {}
def test_commit_force(dvc_repo, repo_dir):
stages = dvc_repo.add(repo_dir.FOO, no_commit=True)
assert len(stages) == 1
stage = stages[0]
def test_commit_force(tmp_dir, dvc):
tmp_dir.gen({"dir": {"file": "text1", "file2": "text2"}})
stage, = dvc.add("dir", no_commit=True)
with dvc_repo.state:
with dvc.state:
assert stage.outs[0].changed_cache()
with open(repo_dir.FOO, "a") as fobj:
fobj.write(repo_dir.FOO_CONTENTS)
tmp_dir.gen("dir/file", "file content modified")
with dvc_repo.state:
with dvc.state:
assert stage.outs[0].changed_cache()
with pytest.raises(StageCommitError):
dvc_repo.commit(stage.path)
dvc.commit(stage.path)
with dvc_repo.state:
with dvc.state:
assert stage.outs[0].changed_cache()
dvc_repo.commit(stage.path, force=True)
assert dvc_repo.status([stage.path]) == {}
dvc.commit(stage.path, force=True)
assert dvc.status([stage.path]) == {}
def test_commit_with_deps(tmp_dir, dvc, run_copy):
@@ -63,16 +60,15 @@ def test_commit_with_deps(tmp_dir, dvc, run_copy):
assert not stage.outs[0].changed_cache()
def test_commit_changed_md5(dvc_repo, repo_dir):
stages = dvc_repo.add(repo_dir.FOO, no_commit=True)
assert len(stages) == 1
stage = stages[0]
def test_commit_changed_md5(tmp_dir, dvc):
tmp_dir.gen({"file": "file content"})
stage, = dvc.add("file", no_commit=True)
stage_file_content = load_stage_file(stage.path)
stage_file_content["md5"] = "1111111111"
dump_stage_file(stage.path, stage_file_content)
with pytest.raises(StageCommitError):
dvc_repo.commit(stage.path)
dvc.commit(stage.path)
dvc_repo.commit(stage.path, force=True)
dvc.commit(stage.path, force=True)
tests/func/test_get.py
+77 -1
View File
@@ -7,12 +7,13 @@ import os
import pytest
from dvc.config import Config
from dvc.exceptions import GetDVCFileError
from dvc.exceptions import UrlNotDvcRepoError
from dvc.repo.get import GetDVCFileError, PathMissingError
from dvc.repo import Repo
from dvc.system import System
from dvc.utils import makedirs
from dvc.utils.compat import fspath
from dvc.utils import fspath_py35
from tests.utils import trees_equal
@@ -38,6 +39,36 @@ def test_get_repo_dir(erepo):
trees_equal(src, dst)
def test_get_regular_file(erepo):
src = "some_file"
dst = "some_file_imported"
src_path = os.path.join(erepo.root_dir, src)
erepo.create(src_path, "hello")
erepo.dvc.scm.add([src_path])
erepo.dvc.scm.commit("add a regular file")
Repo.get(erepo.root_dir, src, dst)
assert os.path.exists(dst)
assert os.path.isfile(dst)
assert filecmp.cmp(src_path, dst, shallow=False)
def test_get_regular_dir(erepo):
src = "some_directory"
dst = "some_directory_imported"
src_file_path = os.path.join(erepo.root_dir, src, "file.txt")
erepo.create(src_file_path, "hello")
erepo.dvc.scm.add([src_file_path])
erepo.dvc.scm.commit("add a regular dir")
Repo.get(erepo.root_dir, src, dst)
assert os.path.exists(dst)
assert os.path.isdir(dst)
trees_equal(os.path.join(erepo.root_dir, src), dst)
def test_cache_type_is_properly_overridden(erepo):
erepo.dvc.config.set(
Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE, "symlink"
@@ -77,6 +108,51 @@ def test_get_a_dvc_file(erepo):
Repo.get(erepo.root_dir, "some_file.dvc")
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
def test_get_full_dvc_path(erepo):
external_data_dir = erepo.mkdtemp()
external_data = os.path.join(external_data_dir, "ext_data")
with open(external_data, "w+") as fobj:
fobj.write("ext_data")
cur_dir = os.getcwd()
os.chdir(erepo.root_dir)
erepo.dvc.add(external_data)
erepo.dvc.scm.add(["ext_data.dvc"])
erepo.dvc.scm.commit("add external data")
os.chdir(cur_dir)
Repo.get(erepo.root_dir, external_data, "ext_data_imported")
assert os.path.isfile("ext_data_imported")
assert filecmp.cmp(external_data, "ext_data_imported", shallow=False)
def test_non_cached_output(tmp_path, erepo):
os.chdir(erepo.root_dir)
erepo.dvc.run(
outs_no_cache=["non_cached_file"], cmd="echo hello > non_cached_file"
)
erepo.dvc.scm.add(["non_cached_file", "non_cached_file.dvc"])
erepo.dvc.scm.commit("add non-cached output")
os.chdir(fspath_py35(tmp_path))
Repo.get(erepo.root_dir, "non_cached_file")
src = os.path.join(erepo.root_dir, "non_cached_file")
assert os.path.isfile("non_cached_file")
assert filecmp.cmp(src, "non_cached_file", shallow=False)
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
def test_absolute_file_outside_repo(erepo):
with pytest.raises(PathMissingError):
Repo.get(erepo.root_dir, "/root/")
def test_unknown_path(erepo):
with pytest.raises(PathMissingError):
Repo.get(erepo.root_dir, "a_non_existing_file")
@pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
def test_get_to_dir(dname, erepo):
src = erepo.FOO
tests/func/test_ignore.py
+76 -142
View File
@@ -1,186 +1,120 @@
import itertools
# encoding: utf-8
from __future__ import unicode_literals
import os
import shutil
import pytest
from dvc.exceptions import DvcIgnoreInCollectedDirError
from dvc.ignore import DvcIgnore
from dvc.ignore import DvcIgnoreDirs
from dvc.ignore import DvcIgnoreFilter
from dvc.ignore import DvcIgnorePatterns
from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns
from dvc.scm.tree import WorkingTree
from dvc.utils.compat import cast_bytes
from dvc.utils import walk_files
from dvc.utils.compat import fspath
from dvc.utils.fs import get_mtime_and_size
from tests.basic_env import TestDvc
from tests.utils import to_posixpath
class TestDvcIgnore(TestDvc):
def setUp(self):
super(TestDvcIgnore, self).setUp()
def test_ignore(tmp_dir, dvc, monkeypatch):
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
def _get_all_paths(self):
assert _files_set("dir", dvc.dvcignore) == {"dir/other"}
paths = []
for root, dirs, files in self.dvc.tree.walk(
self.dvc.root_dir, dvcignore=self.dvc.dvcignore
):
for dname in dirs:
paths.append(os.path.join(root, dname))
for fname in files:
paths.append(os.path.join(root, fname))
return paths
def test_ignore_in_child_dir(self):
ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE)
with open(ignore_file, "w") as fobj:
fobj.write("data_dir/data")
forbidden_path = os.path.join(self.dvc.root_dir, self.DATA)
all_paths = self._get_all_paths()
self.assertNotIn(forbidden_path, all_paths)
def test_ignore_in_child_dir_unicode(self):
ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE)
with open(ignore_file, "wb") as fobj:
fobj.write(cast_bytes(self.UNICODE, "utf-8"))
forbidden_path = os.path.join(self.dvc.root_dir, self.UNICODE)
all_paths = self._get_all_paths()
self.assertNotIn(forbidden_path, all_paths)
def test_ignore_in_parent_dir(self):
ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE)
with open(ignore_file, "w") as fobj:
fobj.write("data_dir/data")
os.chdir(self.DATA_DIR)
forbidden_path = os.path.join(self.dvc.root_dir, self.DATA)
all_paths = self._get_all_paths()
self.assertNotIn(forbidden_path, all_paths)
monkeypatch.chdir("dir")
assert _files_set(".", dvc.dvcignore) == {"./other"}
def test_metadata_unchanged_when_moving_ignored_file(dvc_repo, repo_dir):
new_data_path = repo_dir.DATA_SUB + "_new"
def test_ignore_unicode(tmp_dir, dvc):
tmp_dir.gen({"dir": {"other": "text"}})
# Path() doesn't handle unicode paths in Windows/Python 2
# I don't know to debug it further, waiting till Python 2 EOL
with open("dir/тест", "wb") as fd:
fd.write("проверка".encode("utf-8"))
ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE)
repo_dir.create(
ignore_file,
"\n".join(
[to_posixpath(repo_dir.DATA_SUB), to_posixpath(new_data_path)]
),
)
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест")
mtime_sig, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore)
shutil.move(repo_dir.DATA_SUB, new_data_path)
new_mtime_sig, new_size = get_mtime_and_size(
repo_dir.DATA_DIR, dvc_repo.dvcignore
)
assert new_mtime_sig == mtime_sig
assert new_size == size
assert _files_set("dir", dvc.dvcignore) == {"dir/other"}
def test_mtime_changed_when_moving_non_ignored_file(dvc_repo, repo_dir):
new_data_path = repo_dir.DATA_SUB + "_new"
mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore)
def test_rename_ignored_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
shutil.move(repo_dir.DATA_SUB, new_data_path)
new_mtime, new_size = get_mtime_and_size(
repo_dir.DATA_DIR, dvc_repo.dvcignore
)
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*")
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
assert new_mtime != mtime
assert new_size == size
shutil.move("dir/ignored", "dir/ignored_new")
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
assert new_mtime == mtime and new_size == size
def test_metadata_unchanged_on_ignored_file_deletion(dvc_repo, repo_dir):
ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE)
repo_dir.create(ignore_file, to_posixpath(repo_dir.DATA_SUB))
def test_rename_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore)
shutil.move("dir/foo", "dir/foo_new")
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
os.remove(repo_dir.DATA_SUB)
new_mtime, new_size = get_mtime_and_size(
repo_dir.DATA_DIR, dvc_repo.dvcignore
)
assert new_mtime == mtime
assert new_size == size
assert new_mtime != mtime and new_size == size
def test_metadata_changed_on_non_ignored_file_deletion(dvc_repo, repo_dir):
mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore)
def test_remove_ignored_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
os.remove(repo_dir.DATA_SUB)
new_mtime_sig, new_size = get_mtime_and_size(
repo_dir.DATA_DIR, dvc_repo.dvcignore
)
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
assert new_mtime_sig != mtime
assert new_size != size
os.remove("dir/ignored")
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
assert new_mtime == mtime and new_size == size
def test_should_raise_on_dvcignore_in_out_dir(dvc_repo, repo_dir):
ignore_file = os.path.join(repo_dir.DATA_DIR, DvcIgnore.DVCIGNORE_FILE)
repo_dir.create(ignore_file, "")
def test_remove_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
os.remove("dir/foo")
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
assert new_mtime != mtime and new_size != size
def test_dvcignore_in_out_dir(tmp_dir, dvc):
tmp_dir.gen({"dir": {"foo": "foo", DvcIgnore.DVCIGNORE_FILE: ""}})
with pytest.raises(DvcIgnoreInCollectedDirError):
dvc_repo.add(repo_dir.DATA_DIR)
dvc.add("dir")
@pytest.mark.parametrize("dname", [TestDvc.DATA_DIR, TestDvc.DATA_SUB_DIR])
def test_ignore_collecting_dvcignores(repo_dir, dname):
top_ignore_file = os.path.join(
repo_dir.root_dir, os.path.dirname(dname), DvcIgnore.DVCIGNORE_FILE
)
repo_dir.create(top_ignore_file, os.path.basename(dname))
@pytest.mark.parametrize("dname", ["dir", "dir/subdir"])
def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
tmp_dir.gen({"dir": {"subdir": {}}})
ignore_file = os.path.join(
repo_dir.root_dir, dname, DvcIgnore.DVCIGNORE_FILE
)
repo_dir.create(ignore_file, repo_dir.FOO)
top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE)
top_ignore_file.write_text(os.path.basename(dname))
assert DvcIgnoreFilter(
repo_dir.root_dir, WorkingTree(repo_dir.root_dir)
).ignores == {
ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE
ignore_file.write_text("foo")
assert dvc.dvcignore.ignores == {
DvcIgnoreDirs([".git", ".hg", ".dvc"]),
DvcIgnorePatterns(top_ignore_file, WorkingTree(repo_dir.root_dir)),
DvcIgnorePatterns(fspath(top_ignore_file), WorkingTree(dvc.root_dir)),
}
def test_ignore_on_branch(git, dvc_repo, repo_dir):
dvc_repo.add(repo_dir.DATA_DIR)
dvc_repo.scm.commit("add data dir")
def test_ignore_on_branch(tmp_dir, scm, dvc):
tmp_dir.scm_gen({"foo": "foo", "bar": "bar"}, commit="add files")
branch_name = "branch_one"
dvc_repo.scm.checkout(branch_name, create_new=True)
scm.checkout("branch", create_new=True)
tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore")
repo_dir.create(DvcIgnore.DVCIGNORE_FILE, to_posixpath(repo_dir.DATA_SUB))
dvc_repo.scm.add([DvcIgnore.DVCIGNORE_FILE])
git.index.commit("add ignore")
scm.checkout("master")
assert _files_set(".", dvc.dvcignore) == {"./foo", "./bar"}
dvc_repo.scm.checkout("master")
dvc.tree = scm.get_tree("branch")
assert _files_set(".", dvc.dvcignore) == {"./bar"}
git_tree = dvc_repo.scm.get_tree(branch_name)
branch_data_files = set(
itertools.chain.from_iterable(
[
files
for _, _, files in dvc_repo.tree.walk(
repo_dir.DATA_DIR,
dvcignore=DvcIgnoreFilter(repo_dir.root_dir, git_tree),
)
]
)
)
assert branch_data_files == {"data"}
def _files_set(root, dvcignore):
return {to_posixpath(f) for f in walk_files(root, dvcignore)}
tests/func/test_move.py
+10 -8
View File
@@ -195,19 +195,21 @@ class TestMoveFileInsideDirectory(TestDvc):
self.assertTrue(os.path.exists(dvc_fullpath))
def test_move_should_save_stage_info(dvc_repo, repo_dir):
dvc_repo.add(repo_dir.DATA_DIR)
def test_move_should_save_stage_info(tmp_dir, dvc):
tmp_dir.dvc_gen({"old_name": {"file1": "file1"}})
dvc_repo.move(repo_dir.DATA_DIR, "new_name")
dvc.move("old_name", "new_name")
assert dvc_repo.status() == {}
assert dvc.status() == {}
def test_should_move_to_dir_on_non_default_stage_file(dvc_repo, repo_dir):
def test_should_move_to_dir_on_non_default_stage_file(tmp_dir, dvc):
stage_file_name = "stage.dvc"
tmp_dir.gen({"file": "file_content"})
dvc_repo.add(repo_dir.FOO, fname=stage_file_name)
dvc.add("file", fname=stage_file_name)
os.mkdir("directory")
dvc_repo.move(repo_dir.FOO, repo_dir.DATA_DIR)
dvc.move("file", "directory")
assert os.path.exists(os.path.join(repo_dir.DATA_DIR, repo_dir.FOO))
assert os.path.exists(os.path.join("directory", "file"))
tests/func/test_run.py
+1 -1
View File
@@ -893,7 +893,7 @@ class TestShouldNotCheckoutUponCorruptedLocalHardlinkCache(TestDvc):
)
patch_run = mock.patch.object(stage, "_run", wraps=stage._run)
with self.dvc.state:
with self.dvc.lock, self.dvc.state:
with patch_checkout as mock_checkout:
with patch_run as mock_run:
stage.run()
tests/unit/remote/test_s3.py
+56
View File
@@ -0,0 +1,56 @@
import pytest
from dvc.config import ConfigError
from dvc.remote.s3 import RemoteS3
bucket_name = "bucket-name"
prefix = "some/prefix"
url = "s3://{}/{}".format(bucket_name, prefix)
@pytest.fixture(autouse=True)
def grants():
return {
"grant_read": "id=read-permission-id,id=other-read-permission-id",
"grant_read_acp": "id=read-acp-permission-id",
"grant_write_acp": "id=write-acp-permission-id",
"grant_full_control": "id=full-control-permission-id",
}
def test_init():
config = {"url": url}
remote = RemoteS3(None, config)
assert remote.path_info == url
def test_grants():
config = {
"url": url,
"grant_read": "id=read-permission-id,id=other-read-permission-id",
"grant_read_acp": "id=read-acp-permission-id",
"grant_write_acp": "id=write-acp-permission-id",
"grant_full_control": "id=full-control-permission-id",
}
remote = RemoteS3(None, config)
assert (
remote.extra_args["GrantRead"]
== "id=read-permission-id,id=other-read-permission-id"
)
assert remote.extra_args["GrantReadACP"] == "id=read-acp-permission-id"
assert remote.extra_args["GrantWriteACP"] == "id=write-acp-permission-id"
assert (
remote.extra_args["GrantFullControl"]
== "id=full-control-permission-id"
)
def test_grants_mutually_exclusive_acl_error(grants):
for grant_option, grant_value in grants.items():
config = {"url": url, "acl": "public-read", grant_option: grant_value}
with pytest.raises(ConfigError):
RemoteS3(None, config)
tests/unit/test_analytics.py
+114 -3
View File
@@ -1,6 +1,80 @@
import pytest
import mock
import platform
import json
from dvc.analytics import Analytics
from voluptuous import Schema, Any
from dvc import analytics
from dvc.cli import parse_args
from dvc.utils.compat import str, builtin_str
string = Any(str, builtin_str)
@pytest.fixture
def tmp_global_config(tmp_path):
"""
Fixture to prevent modifying the actual global config
"""
with mock.patch(
"dvc.config.Config.get_global_config_dir", return_value=str(tmp_path)
):
yield
@mock.patch("dvc.daemon._spawn")
@mock.patch("json.dump")
def test_collect_and_send_report(mock_json, mock_daemon, tmp_global_config):
analytics.collect_and_send_report()
report = mock_json.call_args[0][0]
with pytest.raises(KeyError):
report["cmd_class"]
with pytest.raises(KeyError):
report["cmd_return_code"]
args = parse_args(["add", "foo"])
return_code = 0
analytics.collect_and_send_report(args, return_code)
report = mock_json.call_args[0][0]
assert report["cmd_class"] == "CmdAdd"
assert report["cmd_return_code"] == return_code
assert mock_daemon.call_count == 2
def test_runtime_info(tmp_global_config):
schema = Schema(
{
"dvc_version": string,
"is_binary": bool,
"scm_class": Any("Git", None),
"user_id": string,
"system_info": dict,
},
required=True,
)
assert schema(analytics._runtime_info())
@mock.patch("requests.post")
def test_send(mock_post, tmp_path):
url = "https://analytics.dvc.org"
report = {"name": "dummy report"}
fname = str(tmp_path / "report")
with open(fname, "w") as fobj:
json.dump(report, fobj)
analytics.send(fname)
assert mock_post.called
assert mock_post.call_args.args[0] == url
@pytest.mark.parametrize(
@@ -13,7 +87,7 @@ from dvc.analytics import Analytics
({"analytics": "false", "unknown": "broken"}, False),
],
)
def test_is_enabled(dvc_repo, config, result, monkeypatch):
def test_is_enabled(dvc_repo, config, result, monkeypatch, tmp_global_config):
configobj = dvc_repo.config._repo_config
configobj["core"] = config
configobj.write()
@@ -21,4 +95,41 @@ def test_is_enabled(dvc_repo, config, result, monkeypatch):
# reset DVC_TEST env var, which affects `is_enabled()`
monkeypatch.delenv("DVC_TEST")
assert result == Analytics.is_enabled()
assert result == analytics.is_enabled()
def test_system_info():
schema = Schema({"os": Any("windows", "mac", "linux")}, required=True)
system = platform.system()
if system == "Windows":
schema = schema.extend(
{
"windows_version_build": int,
"windows_version_major": int,
"windows_version_minor": int,
"windows_version_service_pack": string,
}
)
if system == "Darwin":
schema = schema.extend({"mac_version": string})
if system == "Linux":
schema = schema.extend(
{
"linux_distro": string,
"linux_distro_like": string,
"linux_distro_version": string,
}
)
assert schema(analytics._system_info())
def test_find_or_create_user_id(tmp_global_config):
created = analytics._find_or_create_user_id()
found = analytics._find_or_create_user_id()
assert created == found
tests/unit/test_rwlock.py
+98
View File
@@ -0,0 +1,98 @@
from __future__ import unicode_literals
import os
import pytest
from dvc.rwlock import (
rwlock,
_edit_rwlock,
RWLockFileFormatError,
RWLockFileCorruptedError,
)
from dvc.lock import LockError
from dvc.utils.compat import fspath
from dvc.path_info import PathInfo
def test_rwlock(tmp_path):
path = fspath(tmp_path)
foo = PathInfo("foo")
with rwlock(path, "cmd1", [foo], []):
with pytest.raises(LockError):
with rwlock(path, "cmd2", [], [foo]):
pass
with rwlock(path, "cmd1", [], [foo]):
with pytest.raises(LockError):
with rwlock(path, "cmd2", [foo], []):
pass
with rwlock(path, "cmd1", [], [foo]):
with pytest.raises(LockError):
with rwlock(path, "cmd2", [], [foo]):
pass
def test_rwlock_reentrant(tmp_path):
path = fspath(tmp_path)
foo = PathInfo("foo")
with rwlock(path, "cmd1", [], [foo]):
with rwlock(path, "cmd1", [], [foo]):
pass
with _edit_rwlock(path) as lock:
assert lock == {
"read": {},
"write": {"foo": {"cmd": "cmd1", "pid": os.getpid()}},
}
with rwlock(path, "cmd", [foo], []):
with rwlock(path, "cmd", [foo], []):
pass
with _edit_rwlock(path) as lock:
assert lock == {
"read": {"foo": [{"cmd": "cmd", "pid": os.getpid()}]},
"write": {},
}
def test_rwlock_subdirs(tmp_path):
path = fspath(tmp_path)
foo = PathInfo("foo")
subfoo = PathInfo("foo/subfoo")
with rwlock(path, "cmd1", [foo], []):
with pytest.raises(LockError):
with rwlock(path, "cmd2", [], [subfoo]):
pass
with rwlock(path, "cmd1", [], [subfoo]):
with pytest.raises(LockError):
with rwlock(path, "cmd2", [foo], []):
pass
with rwlock(path, "cmd1", [], [subfoo]):
with pytest.raises(LockError):
with rwlock(path, "cmd2", [], [foo]):
pass
with rwlock(path, "cmd1", [subfoo], []):
with rwlock(path, "cmd2", [foo], []):
pass
def test_broken_rwlock(tmp_path):
dir_path = fspath(tmp_path)
path = tmp_path / "rwlock"
path.write_text('{"broken": "format"}', encoding="utf-8")
with pytest.raises(RWLockFileFormatError):
with _edit_rwlock(dir_path):
pass
path.write_text("{broken json", encoding="utf-8")
with pytest.raises(RWLockFileCorruptedError):
with _edit_rwlock(dir_path):
pass
tests/unit/test_stage.py
+4 -6
View File
@@ -93,16 +93,14 @@ def test_stage_update(mocker):
not isinstance(threading.current_thread(), threading._MainThread),
reason="Not running in the main thread.",
)
def test_stage_run_ignore_sigint(mocker):
stage = Stage(None, "path")
def test_stage_run_ignore_sigint(dvc_repo, mocker):
proc = mocker.Mock()
communicate = mocker.Mock()
proc.configure_mock(returncode=0, communicate=communicate)
popen = mocker.patch.object(subprocess, "Popen", return_value=proc)
signal_mock = mocker.patch("signal.signal")
stage._run()
dvc_repo.run(cmd="path")
assert popen.called_once()
assert communicate.called_once_with()
@@ -110,8 +108,8 @@ def test_stage_run_ignore_sigint(mocker):
assert signal.getsignal(signal.SIGINT) == signal.default_int_handler
def test_always_changed():
stage = Stage(None, "path", always_changed=True)
def test_always_changed(dvc_repo):
stage = Stage(dvc_repo, "path", always_changed=True)
stage.save()
assert stage.changed()
assert stage.status()["path"] == ["always changed"]