diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 000000000..ebe497cee --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +dvc (0.75.b1) experimental; urgency=low + + * First native Debian package version. + + -- Dmitry Petrov Sat, 07 Dec 2019 09:49:28 +0300 diff --git a/debian/compat b/debian/compat new file mode 100644 index 000000000..f599e28b8 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +10 diff --git a/debian/control b/debian/control new file mode 100644 index 000000000..2ddf61df8 --- /dev/null +++ b/debian/control @@ -0,0 +1,33 @@ +Source: dvc +Maintainer: Dmitry Petrov +Section: python +Priority: optional +Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9) +Standards-Version: 4.0.0 +Homepage: http://dvc.org +Vcs-Git: https://github.com/iterative/dvc.git +Vcs-browser: https://github.com/iterative/dvc + +Package: dvc +Architecture: all +Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2) +Recommends: python3-arrow (=0.14.0), + python3-paramiko (>=2.5.0), + python3-gssapi (>= 2.5.0), + python3-boto3 (>=1.9.86), + python3-pydrive (=1.3.1), + python3-azure-storage-blob (=12.1.0), + python3-oss2 (=2.6.1), + python3-google-cloud-storage (=1.19.0) +Description: Git for data scientists - manage your code and data together + Data Version Control or DVC is an open-source tool for data science and machine learning projects. + . + Key features: + . + 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services. + . + 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID. + . + 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built. + . + 4. Helps manage experiments with Git tags/branches and metrics tracking. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 000000000..523903a1c --- /dev/null +++ b/debian/copyright @@ -0,0 +1,11 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: DVC +Upstream-Contact: Dmitry Petrov +Source: https://github.com/iterative/dvc + +Files: * +Copyright: Dmitry Petrov +License: Apache 2.0 + On Debian systems, the full text of the Apache 2.0 + License can be found in the file + `/usr/share/common-licenses/Apache-2.0`. diff --git a/debian/rules b/debian/rules new file mode 100755 index 000000000..60ae08041 --- /dev/null +++ b/debian/rules @@ -0,0 +1,6 @@ +#!/usr/bin/make -f + +export PYBUILD_NAME=dvc +%: + dh $@ --with python3 --buildsystem=pybuild + diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 000000000..89ae9db8f --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/debian/watch b/debian/watch new file mode 100644 index 000000000..497d9f3d8 --- /dev/null +++ b/debian/watch @@ -0,0 +1,4 @@ +# please also check http://pypi.debian.net/dvc/watch +version=3 +opts=uversionmangle=s/(rc|a|b|c)/~$1/ \ +http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz))) \ No newline at end of file diff --git a/dvc/analytics.py b/dvc/analytics.py index 8b39fc31b..1ae02f60a 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -1,256 +1,167 @@ -"""Collect and send usage analytics""" -from __future__ import unicode_literals - -import errno import json import logging import os +import platform +import requests +import sys +import tempfile +import uuid + +import distro from dvc import __version__ -from dvc.utils import env2bool -from dvc.utils.compat import str +from dvc.config import Config, to_bool +from dvc.daemon import daemon +from dvc.exceptions import NotDvcRepoError +from dvc.lock import Lock, LockError +from dvc.repo import Repo +from dvc.scm import SCM +from dvc.utils import env2bool, is_binary, makedirs +from dvc.utils.compat import str, FileNotFoundError logger = logging.getLogger(__name__) -class Analytics(object): - """Class for collecting and sending usage analytics. - - Args: - info (dict): optional existing analytics report. +def collect_and_send_report(args=None, return_code=None): """ + Collect information from the runtime/environment and the command + being executed into a report and send it over the network. - URL = "https://analytics.dvc.org" - TIMEOUT_POST = 5 + To prevent analytics from blocking the execution of the main thread, + sending the report is done in a separate process. - USER_ID_FILE = "user_id" + The inter-process communication happens through a file containing the + report as a JSON, where the _collector_ generates it and the _sender_ + removes it after sending it. + """ + report = _runtime_info() - PARAM_DVC_VERSION = "dvc_version" - PARAM_USER_ID = "user_id" - PARAM_SYSTEM_INFO = "system_info" + # Include command execution information on the report only when available. + if args and hasattr(args, "func"): + report.update({"cmd_class": args.func.__name__}) - PARAM_OS = "os" + if return_code is not None: + report.update({"cmd_return_code": return_code}) - PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major" - PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor" - PARAM_WINDOWS_VERSION_BUILD = "windows_version_build" - PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack" + with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: + json.dump(report, fobj) + daemon(["analytics", fobj.name]) - PARAM_MAC_VERSION = "mac_version" - PARAM_LINUX_DISTRO = "linux_distro" - PARAM_LINUX_DISTRO_VERSION = "linux_distro_version" - PARAM_LINUX_DISTRO_LIKE = "linux_distro_like" +def is_enabled(): + if env2bool("DVC_TEST"): + return False - PARAM_SCM_CLASS = "scm_class" - PARAM_IS_BINARY = "is_binary" - PARAM_CMD_CLASS = "cmd_class" - PARAM_CMD_RETURN_CODE = "cmd_return_code" + enabled = to_bool( + Config(validate=False) + .config.get(Config.SECTION_CORE, {}) + .get(Config.SECTION_CORE_ANALYTICS, "true") + ) - def __init__(self, info=None): - from dvc.config import Config - from dvc.lock import Lock + logger.debug("Analytics is {}abled.".format("en" if enabled else "dis")) - if info is None: - info = {} + return enabled - self.info = info - cdir = Config.get_global_config_dir() - try: - os.makedirs(cdir) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise +def send(report): + """ + Side effect: Removes the report after sending it. - self.user_id_file = os.path.join(cdir, self.USER_ID_FILE) - self.user_id_file_lock = Lock(self.user_id_file + ".lock") + The report is generated and stored in a temporary file, see: + `collect_and_send_report`. Sending happens on another process, + thus, the need of removing such file afterwards. + """ + url = "https://analytics.dvc.org" + headers = {"content-type": "application/json"} - @staticmethod - def load(path): - """Loads analytics report from json file specified by path. + with open(report, "rb") as fobj: + requests.post(url, data=fobj, headers=headers, timeout=5) - Args: - path (str): path to json file with analytics report. - """ - with open(path, "r") as fobj: - analytics = Analytics(info=json.load(fobj)) - os.unlink(path) - return analytics + os.remove(report) - def _write_user_id(self): - import uuid - with open(self.user_id_file, "w+") as fobj: - user_id = str(uuid.uuid4()) - info = {self.PARAM_USER_ID: user_id} - json.dump(info, fobj) +def _scm_in_use(): + try: + scm = SCM(root_dir=Repo.find_root()) + return type(scm).__name__ + except NotDvcRepoError: + pass + + +def _runtime_info(): + """ + Gather information from the environment where DVC runs to fill a report. + """ + return { + "dvc_version": __version__, + "is_binary": is_binary(), + "scm_class": _scm_in_use(), + "system_info": _system_info(), + "user_id": _find_or_create_user_id(), + } + + +def _system_info(): + system = platform.system() + + if system == "Windows": + version = sys.getwindowsversion() + + return { + "os": "windows", + "windows_version_build": version.build, + "windows_version_major": version.major, + "windows_version_minor": version.minor, + "windows_version_service_pack": version.service_pack, + } + + if system == "Darwin": + return {"os": "mac", "mac_version": platform.mac_ver()[0]} + + if system == "Linux": + return { + "os": "linux", + "linux_distro": distro.id(), + "linux_distro_like": distro.like(), + "linux_distro_version": distro.version(), + } + + # We don't collect data for any other system. + raise NotImplementedError + + +def _find_or_create_user_id(): + """ + The user's ID is stored on a file under the global config directory. + + The file should contain a JSON with a "user_id" key: + + {"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"} + + IDs are generated randomly with UUID. + """ + config_dir = Config.get_global_config_dir() + fname = os.path.join(config_dir, "user_id") + lockfile = os.path.join(config_dir, "user_id.lock") + + # Since the `fname` and `lockfile` are under the global config, + # we need to make sure such directory exist already. + makedirs(config_dir, exist_ok=True) + + try: + with Lock(lockfile): + try: + with open(fname, "r") as fobj: + user_id = json.load(fobj)["user_id"] + + except (FileNotFoundError, ValueError, KeyError): + user_id = str(uuid.uuid4()) + + with open(fname, "w") as fobj: + json.dump({"user_id": user_id}, fobj) + return user_id - def _read_user_id(self): - if not os.path.exists(self.user_id_file): - return None - - with open(self.user_id_file, "r") as fobj: - try: - info = json.load(fobj) - except ValueError as exc: - logger.debug("Failed to load user_id: {}".format(exc)) - return None - - return info[self.PARAM_USER_ID] - - def _get_user_id(self): - from dvc.lock import LockError - - try: - with self.user_id_file_lock: - user_id = self._read_user_id() - if user_id is None: - user_id = self._write_user_id() - return user_id - except LockError: - msg = "Failed to acquire '{}'" - logger.debug(msg.format(self.user_id_file_lock.lockfile)) - - def _collect_windows(self): - import sys - - version = sys.getwindowsversion() # pylint: disable=no-member - info = {} - info[self.PARAM_OS] = "windows" - info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major - info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor - info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build - info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack - return info - - def _collect_darwin(self): - import platform - - info = {} - info[self.PARAM_OS] = "mac" - info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0] - return info - - def _collect_linux(self): - import distro - - info = {} - info[self.PARAM_OS] = "linux" - info[self.PARAM_LINUX_DISTRO] = distro.id() - info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version() - info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like() - return info - - def _collect_system_info(self): - import platform - - system = platform.system() - - if system == "Windows": - return self._collect_windows() - - if system == "Darwin": - return self._collect_darwin() - - if system == "Linux": - return self._collect_linux() - - raise NotImplementedError - - def collect(self): - """Collect analytics report.""" - from dvc.scm import SCM - from dvc.utils import is_binary - from dvc.repo import Repo - from dvc.exceptions import NotDvcRepoError - - self.info[self.PARAM_DVC_VERSION] = __version__ - self.info[self.PARAM_IS_BINARY] = is_binary() - self.info[self.PARAM_USER_ID] = self._get_user_id() - - self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info() - - try: - scm = SCM(root_dir=Repo.find_root()) - self.info[self.PARAM_SCM_CLASS] = type(scm).__name__ - except NotDvcRepoError: - pass - - def collect_cmd(self, args, ret): - """Collect analytics info from a CLI command.""" - from dvc.command.daemon import CmdDaemonAnalytics - - assert isinstance(ret, int) or ret is None - - if ret is not None: - self.info[self.PARAM_CMD_RETURN_CODE] = ret - - if args is not None and hasattr(args, "func"): - assert args.func != CmdDaemonAnalytics - self.info[self.PARAM_CMD_CLASS] = args.func.__name__ - - def dump(self): - """Save analytics report to a temporary file. - - Returns: - str: path to the temporary file that contains the analytics report. - """ - import tempfile - - with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: - json.dump(self.info, fobj) - return fobj.name - - @staticmethod - def is_enabled(cmd=None): - from dvc.config import Config, to_bool - from dvc.command.daemon import CmdDaemonBase - - if env2bool("DVC_TEST"): - return False - - if isinstance(cmd, CmdDaemonBase): - return False - - core = Config(validate=False).config.get(Config.SECTION_CORE, {}) - enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true")) - logger.debug( - "Analytics is {}.".format("enabled" if enabled else "disabled") - ) - return enabled - - @staticmethod - def send_cmd(cmd, args, ret): - """Collect and send analytics for CLI command. - - Args: - args (list): parsed args for the CLI command. - ret (int): return value of the CLI command. - """ - from dvc.daemon import daemon - - if not Analytics.is_enabled(cmd): - return - - analytics = Analytics() - analytics.collect_cmd(args, ret) - daemon(["analytics", analytics.dump()]) - - def send(self): - """Collect and send analytics.""" - import requests - - if not self.is_enabled(): - return - - self.collect() - - logger.debug("Sending analytics: {}".format(self.info)) - - try: - requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST) - except requests.exceptions.RequestException as exc: - logger.debug("Failed to send analytics: {}".format(str(exc))) + except LockError: + logger.debug("Failed to acquire {lockfile}".format(lockfile=lockfile)) diff --git a/dvc/command/base.py b/dvc/command/base.py index 56a9aed23..d8ccb2c8d 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -2,8 +2,6 @@ from __future__ import unicode_literals import logging -import colorama - logger = logging.getLogger(__name__) @@ -24,15 +22,13 @@ def fix_subparsers(subparsers): def append_doc_link(help_message, path): + from dvc.utils import format_link + if not path: return help_message doc_base = "https://man.dvc.org/" - return "{message}\nDocumentation: <{blue}{base}{path}{nc}>".format( - message=help_message, - base=doc_base, - path=path, - blue=colorama.Fore.CYAN, - nc=colorama.Fore.RESET, + return "{message}\nDocumentation: {link}".format( + message=help_message, link=format_link(doc_base + path) ) @@ -44,7 +40,8 @@ class CmdBase(object): self.repo = Repo() self.config = self.repo.config self.args = args - updater = Updater(self.repo.dvc_dir) + hardlink_lock = self.config.config["core"].get("hardlink_lock", False) + updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock) updater.check() @property diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index 2b60bded4..b391e6cee 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -12,11 +12,16 @@ class CmdDaemonUpdater(CmdDaemonBase): def run(self): import os from dvc.repo import Repo + from dvc.config import Config from dvc.updater import Updater root_dir = Repo.find_root() dvc_dir = os.path.join(root_dir, Repo.DVC_DIR) - updater = Updater(dvc_dir) + config = Config(dvc_dir, validate=False) + hardlink_lock = config.config.get("core", {}).get( + "hardlink_lock", False + ) + updater = Updater(dvc_dir, hardlink_lock=hardlink_lock) updater.fetch(detach=False) return 0 @@ -24,10 +29,9 @@ class CmdDaemonUpdater(CmdDaemonBase): class CmdDaemonAnalytics(CmdDaemonBase): def run(self): - from dvc.analytics import Analytics + from dvc import analytics - analytics = Analytics.load(self.args.target) - analytics.send() + analytics.send(self.args.target) return 0 diff --git a/dvc/command/get.py b/dvc/command/get.py index 77235ecef..e3172782d 100644 --- a/dvc/command/get.py +++ b/dvc/command/get.py @@ -33,7 +33,7 @@ class CmdGet(CmdBaseNoRepo): def add_parser(subparsers, parent_parser): - GET_HELP = "Download data from DVC repository." + GET_HELP = "Download/copy files or directories from DVC repository." get_parser = subparsers.add_parser( "get", parents=[parent_parser], @@ -44,9 +44,14 @@ def add_parser(subparsers, parent_parser): get_parser.add_argument( "url", help="URL of Git repository with DVC project to download from." ) - get_parser.add_argument("path", help="Path to data within DVC repository.") get_parser.add_argument( - "-o", "--out", nargs="?", help="Destination path to put data to." + "path", help="Path to a file or directory within a DVC repository." + ) + get_parser.add_argument( + "-o", + "--out", + nargs="?", + help="Destination path to copy/download files to.", ) get_parser.add_argument( "--rev", nargs="?", help="DVC repository git revision." diff --git a/dvc/command/imp_url.py b/dvc/command/imp_url.py index 46b24f8aa..e072bc1e3 100644 --- a/dvc/command/imp_url.py +++ b/dvc/command/imp_url.py @@ -44,12 +44,17 @@ def add_parser(subparsers, parent_parser): "url", help="Supported urls:\n" "/path/to/file\n" + "/path/to/directory\n" "C:\\\\path\\to\\file\n" + "C:\\\\path\\to\\directory\n" "https://example.com/path/to/file\n" "s3://bucket/path/to/file\n" + "s3://bucket/path/to/directory\n" "gs://bucket/path/to/file\n" + "gs://bucket/path/to/directory\n" "hdfs://example.com/path/to/file\n" "ssh://example.com:/path/to/file\n" + "ssh://example.com:/path/to/directory\n" "remote://myremote/path/to/file (see `dvc remote`)", ) import_parser.add_argument( diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 8c6dbc3ef..683c040f4 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -19,6 +19,10 @@ class CmdRemoteConfig(CmdConfig): class CmdRemoteAdd(CmdRemoteConfig): def run(self): + if self.args.default: + logger.info( + "Setting '{}' as a default remote.".format(self.args.name) + ) self.remote_config.add( self.args.name, self.args.url, diff --git a/dvc/config.py b/dvc/config.py index bee6fb8ef..0854ae930 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -13,8 +13,7 @@ from voluptuous import All, Any, Lower, Range, Coerce, Match from dvc.exceptions import DvcException from dvc.exceptions import NotDvcRepoError -from dvc.utils.compat import open -from dvc.utils.compat import str +from dvc.utils.compat import open, str logger = logging.getLogger(__name__) @@ -126,6 +125,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes SECTION_CORE_INTERACTIVE = "interactive" SECTION_CORE_ANALYTICS = "analytics" SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs" + SECTION_CORE_HARDLINK_LOCK = "hardlink_lock" SECTION_CACHE = "cache" SECTION_CACHE_DIR = "dir" @@ -161,6 +161,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional(SECTION_CORE_INTERACTIVE, default=False): Bool, Optional(SECTION_CORE_ANALYTICS, default=True): Bool, SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)), + Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool, } # aws specific options @@ -172,6 +173,10 @@ class Config(object): # pylint: disable=too-many-instance-attributes SECTION_AWS_USE_SSL = "use_ssl" SECTION_AWS_SSE = "sse" SECTION_AWS_ACL = "acl" + SECTION_AWS_GRANT_READ = "grant_read" + SECTION_AWS_GRANT_READ_ACP = "grant_read_acp" + SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp" + SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control" # gcp specific options SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH @@ -211,6 +216,10 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional(SECTION_AWS_USE_SSL, default=True): Bool, SECTION_AWS_SSE: str, SECTION_AWS_ACL: str, + SECTION_AWS_GRANT_READ: str, + SECTION_AWS_GRANT_READ_ACP: str, + SECTION_AWS_GRANT_WRITE_ACP: str, + SECTION_AWS_GRANT_FULL_CONTROL: str, SECTION_GCP_PROJECTNAME: str, SECTION_CACHE_TYPE: supported_cache_type, Optional(SECTION_CACHE_PROTECTED, default=False): Bool, diff --git a/dvc/exceptions.py b/dvc/exceptions.py index e5fb6f00d..1beb8d6e1 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -47,7 +47,8 @@ class OutputDuplicationError(DvcException): ) msg = ( "file/directory '{}' is specified as an output in more than one " - "stage: {}" + "stage: {}\n" + "This is not allowed. Consider using a different output name." ).format(output, "\n ".join(stages)) super(OutputDuplicationError, self).__init__(msg) @@ -283,14 +284,6 @@ class UrlNotDvcRepoError(DvcException): ) -class GetDVCFileError(DvcException): - def __init__(self): - super(GetDVCFileError, self).__init__( - "the given path is a DVC-file, you must specify a data file " - "or a directory" - ) - - class GitHookAlreadyExistsError(DvcException): def __init__(self, hook_name): super(GitHookAlreadyExistsError, self).__init__( diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 8f53da4ef..758d8f421 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -7,8 +7,9 @@ from distutils.dir_util import copy_tree from funcy import retry -from dvc.config import NoRemoteError +from dvc.config import NoRemoteError, ConfigError from dvc.exceptions import NoRemoteInExternalRepoError +from dvc.remote import RemoteConfig from dvc.exceptions import NoOutputInExternalRepoError from dvc.exceptions import OutputNotFoundError from dvc.utils.fs import remove @@ -69,6 +70,22 @@ def _external_repo(url=None, rev=None, cache_dir=None): repo = Repo(new_path) try: + # check if the URL is local and no default remote is present + # add default remote pointing to the original repo's cache location + if os.path.isdir(url): + rconfig = RemoteConfig(repo.config) + if not _default_remote_set(rconfig): + original_repo = Repo(url) + try: + rconfig.add( + "auto-generated-upstream", + original_repo.cache.local.cache_dir, + default=True, + level=Config.LEVEL_LOCAL, + ) + finally: + original_repo.close() + if cache_dir is not None: cache_config = CacheConfig(repo.config) cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL) @@ -113,3 +130,19 @@ def _clone_repo(url, path): git = Git.clone(url, path) git.close() + + +def _default_remote_set(rconfig): + """ + Checks if default remote config is present. + Args: + rconfig: a remote config + + Returns: + True if the default remote config is set, else False + """ + try: + rconfig.get_default() + return True + except ConfigError: + return False diff --git a/dvc/lock.py b/dvc/lock.py index 147118c0f..a8fc08c01 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -6,12 +6,12 @@ import os import time from datetime import timedelta -from funcy.py3 import lkeep +import zc.lockfile from dvc.exceptions import DvcException -from dvc.utils import makedirs -from dvc.utils.compat import is_py3 - +from dvc.utils import format_link +from dvc.utils.compat import is_py3, is_py2 +from dvc.progress import Tqdm DEFAULT_TIMEOUT = 5 @@ -26,10 +26,60 @@ class LockError(DvcException): """Thrown when unable to acquire the lock for dvc repo.""" +class Lock(object): + """Class for dvc repo lock. + + Uses zc.lockfile as backend. + """ + + def __init__(self, lockfile, friendly=False, **kwargs): + self._friendly = friendly + self.lockfile = lockfile + self._lock = None + + @property + def files(self): + return [self.lockfile] + + def _do_lock(self): + try: + with Tqdm( + bar_format="{desc}", + disable=not self._friendly, + desc=( + "If DVC froze, see `hardlink_lock` in {}".format( + format_link("man.dvc.org/config#core") + ) + ), + ): + self._lock = zc.lockfile.LockFile(self.lockfile) + except zc.lockfile.LockError: + raise LockError(FAILED_TO_LOCK_MESSAGE) + + def lock(self): + try: + self._do_lock() + return + except LockError: + time.sleep(DEFAULT_TIMEOUT) + + self._do_lock() + + def unlock(self): + self._lock.close() + self._lock = None + + def __enter__(self): + self.lock() + + def __exit__(self, typ, value, tbck): + self.unlock() + + if is_py3: import flufl.lock - class Lock(flufl.lock.Lock): + class HardlinkLock(flufl.lock.Lock): """Class for dvc repo lock. Args: @@ -38,12 +88,10 @@ if is_py3: tmp_dir (str): a directory to store claim files. """ - def __init__(self, lockfile, tmp_dir=None): + def __init__(self, lockfile, tmp_dir=None, **kwargs): import socket self._tmp_dir = tmp_dir - if self._tmp_dir is not None: - makedirs(self._tmp_dir, exist_ok=True) # NOTE: this is basically Lock.__init__ copy-paste, except that # instead of using `socket.getfqdn()` we use `socket.gethostname()` @@ -69,10 +117,6 @@ if is_py3: def lockfile(self): return self._lockfile - @property - def files(self): - return lkeep([self._lockfile, self._tmp_dir]) - def lock(self): try: super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT)) @@ -101,44 +145,14 @@ if is_py3: pass -else: - import zc.lockfile +def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False): + if hardlink_lock and is_py2: + raise DvcException( + "Hardlink locks are not supported on Python <3.5. " + "See `hardlink_lock` in {}".format( + format_link("man.dvc.org/config#core") + ) + ) - class Lock(object): - """Class for dvc repo lock. - - Uses zc.lockfile as backend. - """ - - def __init__(self, lockfile, tmp_dir=None): - self.lockfile = lockfile - self._lock = None - - @property - def files(self): - return [self.lockfile] - - def _do_lock(self): - try: - self._lock = zc.lockfile.LockFile(self.lockfile) - except zc.lockfile.LockError: - raise LockError(FAILED_TO_LOCK_MESSAGE) - - def lock(self): - try: - self._do_lock() - return - except LockError: - time.sleep(DEFAULT_TIMEOUT) - - self._do_lock() - - def unlock(self): - self._lock.close() - self._lock = None - - def __enter__(self): - self.lock() - - def __exit__(self, typ, value, tbck): - self.unlock() + cls = HardlinkLock if hardlink_lock else Lock + return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly) diff --git a/dvc/main.py b/dvc/main.py index 660f9420d..86e1a9e0d 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import logging -from dvc.analytics import Analytics +from dvc import analytics from dvc.cli import parse_args from dvc.config import ConfigError from dvc.exceptions import DvcParserError @@ -81,6 +81,7 @@ def main(argv=None): if ret != 0: logger.info(FOOTER) - Analytics().send_cmd(cmd, args, ret) + if analytics.is_enabled(): + analytics.collect_and_send_report(args, ret) return ret diff --git a/dvc/path_info.py b/dvc/path_info.py index 860241778..773ae7dcf 100644 --- a/dvc/path_info.py +++ b/dvc/path_info.py @@ -23,7 +23,16 @@ if is_py2: fs_encoding = "utf-8" -class PathInfo(pathlib.PurePath): +class _BasePath(object): + def overlaps(self, other): + if isinstance(other, basestring): + other = self.__class__(other) + elif self.__class__ != other.__class__: + return False + return self == other or self.isin(other) or other.isin(self) + + +class PathInfo(pathlib.PurePath, _BasePath): # Use __slots__ in PathInfo objects following PurePath implementation. # This makes objects smaller and speeds up attribute access. # We don't add any fields so it's empty. @@ -138,7 +147,7 @@ class _URLPathParents(object): return "<{}.parents>".format(self.src) -class URLInfo(object): +class URLInfo(_BasePath): DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22, "hdfs": 0} def __init__(self, url): diff --git a/dvc/remote/config.py b/dvc/remote/config.py index dccad52ba..bbda7fc44 100644 --- a/dvc/remote/config.py +++ b/dvc/remote/config.py @@ -105,7 +105,6 @@ class RemoteConfig(object): level=level, ) if default: - logger.info("Setting '{}' as a default remote.".format(name)) self.config.set( Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 8dd2bdf48..6366fc426 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -5,10 +5,9 @@ import logging import os import threading -from botocore.exceptions import ClientError from funcy import cached_property, wrap_prop -from dvc.config import Config +from dvc.config import Config, ConfigError from dvc.exceptions import DvcException from dvc.exceptions import ETagMismatchError from dvc.path_info import CloudURLInfo @@ -54,6 +53,8 @@ class RemoteS3(RemoteBASE): if self.acl: self.extra_args["ACL"] = self.acl + self._append_aws_grants_to_extra_args(config) + shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH) if shared_creds: os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds) @@ -209,6 +210,8 @@ class RemoteS3(RemoteBASE): return self._list_paths(self.path_info) def isfile(self, path_info): + from botocore.exceptions import ClientError + if path_info.path.endswith("/"): return False @@ -301,3 +304,34 @@ class RemoteS3(RemoteBASE): continue yield path_info.replace(path=fname) + + def _append_aws_grants_to_extra_args(self, config): + # Keys for extra_args can be one of the following list: + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS + """ + ALLOWED_UPLOAD_ARGS = [ + 'ACL', 'CacheControl', 'ContentDisposition', 'ContentEncoding', + 'ContentLanguage', 'ContentType', 'Expires', 'GrantFullControl', + 'GrantRead', 'GrantReadACP', 'GrantWriteACP', 'Metadata', + 'RequestPayer', 'ServerSideEncryption', 'StorageClass', + 'SSECustomerAlgorithm', 'SSECustomerKey', 'SSECustomerKeyMD5', + 'SSEKMSKeyId', 'WebsiteRedirectLocation' + ] + """ + + grants = { + Config.SECTION_AWS_GRANT_FULL_CONTROL: "GrantFullControl", + Config.SECTION_AWS_GRANT_READ: "GrantRead", + Config.SECTION_AWS_GRANT_READ_ACP: "GrantReadACP", + Config.SECTION_AWS_GRANT_WRITE_ACP: "GrantWriteACP", + } + + for grant_option, extra_args_key in grants.items(): + if config.get(grant_option, ""): + if self.acl: + raise ConfigError( + "`acl` and `grant_*` AWS S3 config options " + "are mutually exclusive" + ) + + self.extra_args[extra_args_key] = config.get(grant_option) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index ac013313b..8a0afc2d3 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -69,13 +69,14 @@ class Repo(object): def __init__(self, root_dir=None): from dvc.state import State - from dvc.lock import Lock + from dvc.lock import make_lock from dvc.scm import SCM from dvc.cache import Cache from dvc.data_cloud import DataCloud from dvc.repo.metrics import Metrics from dvc.scm.tree import WorkingTree from dvc.repo.tag import Tag + from dvc.utils import makedirs root_dir = self.find_root(root_dir) @@ -88,10 +89,17 @@ class Repo(object): self.tree = WorkingTree(self.root_dir) - self.lock = Lock( + self.tmp_dir = os.path.join(self.dvc_dir, "tmp") + makedirs(self.tmp_dir, exist_ok=True) + + hardlink_lock = self.config.config["core"].get("hardlink_lock", False) + self.lock = make_lock( os.path.join(self.dvc_dir, "lock"), tmp_dir=os.path.join(self.dvc_dir, "tmp"), + hardlink_lock=hardlink_lock, + friendly=True, ) + # NOTE: storing state and link_state in the repository itself to avoid # any possible state corruption in 'shared cache dir' scenario. self.state = State(self, self.config.config) @@ -162,9 +170,8 @@ class Repo(object): flist = ( [self.config.config_local_file, updater.updater_file] + + [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir] + self.state.files - + self.lock.files - + updater.lock.files ) if path_isin(self.cache.local.cache_dir, self.root_dir): @@ -349,11 +356,7 @@ class Repo(object): continue for out in outs: - if ( - out == dep.path_info - or dep.path_info.isin(out) - or out.isin(dep.path_info) - ): + if out.overlaps(dep.path_info): dep_stage = outs[out].stage dep_node = relpath(dep_stage.path, self.root_dir) G.add_node(dep_node, stage=dep_stage) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 21d01ed7e..60c4c2e4a 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -1,22 +1,42 @@ import logging import os +import shutil +from dvc.utils.compat import FileNotFoundError import shortuuid -from dvc.exceptions import GetDVCFileError -from dvc.exceptions import NotDvcRepoError -from dvc.exceptions import OutputNotFoundError -from dvc.exceptions import UrlNotDvcRepoError +from dvc.exceptions import ( + DvcException, + NotDvcRepoError, + OutputNotFoundError, + UrlNotDvcRepoError, +) from dvc.external_repo import external_repo from dvc.path_info import PathInfo from dvc.stage import Stage -from dvc.state import StateNoop from dvc.utils import resolve_output from dvc.utils.fs import remove logger = logging.getLogger(__name__) +class GetDVCFileError(DvcException): + def __init__(self): + super(GetDVCFileError, self).__init__( + "the given path is a DVC-file, you must specify a data file " + "or a directory" + ) + + +class PathMissingError(DvcException): + def __init__(self, path, repo): + msg = ( + "The path '{}' does not exist in the target repository '{}'" + " neighther as an output nor a git-handled file." + ) + super(PathMissingError, self).__init__(msg.format(path, repo)) + + @staticmethod def get(url, path, out=None, rev=None): out = resolve_output(path, out) @@ -33,11 +53,6 @@ def get(url, path, out=None, rev=None): tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid())) try: with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo: - # Note: we need to replace state, because in case of getting DVC - # dependency on CIFS or NFS filesystems, sqlite-based state - # will be unable to obtain lock - repo.state = StateNoop() - # Try any links possible to avoid data duplication. # # Not using symlink, because we need to remove cache after we are @@ -49,16 +64,40 @@ def get(url, path, out=None, rev=None): # the same cache file might be used a few times in a directory. repo.cache.local.cache_types = ["reflink", "hardlink", "copy"] - o = repo.find_out_by_relpath(path) - with repo.state: - repo.cloud.pull(o.get_used_cache()) - o.path_info = PathInfo(os.path.abspath(out)) - with o.repo.state: - o.checkout() + try: + output = repo.find_out_by_relpath(path) + except OutputNotFoundError: + output = None + if output and output.use_cache: + _get_cached(repo, output, out) + else: + # Either an uncached out with absolute path or a user error + if os.path.isabs(path): + raise FileNotFoundError + + _copy(os.path.join(repo.root_dir, path), out) + + except (OutputNotFoundError, FileNotFoundError): + raise PathMissingError(path, url) except NotDvcRepoError: raise UrlNotDvcRepoError(url) - except OutputNotFoundError: - raise OutputNotFoundError(path) finally: remove(tmp_dir) + + +def _get_cached(repo, output, out): + with repo.state: + repo.cloud.pull(output.get_used_cache()) + output.path_info = PathInfo(os.path.abspath(out)) + failed = output.checkout() + # This might happen when pull haven't really pulled all the files + if failed: + raise FileNotFoundError + + +def _copy(src, dst): + if os.path.isdir(src): + shutil.copytree(src, dst) + else: + shutil.copy2(src, dst) diff --git a/dvc/repo/init.py b/dvc/repo/init.py index e06307e0a..5df1d0f1f 100644 --- a/dvc/repo/init.py +++ b/dvc/repo/init.py @@ -3,7 +3,7 @@ import os import colorama -from dvc.analytics import Analytics +from dvc import analytics from dvc.config import Config from dvc.exceptions import InitError from dvc.repo import Repo @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) def _welcome_message(): - if Analytics.is_enabled(): + if analytics.is_enabled(): logger.info( boxify( "DVC has enabled anonymous aggregate usage analytics.\n" diff --git a/dvc/rwlock.py b/dvc/rwlock.py new file mode 100644 index 000000000..29b7bce88 --- /dev/null +++ b/dvc/rwlock.py @@ -0,0 +1,191 @@ +from __future__ import unicode_literals + +import os +import json + +from collections import defaultdict +from contextlib import contextmanager + +from voluptuous import Schema, Invalid +from funcy.py3 import lmap, lfilter, lmapcat + +from .exceptions import DvcException +from .lock import LockError +from .utils.compat import ( + convert_to_unicode, + FileNotFoundError, + JSONDecodeError, + str, +) +from .utils.fs import relpath + +INFO_SCHEMA = {"pid": int, "cmd": str} + +SCHEMA = Schema({"write": {str: INFO_SCHEMA}, "read": {str: [INFO_SCHEMA]}}) + + +class RWLockFileCorruptedError(DvcException): + def __init__(self, path, cause): + super(RWLockFileCorruptedError, self).__init__( + "Unable to read RWLock-file '{}'. JSON structure is " + "corrupted".format(relpath(path)), + cause=cause, + ) + + +class RWLockFileFormatError(DvcException): + def __init__(self, path, cause): + super(RWLockFileFormatError, self).__init__( + "RWLock-file '{}' format error.".format(relpath(path)), cause=cause + ) + + +@contextmanager +def _edit_rwlock(lock_dir): + path = os.path.join(lock_dir, "rwlock") + try: + with open(path, "r") as fobj: + lock = json.load(fobj) + lock = SCHEMA(convert_to_unicode(lock)) + except FileNotFoundError: + lock = SCHEMA({}) + except JSONDecodeError as exc: + raise RWLockFileCorruptedError(path, cause=exc) + except Invalid as exc: + raise RWLockFileFormatError(path, cause=exc) + lock = defaultdict(dict, lock) + lock["read"] = defaultdict(list, lock["read"]) + lock["write"] = defaultdict(dict, lock["write"]) + yield lock + with open(path, "w+") as fobj: + json.dump(lock, fobj) + + +def _infos_to_str(infos): + return "\n".join( + " (PID {}): {}".format(info["pid"], info["cmd"]) for info in infos + ) + + +def _check_no_writers(lock, info, path_infos): + for path_info in path_infos: + blocking_urls = lfilter(path_info.overlaps, lock["write"]) + if not blocking_urls: + continue + + writers = lmap(lock["write"].get, blocking_urls) + writers = lfilter(lambda i: info != i, writers) + if not writers: + continue + + raise LockError( + "'{}' is busy, it is being written to by:\n{}".format( + str(path_info), _infos_to_str(writers) + ) + ) + + +def _check_no_readers(lock, info, path_infos): + for path_info in path_infos: + blocking_urls = lfilter(path_info.overlaps, lock["read"]) + if not blocking_urls: + continue + + readers = lmapcat(lock["read"].get, blocking_urls) + readers = lfilter(lambda i: info != i, readers) + if not readers: + continue + + raise LockError( + "'{}' is busy, it is being read by:\n{}".format( + str(path_info), _infos_to_str(readers) + ) + ) + + +def _acquire_read(lock, info, path_infos): + changes = [] + + for path_info in path_infos: + url = path_info.url + + readers = lock["read"][url] + if info in readers: + continue + + changes.append(url) + readers.append(info) + + return changes + + +def _acquire_write(lock, info, path_infos): + changes = [] + + for path_info in path_infos: + url = path_info.url + + if lock["write"][url] == info: + continue + + changes.append(url) + lock["write"][url] = info + + return changes + + +def _release_write(lock, info, changes): + for url in changes: + assert "write" in lock + assert url in lock["write"] + assert lock["write"][url] == info + del lock["write"][url] + if not lock["write"]: + del lock["write"] + + +def _release_read(lock, info, changes): + for url in changes: + assert "read" in lock + assert url in lock["read"] + assert info in lock["read"][url] + lock["read"][url].remove(info) + if not lock["read"][url]: + del lock["read"][url] + if not lock["read"]: + del lock["read"] + + +@contextmanager +def rwlock(tmp_dir, cmd, read, write): + """Create non-thread-safe RWLock for PathInfos. + + Args: + tmp_dir (str): existing directory where to create the rwlock file. + cmd (str): command that will be working on these PathInfos. + read ([PathInfo]): PathInfos that are going to be read. + write ([PathInfo]): PathInfos that are going to be written. + + Raises: + LockError: raised if PathInfo we want to read is being written to by + another command or if PathInfo we want to write is being written + to or read from by another command. + RWLockFileCorruptedError: raised if rwlock file is not a valid JSON. + RWLockFileFormatError: raised if rwlock file is a valid JSON, but + has internal format that doesn't pass our schema validation. + """ + info = {"pid": os.getpid(), "cmd": cmd} + + with _edit_rwlock(tmp_dir) as lock: + _check_no_writers(lock, info, read + write) + _check_no_readers(lock, info, write) + + rchanges = _acquire_read(lock, info, read) + wchanges = _acquire_write(lock, info, write) + + try: + yield + finally: + with _edit_rwlock(tmp_dir) as lock: + _release_write(lock, info, wchanges) + _release_read(lock, info, rchanges) diff --git a/dvc/stage.py b/dvc/stage.py index 977347a0e..236a5f2d3 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -7,7 +7,10 @@ import re import signal import subprocess import threading + +from functools import wraps from itertools import chain +from funcy import decorator from voluptuous import Any, Schema, MultipleInvalid @@ -129,6 +132,52 @@ class MissingDataSource(DvcException): super(MissingDataSource, self).__init__(msg) +@decorator +def rwlocked(call, read=None, write=None): + import sys + from dvc.rwlock import rwlock + from dvc.dependency.repo import DependencyREPO + + if read is None: + read = [] + + if write is None: + write = [] + + stage = call._args[0] + + def _chain(names): + return [ + item.path_info + for attr in names + for item in getattr(stage, attr) + # There is no need to lock DependencyREPO deps, as there is no + # corresponding OutputREPO, so we can't even write it. + if not isinstance(item, DependencyREPO) + ] + + cmd = " ".join(sys.argv) + + with rwlock(stage.repo.tmp_dir, cmd, _chain(read), _chain(write)): + return call() + + +def unlocked_repo(f): + @wraps(f) + def wrapper(stage, *args, **kwargs): + stage.repo.state.dump() + stage.repo.lock.unlock() + stage.repo._reset() + try: + ret = f(stage, *args, **kwargs) + finally: + stage.repo.lock.lock() + stage.repo.state.load() + return ret + + return wrapper + + class Stage(object): STAGE_FILE = "Dvcfile" STAGE_FILE_SUFFIX = ".dvc" @@ -286,6 +335,7 @@ class Stage(object): return True return False + @rwlocked(read=["deps", "outs"]) def changed(self): # Short-circuit order: stage md5 is fast, deps are expected to change ret = ( @@ -299,6 +349,7 @@ class Stage(object): return ret + @rwlocked(write=["outs"]) def remove_outs(self, ignore_remove=False, force=False): """Used mainly for `dvc remove --outs` and :func:`Stage.reproduce`.""" for out in self.outs: @@ -316,6 +367,7 @@ class Stage(object): for out in self.outs: out.unprotect() + @rwlocked(write=["outs"]) def remove(self, force=False, remove_outs=True): if remove_outs: self.remove_outs(ignore_remove=True, force=force) @@ -323,6 +375,7 @@ class Stage(object): self.unprotect_outs() os.unlink(self.path) + @rwlocked(read=["deps"], write=["outs"]) def reproduce(self, interactive=False, **kwargs): if not kwargs.get("force", False) and not self.changed(): @@ -755,6 +808,7 @@ class Stage(object): ) self.save() + @rwlocked(write=["outs"]) def commit(self): for out in self.outs: out.commit() @@ -801,6 +855,7 @@ class Stage(object): if occurrence > 1: raise ArgumentDuplicationError(str(path)) + @unlocked_repo def _run(self): self._check_missing_deps() @@ -850,6 +905,7 @@ class Stage(object): if (p is None) or (p.returncode != 0): raise StageCmdFailedError(self) + @rwlocked(read=["deps"], write=["outs"]) def run(self, dry=False, no_commit=False, force=False): if (self.cmd or self.is_import) and not self.locked and not dry: self.remove_outs(ignore_remove=False, force=False) @@ -874,7 +930,6 @@ class Stage(object): self.outs[0].checkout() else: self.deps[0].download(self.outs[0]) - elif self.is_data_source: msg = "Verifying data sources in '{}'".format(self.relpath) logger.info(msg) @@ -904,6 +959,7 @@ class Stage(object): if paths: raise MissingDataSource(paths) + @rwlocked(write=["outs"]) def checkout(self, force=False, progress_callback=None, relink=False): failed_checkouts = [] for out in self.outs: @@ -926,6 +982,7 @@ class Stage(object): return ret + @rwlocked(read=["deps", "outs"]) def status(self): ret = [] diff --git a/dvc/updater.py b/dvc/updater.py index 77145ef9d..661221926 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -9,8 +9,7 @@ import colorama from packaging import version from dvc import __version__ -from dvc.lock import Lock -from dvc.lock import LockError +from dvc.lock import make_lock, LockError from dvc.utils import boxify from dvc.utils import env2bool from dvc.utils.pkg import PKG @@ -24,11 +23,14 @@ class Updater(object): # pragma: no cover TIMEOUT = 24 * 60 * 60 # every day TIMEOUT_GET = 10 - def __init__(self, dvc_dir): + def __init__(self, dvc_dir, friendly=False, hardlink_lock=False): self.dvc_dir = dvc_dir self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE) - self.lock = Lock( - self.updater_file + ".lock", tmp_dir=os.path.join(dvc_dir, "tmp") + self.lock = make_lock( + self.updater_file + ".lock", + tmp_dir=os.path.join(dvc_dir, "tmp"), + friendly=friendly, + hardlink_lock=hardlink_lock, ) self.current = version.parse(__version__).base_version diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 68ba41877..a1d15aa0b 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -420,3 +420,11 @@ def resolve_output(inp, out): if os.path.isdir(out): return os.path.join(out, name) return out + + +def format_link(link): + import colorama + + return "<{blue}{link}{nc}>".format( + blue=colorama.Fore.CYAN, link=link, nc=colorama.Fore.RESET + ) diff --git a/dvc/utils/compat.py b/dvc/utils/compat.py index 372c8d2c6..a4819fccc 100644 --- a/dvc/utils/compat.py +++ b/dvc/utils/compat.py @@ -123,6 +123,7 @@ if is_py2: makedirs = _makedirs range = xrange # noqa: F821 FileNotFoundError = IOError + JSONDecodeError = ValueError import StringIO import io @@ -164,6 +165,7 @@ elif is_py3: import configparser as ConfigParser # noqa: F401 from collections.abc import Mapping # noqa: F401 from contextlib import ExitStack # noqa: F401 + from json.decoder import JSONDecodeError # noqa: F401 builtin_str = str # noqa: F821 str = str # noqa: F821 diff --git a/dvc/version.py b/dvc/version.py index 8a3a2f0ce..867a3fdb0 100644 --- a/dvc/version.py +++ b/dvc/version.py @@ -7,7 +7,7 @@ import os import subprocess -_BASE_VERSION = "0.75.0" +_BASE_VERSION = "0.77.1" def _generate_version(base_version): diff --git a/setup.py b/setup.py index 0ee7cda27..29a4a0936 100644 --- a/setup.py +++ b/setup.py @@ -79,6 +79,7 @@ install_requires = [ "shortuuid>=0.5.0", "tqdm>=4.40.0,<5", "packaging>=19.0", + "zc.lockfile>=1.2.1", "win-unicode-console>=0.5; sys_platform == 'win32'", "pywin32>=225; sys_platform == 'win32'", ] @@ -104,7 +105,7 @@ ssh = ["paramiko>=2.5.0"] # requirements, including kerberos itself. Once all the wheels are available, # we can start shipping it by default. ssh_gssapi = ["paramiko[gssapi]>=2.5.0"] -hdfs = ["pyarrow==0.14.0"] +hdfs = ["pyarrow==0.15.1"] all_remotes = gs + s3 + azure + ssh + oss + gdrive if os.name != "nt" or sys.version_info[0] != 2: @@ -163,12 +164,7 @@ setup( "ssh_gssapi": ssh_gssapi, "hdfs": hdfs, # NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1 - ":python_version=='2.7'": [ - "futures", - "pathlib2", - "contextlib2", - "zc.lockfile>=1.2.1", - ], + ":python_version=='2.7'": ["futures", "pathlib2", "contextlib2"], ":python_version>='3.0'": ["flufl.lock>=3.2"], "tests": tests_requirements, }, diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 1f8a0b39f..b1e9ed058 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -6,13 +6,6 @@ grade: stable confinement: classic # for same reasons as snap `git-ubuntu` base: core18 license: Apache-2.0 -plugs: - config-dvc: - interface: system-files - read: - - /etc/host.conf - - /etc/resolv.conf - - /etc/hosts layout: /etc/dvc: bind: $SNAP_DATA/etc/dvc @@ -48,18 +41,7 @@ apps: environment: GIT_PYTHON_GIT_EXECUTABLE: $SNAP/usr/bin/git XDG_CONFIG_DIRS: /etc - plugs: - - config-dvc - - home - - mount-observe - - network-control - - removable-media git: command: usr/bin/git environment: GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 - plugs: - - home - - mount-observe - - network-control - - removable-media diff --git a/tests/conftest.py b/tests/conftest.py index 46b059d5e..3eca524c0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,6 @@ import pytest from git import Repo from git.exc import GitCommandNotFound -from dvc.remote.config import RemoteConfig from dvc.remote.ssh.connection import SSHConnection from dvc.repo import Repo as DvcRepo from dvc.utils.compat import cast_bytes_py2 @@ -148,11 +147,6 @@ def erepo(repo_dir): repo.dvc.scm.add([stage_foo.path, stage_bar.path, stage_data_dir.path]) repo.dvc.scm.commit("init repo") - rconfig = RemoteConfig(repo.dvc.config) - rconfig.add("upstream", repo.dvc.cache.local.cache_dir, default=True) - repo.dvc.scm.add([repo.dvc.config.config_file]) - repo.dvc.scm.commit("add remote") - repo.create("version", "master") repo.dvc.add("version") repo.dvc.scm.add([".gitignore", "version.dvc"]) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 1d9789790..4487e79b4 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -1,3 +1,47 @@ +""" +The goal of this module is making dvc functional tests setup a breeze. This +includes a temporary dir, initializing git and dvc repos and bootstrapping some +file structure. + +The cornerstone of these fixtures is `tmp_dir`, which creates a temporary dir +and changes path to it, it might be combined with `scm` and `dvc` to initialize +empty git and dvc repos. `tmp_dir` returns a Path instance, which should save +you from using `open()`, `os` and `os.path` utils many times: + + (tmp_dir / "some_file").write_text("some text") + # ... + assert "some text" == (tmp_dir / "some_file").read_text() + assert (tmp_dir / "some_file").exists() + +Additionally it provides `.gen()`, `.scm_gen()` and `.dvc_gen()` methods to +bootstrap a required file structure in a single call: + + # Generate a dir with files + tmp_dir.gen({"dir": {"file": "file text", "second_file": "..."}}) + + # Generate a single file, dirs will be created along the way + tmp_dir.gen("dir/file", "file text") + + # Generate + git add + tmp_dir.scm_gen({"file1": "...", ...}) + + # Generate + git add + git commit + tmp_dir.scm_gen({"file1": "...", ...}, commit="add files") + + # Generate + dvc add + tmp_dir.dvc_gen({"file1": "...", ...}) + + # Generate + dvc add + git commit -am "..." + # This commits stages to git not the generated files. + tmp_dir.dvc_gen({"file1": "...", ...}, commit="add files") + +Making it easier to bootstrap things has a supergoal of incentivizing a move +from global repo template to creating everything inplace, which: + + - makes all path references local to test, enhancing readability + - allows using telling filenames, e.g. "git_tracked_file" instead of "foo" + - does not create unnecessary files +""" from __future__ import unicode_literals import os @@ -6,7 +50,7 @@ import pytest from funcy.py3 import lmap, retry from dvc.utils import makedirs -from dvc.utils.compat import basestring, is_py2, pathlib, fspath, fspath_py35 +from dvc.utils.compat import basestring, pathlib, fspath, fspath_py35, bytes __all__ = ["tmp_dir", "scm", "dvc", "repo_template", "run_copy", "erepo_dir"] @@ -44,7 +88,7 @@ class TmpDir(pathlib.Path): ) def gen(self, struct, text=""): - if isinstance(struct, basestring): + if isinstance(struct, (basestring, pathlib.PurePath)): struct = {struct: text} self._gen(struct) @@ -55,13 +99,16 @@ class TmpDir(pathlib.Path): path = (prefix or self) / name if isinstance(contents, dict): - self._gen(contents, prefix=path) + if not contents: + makedirs(path, exist_ok=True) + else: + self._gen(contents, prefix=path) else: makedirs(path.parent, exist_ok=True) - if is_py2 and isinstance(contents, str): + if isinstance(contents, bytes): path.write_bytes(contents) else: - path.write_text(contents) + path.write_text(contents, encoding="utf-8") def dvc_gen(self, struct, text="", commit=None): paths = self.gen(struct, text) @@ -182,7 +229,7 @@ def run_copy(tmp_dir, dvc, request): # Do we need this? if "scm" in request.fixturenames: request.getfixturevalue("scm") - tmp_dir.git_add("copy.py", commit="add copy.py") + tmp_dir.scm_add("copy.py", commit="add copy.py") def run_copy(src, dst, **run_kwargs): return dvc.run( @@ -203,27 +250,27 @@ def erepo_dir(tmp_path_factory, monkeypatch): path = TmpDir(fspath_py35(tmp_path_factory.mktemp("erepo"))) # Chdir for git and dvc to work locally - monkeypatch.chdir(fspath_py35(path)) + with monkeypatch.context() as m: + m.chdir(fspath_py35(path)) - _git_init() - path.dvc = Repo.init() - path.scm = path.dvc.scm - path.dvc_gen(REPO_TEMPLATE, commit="init repo") + _git_init() + path.dvc = Repo.init() + path.scm = path.dvc.scm + path.dvc_gen(REPO_TEMPLATE, commit="init repo") - rconfig = RemoteConfig(path.dvc.config) - rconfig.add("upstream", path.dvc.cache.local.cache_dir, default=True) - path.scm_add([path.dvc.config.config_file], commit="add remote") + rconfig = RemoteConfig(path.dvc.config) + rconfig.add("upstream", path.dvc.cache.local.cache_dir, default=True) + path.scm_add([path.dvc.config.config_file], commit="add remote") - path.dvc_gen("version", "master") - path.scm_add([".gitignore", "version.dvc"], commit="master") + path.dvc_gen("version", "master") + path.scm_add([".gitignore", "version.dvc"], commit="master") - path.scm.checkout("branch", create_new=True) - (path / "version").unlink() # For mac ??? - path.dvc_gen("version", "branch") - path.scm_add([".gitignore", "version.dvc"], commit="branch") + path.scm.checkout("branch", create_new=True) + (path / "version").unlink() # For mac ??? + path.dvc_gen("version", "branch") + path.scm_add([".gitignore", "version.dvc"], commit="branch") - path.scm.checkout("master") - path.dvc.close() - monkeypatch.undo() # Undo chdir + path.scm.checkout("master") + path.dvc.close() return path diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 00ba50122..300cccc45 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -31,57 +31,47 @@ from tests.utils import get_gitignore_content from tests.utils import spy -class TestAdd(TestDvc): - def test(self): - md5 = file_md5(self.FOO)[0] +def test_add(tmp_dir, dvc): + stage, = tmp_dir.dvc_gen({"foo": "foo"}) + md5, _ = file_md5("foo") - stages = self.dvc.add(self.FOO) - self.assertEqual(len(stages), 1) - stage = stages[0] - self.assertTrue(stage is not None) + assert stage is not None - self.assertIsInstance(stage, Stage) - self.assertTrue(os.path.isfile(stage.path)) - self.assertEqual(len(stage.outs), 1) - self.assertEqual(len(stage.deps), 0) - self.assertEqual(stage.cmd, None) - self.assertEqual(stage.outs[0].info["md5"], md5) - self.assertEqual(stage.md5, "ee343f2482f53efffc109be83cc976ac") - - def test_unicode(self): - fname = "\xe1" - - with open(fname, "w") as fobj: - fobj.write("something") - - stage = self.dvc.add(fname)[0] - - self.assertTrue(os.path.isfile(stage.path)) + assert isinstance(stage, Stage) + assert os.path.isfile(stage.path) + assert len(stage.outs) == 1 + assert len(stage.deps) == 0 + assert stage.cmd is None + assert stage.outs[0].info["md5"] == md5 + assert stage.md5 == "ee343f2482f53efffc109be83cc976ac" -class TestAddUnsupportedFile(TestDvc): - def test(self): - with self.assertRaises(DvcException): - self.dvc.add("unsupported://unsupported") +def test_add_unicode(tmp_dir, dvc): + with open("\xe1", "wb") as fd: + fd.write("something".encode("utf-8")) + + stage, = dvc.add("\xe1") + + assert os.path.isfile(stage.path) -class TestAddDirectory(TestDvc): - def test(self): - dname = "directory" - os.mkdir(dname) - self.create(os.path.join(dname, "file"), "file") - stages = self.dvc.add(dname) - self.assertEqual(len(stages), 1) - stage = stages[0] - self.assertTrue(stage is not None) - self.assertEqual(len(stage.deps), 0) - self.assertEqual(len(stage.outs), 1) +def test_add_unsupported_file(dvc): + with pytest.raises(DvcException): + dvc.add("unsupported://unsupported") - md5 = stage.outs[0].info["md5"] - dir_info = self.dvc.cache.local.load_dir_cache(md5) - for info in dir_info: - self.assertTrue("\\" not in info["relpath"]) +def test_add_directory(tmp_dir, dvc): + stage, = tmp_dir.dvc_gen({"dir": {"file": "file"}}) + + assert stage is not None + assert len(stage.deps) == 0 + assert len(stage.outs) == 1 + + md5 = stage.outs[0].info["md5"] + + dir_info = dvc.cache.local.load_dir_cache(md5) + for info in dir_info: + assert "\\" not in info["relpath"] class TestAddDirectoryRecursive(TestDvc): diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py index b42a11727..a2eccfd62 100644 --- a/tests/func/test_analytics.py +++ b/tests/func/test_analytics.py @@ -1,65 +1,20 @@ -import os - import mock -import requests -from dvc.analytics import Analytics from dvc.main import main -from tests.basic_env import TestDir -from tests.basic_env import TestDvc -from tests.basic_env import TestGit +from dvc.utils.compat import fspath -def _clean_getenv(key, default=None): - """ - Remove env vars that affect dvc behavior in tests - """ - if key in ["DVC_TEST", "CI"]: - return None - return os.environ.get(key, default) +@mock.patch("dvc.analytics.send") +def test_daemon_analytics(mock_send, tmp_path): + report = fspath(tmp_path) + assert 0 == main(["daemon", "analytics", report]) + + mock_send.assert_called_with(report) -class TestAnalytics(TestDir): - def test(self): - a = Analytics() - a.collect() - self.assertTrue(isinstance(a.info, dict)) - self.assertNotEqual(a.info, {}) - self.assertTrue(a.PARAM_USER_ID in a.info.keys()) - self.assertTrue(a.PARAM_SYSTEM_INFO in a.info.keys()) - self.assertNotEqual(a.info[a.PARAM_SYSTEM_INFO], {}) - - @mock.patch.object(os, "getenv", new=_clean_getenv) - @mock.patch("requests.post") - def test_send(self, mockpost): - ret = main(["daemon", "analytics", Analytics().dump(), "-v"]) - self.assertEqual(ret, 0) - - self.assertTrue(mockpost.called) - - @mock.patch.object(os, "getenv", new=_clean_getenv) - @mock.patch.object( - requests, "post", side_effect=requests.exceptions.RequestException() - ) - def test_send_failed(self, mockpost): - ret = main(["daemon", "analytics", Analytics().dump(), "-v"]) - self.assertEqual(ret, 0) - - self.assertTrue(mockpost.called) - - -class TestAnalyticsGit(TestAnalytics, TestGit): - pass - - -class TestAnalyticsDvc(TestAnalytics, TestDvc): - @mock.patch("requests.post") - def test_send_disabled(self, mockpost): - ret = main(["config", "core.analytics", "false"]) - self.assertEqual(ret, 0) - - with mock.patch.object(os, "getenv", new=_clean_getenv): - ret = main(["daemon", "analytics", Analytics().dump(), "-v"]) - self.assertEqual(ret, 0) - - self.assertFalse(mockpost.called) +@mock.patch("dvc.analytics.collect_and_send_report") +@mock.patch("dvc.analytics.is_enabled", return_value=True) +def test_main_analytics(mock_is_enabled, mock_report, dvc_repo): + assert 0 == main(["add", "foo"]) + assert mock_is_enabled.called + assert mock_report.called diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 5ff3b5f40..e7991e141 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -474,11 +474,14 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc): ) -def test_checkout_no_checksum(repo_dir, dvc_repo): - stage = dvc_repo.run(outs=[repo_dir.FOO], no_exec=True, cmd="somecmd") +def test_checkout_no_checksum(tmp_dir, dvc): + tmp_dir.gen("file", "file content") + stage = dvc.run(outs=["file"], no_exec=True, cmd="somecmd") + with pytest.raises(CheckoutError): - dvc_repo.checkout([stage.path], force=True) - assert not os.path.exists(repo_dir.FOO) + dvc.checkout([stage.path], force=True) + + assert not os.path.exists("file") @pytest.mark.parametrize( diff --git a/tests/func/test_commit.py b/tests/func/test_commit.py index c3238e6ad..8d6a8e57d 100644 --- a/tests/func/test_commit.py +++ b/tests/func/test_commit.py @@ -7,40 +7,37 @@ from dvc.utils.stage import dump_stage_file from dvc.utils.stage import load_stage_file -def test_commit_recursive(dvc_repo, repo_dir): - stages = dvc_repo.add(repo_dir.DATA_DIR, recursive=True, no_commit=True) +def test_commit_recursive(tmp_dir, dvc): + tmp_dir.gen({"dir": {"file": "text1", "subdir": {"file2": "text2"}}}) + stages = dvc.add("dir", recursive=True, no_commit=True) + assert len(stages) == 2 + assert dvc.status() != {} - assert dvc_repo.status() != {} - - dvc_repo.commit(repo_dir.DATA_DIR, recursive=True) - - assert dvc_repo.status() == {} + dvc.commit("dir", recursive=True) + assert dvc.status() == {} -def test_commit_force(dvc_repo, repo_dir): - stages = dvc_repo.add(repo_dir.FOO, no_commit=True) - assert len(stages) == 1 - stage = stages[0] +def test_commit_force(tmp_dir, dvc): + tmp_dir.gen({"dir": {"file": "text1", "file2": "text2"}}) + stage, = dvc.add("dir", no_commit=True) - with dvc_repo.state: + with dvc.state: assert stage.outs[0].changed_cache() - with open(repo_dir.FOO, "a") as fobj: - fobj.write(repo_dir.FOO_CONTENTS) + tmp_dir.gen("dir/file", "file content modified") - with dvc_repo.state: + with dvc.state: assert stage.outs[0].changed_cache() with pytest.raises(StageCommitError): - dvc_repo.commit(stage.path) + dvc.commit(stage.path) - with dvc_repo.state: + with dvc.state: assert stage.outs[0].changed_cache() - dvc_repo.commit(stage.path, force=True) - - assert dvc_repo.status([stage.path]) == {} + dvc.commit(stage.path, force=True) + assert dvc.status([stage.path]) == {} def test_commit_with_deps(tmp_dir, dvc, run_copy): @@ -63,16 +60,15 @@ def test_commit_with_deps(tmp_dir, dvc, run_copy): assert not stage.outs[0].changed_cache() -def test_commit_changed_md5(dvc_repo, repo_dir): - stages = dvc_repo.add(repo_dir.FOO, no_commit=True) - assert len(stages) == 1 - stage = stages[0] +def test_commit_changed_md5(tmp_dir, dvc): + tmp_dir.gen({"file": "file content"}) + stage, = dvc.add("file", no_commit=True) stage_file_content = load_stage_file(stage.path) stage_file_content["md5"] = "1111111111" dump_stage_file(stage.path, stage_file_content) with pytest.raises(StageCommitError): - dvc_repo.commit(stage.path) + dvc.commit(stage.path) - dvc_repo.commit(stage.path, force=True) + dvc.commit(stage.path, force=True) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 07551c5f3..a48e76bd9 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -7,12 +7,13 @@ import os import pytest from dvc.config import Config -from dvc.exceptions import GetDVCFileError from dvc.exceptions import UrlNotDvcRepoError +from dvc.repo.get import GetDVCFileError, PathMissingError from dvc.repo import Repo from dvc.system import System from dvc.utils import makedirs from dvc.utils.compat import fspath +from dvc.utils import fspath_py35 from tests.utils import trees_equal @@ -38,6 +39,36 @@ def test_get_repo_dir(erepo): trees_equal(src, dst) +def test_get_regular_file(erepo): + src = "some_file" + dst = "some_file_imported" + + src_path = os.path.join(erepo.root_dir, src) + erepo.create(src_path, "hello") + erepo.dvc.scm.add([src_path]) + erepo.dvc.scm.commit("add a regular file") + Repo.get(erepo.root_dir, src, dst) + + assert os.path.exists(dst) + assert os.path.isfile(dst) + assert filecmp.cmp(src_path, dst, shallow=False) + + +def test_get_regular_dir(erepo): + src = "some_directory" + dst = "some_directory_imported" + + src_file_path = os.path.join(erepo.root_dir, src, "file.txt") + erepo.create(src_file_path, "hello") + erepo.dvc.scm.add([src_file_path]) + erepo.dvc.scm.commit("add a regular dir") + Repo.get(erepo.root_dir, src, dst) + + assert os.path.exists(dst) + assert os.path.isdir(dst) + trees_equal(os.path.join(erepo.root_dir, src), dst) + + def test_cache_type_is_properly_overridden(erepo): erepo.dvc.config.set( Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE, "symlink" @@ -77,6 +108,51 @@ def test_get_a_dvc_file(erepo): Repo.get(erepo.root_dir, "some_file.dvc") +# https://github.com/iterative/dvc/pull/2837#discussion_r352123053 +def test_get_full_dvc_path(erepo): + external_data_dir = erepo.mkdtemp() + external_data = os.path.join(external_data_dir, "ext_data") + with open(external_data, "w+") as fobj: + fobj.write("ext_data") + + cur_dir = os.getcwd() + os.chdir(erepo.root_dir) + erepo.dvc.add(external_data) + erepo.dvc.scm.add(["ext_data.dvc"]) + erepo.dvc.scm.commit("add external data") + os.chdir(cur_dir) + + Repo.get(erepo.root_dir, external_data, "ext_data_imported") + assert os.path.isfile("ext_data_imported") + assert filecmp.cmp(external_data, "ext_data_imported", shallow=False) + + +def test_non_cached_output(tmp_path, erepo): + os.chdir(erepo.root_dir) + erepo.dvc.run( + outs_no_cache=["non_cached_file"], cmd="echo hello > non_cached_file" + ) + erepo.dvc.scm.add(["non_cached_file", "non_cached_file.dvc"]) + erepo.dvc.scm.commit("add non-cached output") + os.chdir(fspath_py35(tmp_path)) + Repo.get(erepo.root_dir, "non_cached_file") + + src = os.path.join(erepo.root_dir, "non_cached_file") + assert os.path.isfile("non_cached_file") + assert filecmp.cmp(src, "non_cached_file", shallow=False) + + +# https://github.com/iterative/dvc/pull/2837#discussion_r352123053 +def test_absolute_file_outside_repo(erepo): + with pytest.raises(PathMissingError): + Repo.get(erepo.root_dir, "/root/") + + +def test_unknown_path(erepo): + with pytest.raises(PathMissingError): + Repo.get(erepo.root_dir, "a_non_existing_file") + + @pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"]) def test_get_to_dir(dname, erepo): src = erepo.FOO diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index b09a06a78..97fac4644 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -1,186 +1,120 @@ -import itertools +# encoding: utf-8 +from __future__ import unicode_literals import os import shutil - import pytest from dvc.exceptions import DvcIgnoreInCollectedDirError -from dvc.ignore import DvcIgnore -from dvc.ignore import DvcIgnoreDirs -from dvc.ignore import DvcIgnoreFilter -from dvc.ignore import DvcIgnorePatterns +from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns from dvc.scm.tree import WorkingTree -from dvc.utils.compat import cast_bytes +from dvc.utils import walk_files +from dvc.utils.compat import fspath from dvc.utils.fs import get_mtime_and_size -from tests.basic_env import TestDvc + from tests.utils import to_posixpath -class TestDvcIgnore(TestDvc): - def setUp(self): - super(TestDvcIgnore, self).setUp() +def test_ignore(tmp_dir, dvc, monkeypatch): + tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}}) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - def _get_all_paths(self): + assert _files_set("dir", dvc.dvcignore) == {"dir/other"} - paths = [] - for root, dirs, files in self.dvc.tree.walk( - self.dvc.root_dir, dvcignore=self.dvc.dvcignore - ): - for dname in dirs: - paths.append(os.path.join(root, dname)) - - for fname in files: - paths.append(os.path.join(root, fname)) - - return paths - - def test_ignore_in_child_dir(self): - ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE) - with open(ignore_file, "w") as fobj: - fobj.write("data_dir/data") - - forbidden_path = os.path.join(self.dvc.root_dir, self.DATA) - all_paths = self._get_all_paths() - - self.assertNotIn(forbidden_path, all_paths) - - def test_ignore_in_child_dir_unicode(self): - ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE) - with open(ignore_file, "wb") as fobj: - fobj.write(cast_bytes(self.UNICODE, "utf-8")) - - forbidden_path = os.path.join(self.dvc.root_dir, self.UNICODE) - all_paths = self._get_all_paths() - - self.assertNotIn(forbidden_path, all_paths) - - def test_ignore_in_parent_dir(self): - ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE) - with open(ignore_file, "w") as fobj: - fobj.write("data_dir/data") - - os.chdir(self.DATA_DIR) - - forbidden_path = os.path.join(self.dvc.root_dir, self.DATA) - all_paths = self._get_all_paths() - - self.assertNotIn(forbidden_path, all_paths) + monkeypatch.chdir("dir") + assert _files_set(".", dvc.dvcignore) == {"./other"} -def test_metadata_unchanged_when_moving_ignored_file(dvc_repo, repo_dir): - new_data_path = repo_dir.DATA_SUB + "_new" +def test_ignore_unicode(tmp_dir, dvc): + tmp_dir.gen({"dir": {"other": "text"}}) + # Path() doesn't handle unicode paths in Windows/Python 2 + # I don't know to debug it further, waiting till Python 2 EOL + with open("dir/тест", "wb") as fd: + fd.write("проверка".encode("utf-8")) - ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE) - repo_dir.create( - ignore_file, - "\n".join( - [to_posixpath(repo_dir.DATA_SUB), to_posixpath(new_data_path)] - ), - ) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест") - mtime_sig, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) - - shutil.move(repo_dir.DATA_SUB, new_data_path) - - new_mtime_sig, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) - - assert new_mtime_sig == mtime_sig - assert new_size == size + assert _files_set("dir", dvc.dvcignore) == {"dir/other"} -def test_mtime_changed_when_moving_non_ignored_file(dvc_repo, repo_dir): - new_data_path = repo_dir.DATA_SUB + "_new" - mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) +def test_rename_ignored_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}}) - shutil.move(repo_dir.DATA_SUB, new_data_path) - new_mtime, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*") + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) - assert new_mtime != mtime - assert new_size == size + shutil.move("dir/ignored", "dir/ignored_new") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) + + assert new_mtime == mtime and new_size == size -def test_metadata_unchanged_on_ignored_file_deletion(dvc_repo, repo_dir): - ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE) - repo_dir.create(ignore_file, to_posixpath(repo_dir.DATA_SUB)) +def test_rename_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) - mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) + shutil.move("dir/foo", "dir/foo_new") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) - os.remove(repo_dir.DATA_SUB) - new_mtime, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) - - assert new_mtime == mtime - assert new_size == size + assert new_mtime != mtime and new_size == size -def test_metadata_changed_on_non_ignored_file_deletion(dvc_repo, repo_dir): - mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) +def test_remove_ignored_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}}) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - os.remove(repo_dir.DATA_SUB) - new_mtime_sig, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) - assert new_mtime_sig != mtime - assert new_size != size + os.remove("dir/ignored") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) + + assert new_mtime == mtime and new_size == size -def test_should_raise_on_dvcignore_in_out_dir(dvc_repo, repo_dir): - ignore_file = os.path.join(repo_dir.DATA_DIR, DvcIgnore.DVCIGNORE_FILE) - repo_dir.create(ignore_file, "") +def test_remove_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) + + os.remove("dir/foo") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) + + assert new_mtime != mtime and new_size != size + + +def test_dvcignore_in_out_dir(tmp_dir, dvc): + tmp_dir.gen({"dir": {"foo": "foo", DvcIgnore.DVCIGNORE_FILE: ""}}) with pytest.raises(DvcIgnoreInCollectedDirError): - dvc_repo.add(repo_dir.DATA_DIR) + dvc.add("dir") -@pytest.mark.parametrize("dname", [TestDvc.DATA_DIR, TestDvc.DATA_SUB_DIR]) -def test_ignore_collecting_dvcignores(repo_dir, dname): - top_ignore_file = os.path.join( - repo_dir.root_dir, os.path.dirname(dname), DvcIgnore.DVCIGNORE_FILE - ) - repo_dir.create(top_ignore_file, os.path.basename(dname)) +@pytest.mark.parametrize("dname", ["dir", "dir/subdir"]) +def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname): + tmp_dir.gen({"dir": {"subdir": {}}}) - ignore_file = os.path.join( - repo_dir.root_dir, dname, DvcIgnore.DVCIGNORE_FILE - ) - repo_dir.create(ignore_file, repo_dir.FOO) + top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE) + top_ignore_file.write_text(os.path.basename(dname)) - assert DvcIgnoreFilter( - repo_dir.root_dir, WorkingTree(repo_dir.root_dir) - ).ignores == { + ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE + ignore_file.write_text("foo") + + assert dvc.dvcignore.ignores == { DvcIgnoreDirs([".git", ".hg", ".dvc"]), - DvcIgnorePatterns(top_ignore_file, WorkingTree(repo_dir.root_dir)), + DvcIgnorePatterns(fspath(top_ignore_file), WorkingTree(dvc.root_dir)), } -def test_ignore_on_branch(git, dvc_repo, repo_dir): - dvc_repo.add(repo_dir.DATA_DIR) - dvc_repo.scm.commit("add data dir") +def test_ignore_on_branch(tmp_dir, scm, dvc): + tmp_dir.scm_gen({"foo": "foo", "bar": "bar"}, commit="add files") - branch_name = "branch_one" - dvc_repo.scm.checkout(branch_name, create_new=True) + scm.checkout("branch", create_new=True) + tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore") - repo_dir.create(DvcIgnore.DVCIGNORE_FILE, to_posixpath(repo_dir.DATA_SUB)) - dvc_repo.scm.add([DvcIgnore.DVCIGNORE_FILE]) - git.index.commit("add ignore") + scm.checkout("master") + assert _files_set(".", dvc.dvcignore) == {"./foo", "./bar"} - dvc_repo.scm.checkout("master") + dvc.tree = scm.get_tree("branch") + assert _files_set(".", dvc.dvcignore) == {"./bar"} - git_tree = dvc_repo.scm.get_tree(branch_name) - branch_data_files = set( - itertools.chain.from_iterable( - [ - files - for _, _, files in dvc_repo.tree.walk( - repo_dir.DATA_DIR, - dvcignore=DvcIgnoreFilter(repo_dir.root_dir, git_tree), - ) - ] - ) - ) - assert branch_data_files == {"data"} + +def _files_set(root, dvcignore): + return {to_posixpath(f) for f in walk_files(root, dvcignore)} diff --git a/tests/func/test_move.py b/tests/func/test_move.py index eef8a603c..416ea3682 100644 --- a/tests/func/test_move.py +++ b/tests/func/test_move.py @@ -195,19 +195,21 @@ class TestMoveFileInsideDirectory(TestDvc): self.assertTrue(os.path.exists(dvc_fullpath)) -def test_move_should_save_stage_info(dvc_repo, repo_dir): - dvc_repo.add(repo_dir.DATA_DIR) +def test_move_should_save_stage_info(tmp_dir, dvc): + tmp_dir.dvc_gen({"old_name": {"file1": "file1"}}) - dvc_repo.move(repo_dir.DATA_DIR, "new_name") + dvc.move("old_name", "new_name") - assert dvc_repo.status() == {} + assert dvc.status() == {} -def test_should_move_to_dir_on_non_default_stage_file(dvc_repo, repo_dir): +def test_should_move_to_dir_on_non_default_stage_file(tmp_dir, dvc): stage_file_name = "stage.dvc" + tmp_dir.gen({"file": "file_content"}) - dvc_repo.add(repo_dir.FOO, fname=stage_file_name) + dvc.add("file", fname=stage_file_name) + os.mkdir("directory") - dvc_repo.move(repo_dir.FOO, repo_dir.DATA_DIR) + dvc.move("file", "directory") - assert os.path.exists(os.path.join(repo_dir.DATA_DIR, repo_dir.FOO)) + assert os.path.exists(os.path.join("directory", "file")) diff --git a/tests/func/test_run.py b/tests/func/test_run.py index 7d9a51f29..61db6c98e 100644 --- a/tests/func/test_run.py +++ b/tests/func/test_run.py @@ -893,7 +893,7 @@ class TestShouldNotCheckoutUponCorruptedLocalHardlinkCache(TestDvc): ) patch_run = mock.patch.object(stage, "_run", wraps=stage._run) - with self.dvc.state: + with self.dvc.lock, self.dvc.state: with patch_checkout as mock_checkout: with patch_run as mock_run: stage.run() diff --git a/tests/unit/remote/test_s3.py b/tests/unit/remote/test_s3.py new file mode 100644 index 000000000..57fcfcec7 --- /dev/null +++ b/tests/unit/remote/test_s3.py @@ -0,0 +1,56 @@ +import pytest + +from dvc.config import ConfigError +from dvc.remote.s3 import RemoteS3 + + +bucket_name = "bucket-name" +prefix = "some/prefix" +url = "s3://{}/{}".format(bucket_name, prefix) + + +@pytest.fixture(autouse=True) +def grants(): + return { + "grant_read": "id=read-permission-id,id=other-read-permission-id", + "grant_read_acp": "id=read-acp-permission-id", + "grant_write_acp": "id=write-acp-permission-id", + "grant_full_control": "id=full-control-permission-id", + } + + +def test_init(): + config = {"url": url} + remote = RemoteS3(None, config) + + assert remote.path_info == url + + +def test_grants(): + config = { + "url": url, + "grant_read": "id=read-permission-id,id=other-read-permission-id", + "grant_read_acp": "id=read-acp-permission-id", + "grant_write_acp": "id=write-acp-permission-id", + "grant_full_control": "id=full-control-permission-id", + } + remote = RemoteS3(None, config) + + assert ( + remote.extra_args["GrantRead"] + == "id=read-permission-id,id=other-read-permission-id" + ) + assert remote.extra_args["GrantReadACP"] == "id=read-acp-permission-id" + assert remote.extra_args["GrantWriteACP"] == "id=write-acp-permission-id" + assert ( + remote.extra_args["GrantFullControl"] + == "id=full-control-permission-id" + ) + + +def test_grants_mutually_exclusive_acl_error(grants): + for grant_option, grant_value in grants.items(): + config = {"url": url, "acl": "public-read", grant_option: grant_value} + + with pytest.raises(ConfigError): + RemoteS3(None, config) diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 9e7e2dab1..43eb37dcb 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -1,6 +1,80 @@ import pytest +import mock +import platform +import json -from dvc.analytics import Analytics +from voluptuous import Schema, Any + +from dvc import analytics +from dvc.cli import parse_args +from dvc.utils.compat import str, builtin_str + + +string = Any(str, builtin_str) + + +@pytest.fixture +def tmp_global_config(tmp_path): + """ + Fixture to prevent modifying the actual global config + """ + with mock.patch( + "dvc.config.Config.get_global_config_dir", return_value=str(tmp_path) + ): + yield + + +@mock.patch("dvc.daemon._spawn") +@mock.patch("json.dump") +def test_collect_and_send_report(mock_json, mock_daemon, tmp_global_config): + analytics.collect_and_send_report() + report = mock_json.call_args[0][0] + + with pytest.raises(KeyError): + report["cmd_class"] + + with pytest.raises(KeyError): + report["cmd_return_code"] + + args = parse_args(["add", "foo"]) + return_code = 0 + + analytics.collect_and_send_report(args, return_code) + report = mock_json.call_args[0][0] + + assert report["cmd_class"] == "CmdAdd" + assert report["cmd_return_code"] == return_code + + assert mock_daemon.call_count == 2 + + +def test_runtime_info(tmp_global_config): + schema = Schema( + { + "dvc_version": string, + "is_binary": bool, + "scm_class": Any("Git", None), + "user_id": string, + "system_info": dict, + }, + required=True, + ) + + assert schema(analytics._runtime_info()) + + +@mock.patch("requests.post") +def test_send(mock_post, tmp_path): + url = "https://analytics.dvc.org" + report = {"name": "dummy report"} + fname = str(tmp_path / "report") + + with open(fname, "w") as fobj: + json.dump(report, fobj) + + analytics.send(fname) + assert mock_post.called + assert mock_post.call_args.args[0] == url @pytest.mark.parametrize( @@ -13,7 +87,7 @@ from dvc.analytics import Analytics ({"analytics": "false", "unknown": "broken"}, False), ], ) -def test_is_enabled(dvc_repo, config, result, monkeypatch): +def test_is_enabled(dvc_repo, config, result, monkeypatch, tmp_global_config): configobj = dvc_repo.config._repo_config configobj["core"] = config configobj.write() @@ -21,4 +95,41 @@ def test_is_enabled(dvc_repo, config, result, monkeypatch): # reset DVC_TEST env var, which affects `is_enabled()` monkeypatch.delenv("DVC_TEST") - assert result == Analytics.is_enabled() + assert result == analytics.is_enabled() + + +def test_system_info(): + schema = Schema({"os": Any("windows", "mac", "linux")}, required=True) + + system = platform.system() + + if system == "Windows": + schema = schema.extend( + { + "windows_version_build": int, + "windows_version_major": int, + "windows_version_minor": int, + "windows_version_service_pack": string, + } + ) + + if system == "Darwin": + schema = schema.extend({"mac_version": string}) + + if system == "Linux": + schema = schema.extend( + { + "linux_distro": string, + "linux_distro_like": string, + "linux_distro_version": string, + } + ) + + assert schema(analytics._system_info()) + + +def test_find_or_create_user_id(tmp_global_config): + created = analytics._find_or_create_user_id() + found = analytics._find_or_create_user_id() + + assert created == found diff --git a/tests/unit/test_rwlock.py b/tests/unit/test_rwlock.py new file mode 100644 index 000000000..9e554960e --- /dev/null +++ b/tests/unit/test_rwlock.py @@ -0,0 +1,98 @@ +from __future__ import unicode_literals + +import os + +import pytest + +from dvc.rwlock import ( + rwlock, + _edit_rwlock, + RWLockFileFormatError, + RWLockFileCorruptedError, +) +from dvc.lock import LockError +from dvc.utils.compat import fspath +from dvc.path_info import PathInfo + + +def test_rwlock(tmp_path): + path = fspath(tmp_path) + foo = PathInfo("foo") + + with rwlock(path, "cmd1", [foo], []): + with pytest.raises(LockError): + with rwlock(path, "cmd2", [], [foo]): + pass + + with rwlock(path, "cmd1", [], [foo]): + with pytest.raises(LockError): + with rwlock(path, "cmd2", [foo], []): + pass + + with rwlock(path, "cmd1", [], [foo]): + with pytest.raises(LockError): + with rwlock(path, "cmd2", [], [foo]): + pass + + +def test_rwlock_reentrant(tmp_path): + path = fspath(tmp_path) + foo = PathInfo("foo") + + with rwlock(path, "cmd1", [], [foo]): + with rwlock(path, "cmd1", [], [foo]): + pass + with _edit_rwlock(path) as lock: + assert lock == { + "read": {}, + "write": {"foo": {"cmd": "cmd1", "pid": os.getpid()}}, + } + + with rwlock(path, "cmd", [foo], []): + with rwlock(path, "cmd", [foo], []): + pass + with _edit_rwlock(path) as lock: + assert lock == { + "read": {"foo": [{"cmd": "cmd", "pid": os.getpid()}]}, + "write": {}, + } + + +def test_rwlock_subdirs(tmp_path): + path = fspath(tmp_path) + foo = PathInfo("foo") + subfoo = PathInfo("foo/subfoo") + + with rwlock(path, "cmd1", [foo], []): + with pytest.raises(LockError): + with rwlock(path, "cmd2", [], [subfoo]): + pass + + with rwlock(path, "cmd1", [], [subfoo]): + with pytest.raises(LockError): + with rwlock(path, "cmd2", [foo], []): + pass + + with rwlock(path, "cmd1", [], [subfoo]): + with pytest.raises(LockError): + with rwlock(path, "cmd2", [], [foo]): + pass + + with rwlock(path, "cmd1", [subfoo], []): + with rwlock(path, "cmd2", [foo], []): + pass + + +def test_broken_rwlock(tmp_path): + dir_path = fspath(tmp_path) + path = tmp_path / "rwlock" + + path.write_text('{"broken": "format"}', encoding="utf-8") + with pytest.raises(RWLockFileFormatError): + with _edit_rwlock(dir_path): + pass + + path.write_text("{broken json", encoding="utf-8") + with pytest.raises(RWLockFileCorruptedError): + with _edit_rwlock(dir_path): + pass diff --git a/tests/unit/test_stage.py b/tests/unit/test_stage.py index 652c4b529..354fb6007 100644 --- a/tests/unit/test_stage.py +++ b/tests/unit/test_stage.py @@ -93,16 +93,14 @@ def test_stage_update(mocker): not isinstance(threading.current_thread(), threading._MainThread), reason="Not running in the main thread.", ) -def test_stage_run_ignore_sigint(mocker): - stage = Stage(None, "path") - +def test_stage_run_ignore_sigint(dvc_repo, mocker): proc = mocker.Mock() communicate = mocker.Mock() proc.configure_mock(returncode=0, communicate=communicate) popen = mocker.patch.object(subprocess, "Popen", return_value=proc) signal_mock = mocker.patch("signal.signal") - stage._run() + dvc_repo.run(cmd="path") assert popen.called_once() assert communicate.called_once_with() @@ -110,8 +108,8 @@ def test_stage_run_ignore_sigint(mocker): assert signal.getsignal(signal.SIGINT) == signal.default_int_handler -def test_always_changed(): - stage = Stage(None, "path", always_changed=True) +def test_always_changed(dvc_repo): + stage = Stage(dvc_repo, "path", always_changed=True) stage.save() assert stage.changed() assert stage.status()["path"] == ["always changed"]