From ef1f038c322cc9a11cea1a631979e14faa65b57b Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Wed, 20 Nov 2019 13:29:43 -0600 Subject: [PATCH 001/118] analytics: refactor into a module --- dvc/analytics.py | 256 ----------------------------------- dvc/analytics/__init__.py | 64 +++++++++ dvc/analytics/__main__.py | 8 ++ dvc/analytics/system_info.py | 29 ++++ dvc/analytics/user_id.py | 55 ++++++++ dvc/command/daemon.py | 20 --- dvc/config.py | 11 +- dvc/lock.py | 6 +- dvc/main.py | 5 +- dvc/repo/init.py | 4 +- tests/func/test_analytics.py | 65 --------- tests/unit/test_analytics.py | 4 +- 12 files changed, 171 insertions(+), 356 deletions(-) delete mode 100644 dvc/analytics.py create mode 100644 dvc/analytics/__init__.py create mode 100644 dvc/analytics/__main__.py create mode 100644 dvc/analytics/system_info.py create mode 100644 dvc/analytics/user_id.py delete mode 100644 tests/func/test_analytics.py diff --git a/dvc/analytics.py b/dvc/analytics.py deleted file mode 100644 index 8b39fc31b..000000000 --- a/dvc/analytics.py +++ /dev/null @@ -1,256 +0,0 @@ -"""Collect and send usage analytics""" -from __future__ import unicode_literals - -import errno -import json -import logging -import os - -from dvc import __version__ -from dvc.utils import env2bool -from dvc.utils.compat import str - - -logger = logging.getLogger(__name__) - - -class Analytics(object): - """Class for collecting and sending usage analytics. - - Args: - info (dict): optional existing analytics report. - """ - - URL = "https://analytics.dvc.org" - TIMEOUT_POST = 5 - - USER_ID_FILE = "user_id" - - PARAM_DVC_VERSION = "dvc_version" - PARAM_USER_ID = "user_id" - PARAM_SYSTEM_INFO = "system_info" - - PARAM_OS = "os" - - PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major" - PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor" - PARAM_WINDOWS_VERSION_BUILD = "windows_version_build" - PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack" - - PARAM_MAC_VERSION = "mac_version" - - PARAM_LINUX_DISTRO = "linux_distro" - PARAM_LINUX_DISTRO_VERSION = "linux_distro_version" - PARAM_LINUX_DISTRO_LIKE = "linux_distro_like" - - PARAM_SCM_CLASS = "scm_class" - PARAM_IS_BINARY = "is_binary" - PARAM_CMD_CLASS = "cmd_class" - PARAM_CMD_RETURN_CODE = "cmd_return_code" - - def __init__(self, info=None): - from dvc.config import Config - from dvc.lock import Lock - - if info is None: - info = {} - - self.info = info - - cdir = Config.get_global_config_dir() - try: - os.makedirs(cdir) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - - self.user_id_file = os.path.join(cdir, self.USER_ID_FILE) - self.user_id_file_lock = Lock(self.user_id_file + ".lock") - - @staticmethod - def load(path): - """Loads analytics report from json file specified by path. - - Args: - path (str): path to json file with analytics report. - """ - with open(path, "r") as fobj: - analytics = Analytics(info=json.load(fobj)) - os.unlink(path) - return analytics - - def _write_user_id(self): - import uuid - - with open(self.user_id_file, "w+") as fobj: - user_id = str(uuid.uuid4()) - info = {self.PARAM_USER_ID: user_id} - json.dump(info, fobj) - return user_id - - def _read_user_id(self): - if not os.path.exists(self.user_id_file): - return None - - with open(self.user_id_file, "r") as fobj: - try: - info = json.load(fobj) - except ValueError as exc: - logger.debug("Failed to load user_id: {}".format(exc)) - return None - - return info[self.PARAM_USER_ID] - - def _get_user_id(self): - from dvc.lock import LockError - - try: - with self.user_id_file_lock: - user_id = self._read_user_id() - if user_id is None: - user_id = self._write_user_id() - return user_id - except LockError: - msg = "Failed to acquire '{}'" - logger.debug(msg.format(self.user_id_file_lock.lockfile)) - - def _collect_windows(self): - import sys - - version = sys.getwindowsversion() # pylint: disable=no-member - info = {} - info[self.PARAM_OS] = "windows" - info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major - info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor - info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build - info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack - return info - - def _collect_darwin(self): - import platform - - info = {} - info[self.PARAM_OS] = "mac" - info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0] - return info - - def _collect_linux(self): - import distro - - info = {} - info[self.PARAM_OS] = "linux" - info[self.PARAM_LINUX_DISTRO] = distro.id() - info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version() - info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like() - return info - - def _collect_system_info(self): - import platform - - system = platform.system() - - if system == "Windows": - return self._collect_windows() - - if system == "Darwin": - return self._collect_darwin() - - if system == "Linux": - return self._collect_linux() - - raise NotImplementedError - - def collect(self): - """Collect analytics report.""" - from dvc.scm import SCM - from dvc.utils import is_binary - from dvc.repo import Repo - from dvc.exceptions import NotDvcRepoError - - self.info[self.PARAM_DVC_VERSION] = __version__ - self.info[self.PARAM_IS_BINARY] = is_binary() - self.info[self.PARAM_USER_ID] = self._get_user_id() - - self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info() - - try: - scm = SCM(root_dir=Repo.find_root()) - self.info[self.PARAM_SCM_CLASS] = type(scm).__name__ - except NotDvcRepoError: - pass - - def collect_cmd(self, args, ret): - """Collect analytics info from a CLI command.""" - from dvc.command.daemon import CmdDaemonAnalytics - - assert isinstance(ret, int) or ret is None - - if ret is not None: - self.info[self.PARAM_CMD_RETURN_CODE] = ret - - if args is not None and hasattr(args, "func"): - assert args.func != CmdDaemonAnalytics - self.info[self.PARAM_CMD_CLASS] = args.func.__name__ - - def dump(self): - """Save analytics report to a temporary file. - - Returns: - str: path to the temporary file that contains the analytics report. - """ - import tempfile - - with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: - json.dump(self.info, fobj) - return fobj.name - - @staticmethod - def is_enabled(cmd=None): - from dvc.config import Config, to_bool - from dvc.command.daemon import CmdDaemonBase - - if env2bool("DVC_TEST"): - return False - - if isinstance(cmd, CmdDaemonBase): - return False - - core = Config(validate=False).config.get(Config.SECTION_CORE, {}) - enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true")) - logger.debug( - "Analytics is {}.".format("enabled" if enabled else "disabled") - ) - return enabled - - @staticmethod - def send_cmd(cmd, args, ret): - """Collect and send analytics for CLI command. - - Args: - args (list): parsed args for the CLI command. - ret (int): return value of the CLI command. - """ - from dvc.daemon import daemon - - if not Analytics.is_enabled(cmd): - return - - analytics = Analytics() - analytics.collect_cmd(args, ret) - daemon(["analytics", analytics.dump()]) - - def send(self): - """Collect and send analytics.""" - import requests - - if not self.is_enabled(): - return - - self.collect() - - logger.debug("Sending analytics: {}".format(self.info)) - - try: - requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST) - except requests.exceptions.RequestException as exc: - logger.debug("Failed to send analytics: {}".format(str(exc))) diff --git a/dvc/analytics/__init__.py b/dvc/analytics/__init__.py new file mode 100644 index 000000000..739ef253e --- /dev/null +++ b/dvc/analytics/__init__.py @@ -0,0 +1,64 @@ +import json +import logging +import requests +import subprocess +import tempfile + +from dvc import __version__ +from dvc.analytics import user_id, system_info +from dvc.config import Config, to_bool +from dvc.exceptions import NotDvcRepoError +from dvc.repo import Repo +from dvc.scm import SCM +from dvc.utils import env2bool, is_binary + + +logger = logging.getLogger(__name__) + + +def collect_and_send_report(arguments=None, exit_code=None): + report = { + "cmd_class": arguments.func.__name__, + "cmd_return_code": exit_code, + "dvc_version": __version__, + "is_binary": is_binary(), + "scm_class": scm_in_use(), + "system_info": system_info.collect(), + "user_id": user_id.find_or_create(), + } + + with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: + json.dump(report, fobj) + subprocess.Popen(["python", "-m", "dvc.analytics", fobj.name]) + + +def send(path): + url = "https://analytics.dvc.org" + + with open(path) as fobj: + report = json.load(fobj) + + requests.post(url, json=report, timeout=5) + + +def is_enabled(): + if env2bool("DVC_TEST"): + return False + + enabled = to_bool( + Config(validate=False) + .config.get(Config.SECTION_CORE, {}) + .get(Config.SECTION_CORE_ANALYTICS, "true") + ) + + logger.debug("Analytics is {}enabled.".format("" if enabled else "dis")) + + return enabled + + +def scm_in_use(): + try: + scm = SCM(root_dir=Repo.find_root()) + return type(scm).__name__ + except NotDvcRepoError: + pass diff --git a/dvc/analytics/__main__.py b/dvc/analytics/__main__.py new file mode 100644 index 000000000..13ebfa53b --- /dev/null +++ b/dvc/analytics/__main__.py @@ -0,0 +1,8 @@ +import sys + +from dvc.analytics import send + + +if __name__ == "__main__": + report_fname = sys.argv[1] + send(report_fname) diff --git a/dvc/analytics/system_info.py b/dvc/analytics/system_info.py new file mode 100644 index 000000000..666324cfd --- /dev/null +++ b/dvc/analytics/system_info.py @@ -0,0 +1,29 @@ +import platform +import sys +import distro + + +def collect(): + system = platform.system() + + if system == "Windows": + version = sys.getwindowsversion() + + return { + "os": "windows", + "windows_version_build": version.build, + "windows_version_major": version.major, + "windows_version_minor": version.minor, + "windows_version_service_pack": version.service_pack, + } + + if system == "Darwing": + return {"os": "mac", "mac_version": platform.mac_ver()[0]} + + if system == "Linux": + return { + "os": "linux", + "linux_distro": distro.id(), + "linux_distro_like": distro.like(), + "linux_distro_version": distro.version(), + } diff --git a/dvc/analytics/user_id.py b/dvc/analytics/user_id.py new file mode 100644 index 000000000..4c360cd98 --- /dev/null +++ b/dvc/analytics/user_id.py @@ -0,0 +1,55 @@ +""" +Interact with the user's ID stored under the global config directory. + +The file should contain a JSON with a "user_id" key: + + {"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"} + +IDs are generated randomly with UUID. +""" + +import json +import logging +import uuid + +from dvc.config import Config +from dvc.lock import Lock, LockError +from dvc.utils import makedirs + + +logger = logging.getLogger(__name__) + +config_dir = Config.get_global_config_dir() +fname = config_dir / "user_id" + + +def find_or_create(): + lockfile = fname.with_suffix(".lock") + + try: + with Lock(lockfile): + return _load() or _create() + except LockError: + logger.debug("Failed to acquire {lock}".format(lockfile)) + + +def _load(): + if not fname.exists: + return None + + with open(fname) as fobj: + try: + return json.load(fobj).get("user_id") + except json.JSONDecodeError: + pass + + +def _create(): + user_id = str(uuid.uuid4()) + + makedirs(fname.parent, exist_ok=True) + + with open(fname, "w") as fobj: + json.dump({"user_id": user_id}, fobj) + + return user_id diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index 2b60bded4..3953ab706 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -22,16 +22,6 @@ class CmdDaemonUpdater(CmdDaemonBase): return 0 -class CmdDaemonAnalytics(CmdDaemonBase): - def run(self): - from dvc.analytics import Analytics - - analytics = Analytics.load(self.args.target) - analytics.send() - - return 0 - - def add_parser(subparsers, parent_parser): DAEMON_HELP = "Service daemon." daemon_parser = subparsers.add_parser( @@ -55,13 +45,3 @@ def add_parser(subparsers, parent_parser): help=DAEMON_UPDATER_HELP, ) daemon_updater_parser.set_defaults(func=CmdDaemonUpdater) - - DAEMON_ANALYTICS_HELP = "Send dvc usage analytics." - daemon_analytics_parser = daemon_subparsers.add_parser( - "analytics", - parents=[parent_parser], - description=DAEMON_ANALYTICS_HELP, - help=DAEMON_ANALYTICS_HELP, - ) - daemon_analytics_parser.add_argument("target", help="Analytics file.") - daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics) diff --git a/dvc/config.py b/dvc/config.py index b883ef259..8e9b11c2b 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -13,8 +13,7 @@ from voluptuous import All, Any, Lower, Range, Coerce, Match from dvc.exceptions import DvcException from dvc.exceptions import NotDvcRepoError -from dvc.utils.compat import open -from dvc.utils.compat import str +from dvc.utils.compat import open, str, pathlib logger = logging.getLogger(__name__) @@ -285,8 +284,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes """ from appdirs import user_config_dir - return user_config_dir( - appname=Config.APPNAME, appauthor=Config.APPAUTHOR + return pathlib.Path( + user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) ) @staticmethod @@ -298,8 +297,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes """ from appdirs import site_config_dir - return site_config_dir( - appname=Config.APPNAME, appauthor=Config.APPAUTHOR + return pathlib.Path( + site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) ) @staticmethod diff --git a/dvc/lock.py b/dvc/lock.py index 147118c0f..7ffe54fc3 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -10,7 +10,7 @@ from funcy.py3 import lkeep from dvc.exceptions import DvcException from dvc.utils import makedirs -from dvc.utils.compat import is_py3 +from dvc.utils.compat import is_py3, str DEFAULT_TIMEOUT = 5 @@ -58,7 +58,7 @@ if is_py3: # [1] https://github.com/iterative/dvc/issues/2582 self._hostname = socket.gethostname() - self._lockfile = lockfile + self._lockfile = str(lockfile) self._lifetime = timedelta(days=365) # Lock for good by default self._separator = flufl.lock.SEP self._set_claimfile() @@ -111,7 +111,7 @@ else: """ def __init__(self, lockfile, tmp_dir=None): - self.lockfile = lockfile + self.lockfile = str(lockfile) self._lock = None @property diff --git a/dvc/main.py b/dvc/main.py index 660f9420d..86e1a9e0d 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import logging -from dvc.analytics import Analytics +from dvc import analytics from dvc.cli import parse_args from dvc.config import ConfigError from dvc.exceptions import DvcParserError @@ -81,6 +81,7 @@ def main(argv=None): if ret != 0: logger.info(FOOTER) - Analytics().send_cmd(cmd, args, ret) + if analytics.is_enabled(): + analytics.collect_and_send_report(args, ret) return ret diff --git a/dvc/repo/init.py b/dvc/repo/init.py index e06307e0a..5df1d0f1f 100644 --- a/dvc/repo/init.py +++ b/dvc/repo/init.py @@ -3,7 +3,7 @@ import os import colorama -from dvc.analytics import Analytics +from dvc import analytics from dvc.config import Config from dvc.exceptions import InitError from dvc.repo import Repo @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) def _welcome_message(): - if Analytics.is_enabled(): + if analytics.is_enabled(): logger.info( boxify( "DVC has enabled anonymous aggregate usage analytics.\n" diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py deleted file mode 100644 index b42a11727..000000000 --- a/tests/func/test_analytics.py +++ /dev/null @@ -1,65 +0,0 @@ -import os - -import mock -import requests - -from dvc.analytics import Analytics -from dvc.main import main -from tests.basic_env import TestDir -from tests.basic_env import TestDvc -from tests.basic_env import TestGit - - -def _clean_getenv(key, default=None): - """ - Remove env vars that affect dvc behavior in tests - """ - if key in ["DVC_TEST", "CI"]: - return None - return os.environ.get(key, default) - - -class TestAnalytics(TestDir): - def test(self): - a = Analytics() - a.collect() - self.assertTrue(isinstance(a.info, dict)) - self.assertNotEqual(a.info, {}) - self.assertTrue(a.PARAM_USER_ID in a.info.keys()) - self.assertTrue(a.PARAM_SYSTEM_INFO in a.info.keys()) - self.assertNotEqual(a.info[a.PARAM_SYSTEM_INFO], {}) - - @mock.patch.object(os, "getenv", new=_clean_getenv) - @mock.patch("requests.post") - def test_send(self, mockpost): - ret = main(["daemon", "analytics", Analytics().dump(), "-v"]) - self.assertEqual(ret, 0) - - self.assertTrue(mockpost.called) - - @mock.patch.object(os, "getenv", new=_clean_getenv) - @mock.patch.object( - requests, "post", side_effect=requests.exceptions.RequestException() - ) - def test_send_failed(self, mockpost): - ret = main(["daemon", "analytics", Analytics().dump(), "-v"]) - self.assertEqual(ret, 0) - - self.assertTrue(mockpost.called) - - -class TestAnalyticsGit(TestAnalytics, TestGit): - pass - - -class TestAnalyticsDvc(TestAnalytics, TestDvc): - @mock.patch("requests.post") - def test_send_disabled(self, mockpost): - ret = main(["config", "core.analytics", "false"]) - self.assertEqual(ret, 0) - - with mock.patch.object(os, "getenv", new=_clean_getenv): - ret = main(["daemon", "analytics", Analytics().dump(), "-v"]) - self.assertEqual(ret, 0) - - self.assertFalse(mockpost.called) diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 9e7e2dab1..2fbe134d4 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -1,6 +1,6 @@ import pytest -from dvc.analytics import Analytics +from dvc import analytics @pytest.mark.parametrize( @@ -21,4 +21,4 @@ def test_is_enabled(dvc_repo, config, result, monkeypatch): # reset DVC_TEST env var, which affects `is_enabled()` monkeypatch.delenv("DVC_TEST") - assert result == Analytics.is_enabled() + assert result == analytics.is_enabled() From b618d03cfd85ea03de564bbcba92556083374db1 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 25 Nov 2019 21:30:25 -0600 Subject: [PATCH 002/118] analytics: return OS when system info not supported --- dvc/analytics/system_info.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dvc/analytics/system_info.py b/dvc/analytics/system_info.py index 666324cfd..6fe149ba0 100644 --- a/dvc/analytics/system_info.py +++ b/dvc/analytics/system_info.py @@ -27,3 +27,5 @@ def collect(): "linux_distro_like": distro.like(), "linux_distro_version": distro.version(), } + + return {"os": system.lower()} From 7d3cbd789cbe561a526f14420ca503b2ae8a3393 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 25 Nov 2019 21:37:03 -0600 Subject: [PATCH 003/118] analytics: collect system info under the same module --- dvc/analytics/__init__.py | 36 ++++++++++++++++++++++++++++++++++-- dvc/analytics/system_info.py | 31 ------------------------------- 2 files changed, 34 insertions(+), 33 deletions(-) delete mode 100644 dvc/analytics/system_info.py diff --git a/dvc/analytics/__init__.py b/dvc/analytics/__init__.py index 739ef253e..0bb0c70b2 100644 --- a/dvc/analytics/__init__.py +++ b/dvc/analytics/__init__.py @@ -1,11 +1,15 @@ import json import logging +import platform import requests import subprocess +import sys import tempfile +import distro + from dvc import __version__ -from dvc.analytics import user_id, system_info +from dvc.analytics import user_id from dvc.config import Config, to_bool from dvc.exceptions import NotDvcRepoError from dvc.repo import Repo @@ -23,7 +27,7 @@ def collect_and_send_report(arguments=None, exit_code=None): "dvc_version": __version__, "is_binary": is_binary(), "scm_class": scm_in_use(), - "system_info": system_info.collect(), + "system_info": system_info(), "user_id": user_id.find_or_create(), } @@ -62,3 +66,31 @@ def scm_in_use(): return type(scm).__name__ except NotDvcRepoError: pass + + +def system_info(): + system = platform.system() + + if system == "Windows": + version = sys.getwindowsversion() + + return { + "os": "windows", + "windows_version_build": version.build, + "windows_version_major": version.major, + "windows_version_minor": version.minor, + "windows_version_service_pack": version.service_pack, + } + + if system == "Darwin": + return {"os": "mac", "mac_version": platform.mac_ver()[0]} + + if system == "Linux": + return { + "os": "linux", + "linux_distro": distro.id(), + "linux_distro_like": distro.like(), + "linux_distro_version": distro.version(), + } + + return {"os": system.lower()} diff --git a/dvc/analytics/system_info.py b/dvc/analytics/system_info.py deleted file mode 100644 index 6fe149ba0..000000000 --- a/dvc/analytics/system_info.py +++ /dev/null @@ -1,31 +0,0 @@ -import platform -import sys -import distro - - -def collect(): - system = platform.system() - - if system == "Windows": - version = sys.getwindowsversion() - - return { - "os": "windows", - "windows_version_build": version.build, - "windows_version_major": version.major, - "windows_version_minor": version.minor, - "windows_version_service_pack": version.service_pack, - } - - if system == "Darwing": - return {"os": "mac", "mac_version": platform.mac_ver()[0]} - - if system == "Linux": - return { - "os": "linux", - "linux_distro": distro.id(), - "linux_distro_like": distro.like(), - "linux_distro_version": distro.version(), - } - - return {"os": system.lower()} From 64d102caaa781caf3d08fc303c44e2d2c6880c9f Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 25 Nov 2019 21:58:08 -0600 Subject: [PATCH 004/118] analytics: move user_id module to a method shout-out to @Suor for the suggestion --- dvc/analytics/__init__.py | 36 ++++++++++++++++++++++--- dvc/analytics/user_id.py | 55 --------------------------------------- 2 files changed, 33 insertions(+), 58 deletions(-) delete mode 100644 dvc/analytics/user_id.py diff --git a/dvc/analytics/__init__.py b/dvc/analytics/__init__.py index 0bb0c70b2..48bced715 100644 --- a/dvc/analytics/__init__.py +++ b/dvc/analytics/__init__.py @@ -5,16 +5,17 @@ import requests import subprocess import sys import tempfile +import uuid import distro from dvc import __version__ -from dvc.analytics import user_id from dvc.config import Config, to_bool from dvc.exceptions import NotDvcRepoError +from dvc.lock import Lock, LockError from dvc.repo import Repo from dvc.scm import SCM -from dvc.utils import env2bool, is_binary +from dvc.utils import env2bool, is_binary, makedirs logger = logging.getLogger(__name__) @@ -28,7 +29,7 @@ def collect_and_send_report(arguments=None, exit_code=None): "is_binary": is_binary(), "scm_class": scm_in_use(), "system_info": system_info(), - "user_id": user_id.find_or_create(), + "user_id": find_or_create_user_id(), } with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: @@ -94,3 +95,32 @@ def system_info(): } return {"os": system.lower()} + + +def find_or_create_user_id(): + """ + The user's ID is stored on a file under the global config directory. + + The file should contain a JSON with a "user_id" key: + + {"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"} + + IDs are generated randomly with UUID. + """ + config_dir = Config.get_global_config_dir() + fname = config_dir / "user_id" + lockfile = fname.with_suffix(".lock") + + try: + with Lock(lockfile): + try: + user_id = json.load(fname.read_text())["user_id"] + except (FileNotFoundError, json.JSONDecodeError, AttributeError): + user_id = str(uuid.uuid4()) + makedirs(fname.parent, exist_ok=True) + fname.write_text(json.dumps({"user_id": user_id})) + + return user_id + + except LockError: + logger.debug("Failed to acquire {lock}".format(lockfile)) diff --git a/dvc/analytics/user_id.py b/dvc/analytics/user_id.py deleted file mode 100644 index 4c360cd98..000000000 --- a/dvc/analytics/user_id.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Interact with the user's ID stored under the global config directory. - -The file should contain a JSON with a "user_id" key: - - {"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"} - -IDs are generated randomly with UUID. -""" - -import json -import logging -import uuid - -from dvc.config import Config -from dvc.lock import Lock, LockError -from dvc.utils import makedirs - - -logger = logging.getLogger(__name__) - -config_dir = Config.get_global_config_dir() -fname = config_dir / "user_id" - - -def find_or_create(): - lockfile = fname.with_suffix(".lock") - - try: - with Lock(lockfile): - return _load() or _create() - except LockError: - logger.debug("Failed to acquire {lock}".format(lockfile)) - - -def _load(): - if not fname.exists: - return None - - with open(fname) as fobj: - try: - return json.load(fobj).get("user_id") - except json.JSONDecodeError: - pass - - -def _create(): - user_id = str(uuid.uuid4()) - - makedirs(fname.parent, exist_ok=True) - - with open(fname, "w") as fobj: - json.dump({"user_id": user_id}, fobj) - - return user_id From 2e7a6d3fe5ec7e0a6b23f332dd2de7831aa9f566 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 25 Nov 2019 21:59:41 -0600 Subject: [PATCH 005/118] :nail_care: sort methods --- dvc/analytics/__init__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dvc/analytics/__init__.py b/dvc/analytics/__init__.py index 48bced715..23211bb3f 100644 --- a/dvc/analytics/__init__.py +++ b/dvc/analytics/__init__.py @@ -37,15 +37,6 @@ def collect_and_send_report(arguments=None, exit_code=None): subprocess.Popen(["python", "-m", "dvc.analytics", fobj.name]) -def send(path): - url = "https://analytics.dvc.org" - - with open(path) as fobj: - report = json.load(fobj) - - requests.post(url, json=report, timeout=5) - - def is_enabled(): if env2bool("DVC_TEST"): return False @@ -61,6 +52,15 @@ def is_enabled(): return enabled +def send(path): + url = "https://analytics.dvc.org" + + with open(path) as fobj: + report = json.load(fobj) + + requests.post(url, json=report, timeout=5) + + def scm_in_use(): try: scm = SCM(root_dir=Repo.find_root()) From 5cb437b5efccfe06c1e392d1193c8e486c799a3e Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 25 Nov 2019 22:03:54 -0600 Subject: [PATCH 006/118] analytics: move to single file shout-out to @Suor --- dvc/{analytics/__init__.py => analytics.py} | 0 dvc/analytics/__main__.py | 8 -------- 2 files changed, 8 deletions(-) rename dvc/{analytics/__init__.py => analytics.py} (100%) delete mode 100644 dvc/analytics/__main__.py diff --git a/dvc/analytics/__init__.py b/dvc/analytics.py similarity index 100% rename from dvc/analytics/__init__.py rename to dvc/analytics.py diff --git a/dvc/analytics/__main__.py b/dvc/analytics/__main__.py deleted file mode 100644 index 13ebfa53b..000000000 --- a/dvc/analytics/__main__.py +++ /dev/null @@ -1,8 +0,0 @@ -import sys - -from dvc.analytics import send - - -if __name__ == "__main__": - report_fname = sys.argv[1] - send(report_fname) From 610d1ee28e57d854fd80756e6ece536b0abd9a87 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 25 Nov 2019 22:10:32 -0600 Subject: [PATCH 007/118] analytics: go back to daemon implementation Lack of a better way to spawn a detached process on both windows an linux --- dvc/analytics.py | 11 ++++++++--- dvc/command/daemon.py | 23 ++++++++++++++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 23211bb3f..2a8190773 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -2,7 +2,6 @@ import json import logging import platform import requests -import subprocess import sys import tempfile import uuid @@ -11,17 +10,23 @@ import distro from dvc import __version__ from dvc.config import Config, to_bool +from dvc.daemon import daemon from dvc.exceptions import NotDvcRepoError from dvc.lock import Lock, LockError from dvc.repo import Repo from dvc.scm import SCM from dvc.utils import env2bool, is_binary, makedirs - logger = logging.getLogger(__name__) def collect_and_send_report(arguments=None, exit_code=None): + """ + Query the system to fill a report and send it on a detached process. + + A temporary file is used as a mean of communication between the + current and detached process. + """ report = { "cmd_class": arguments.func.__name__, "cmd_return_code": exit_code, @@ -34,7 +39,7 @@ def collect_and_send_report(arguments=None, exit_code=None): with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: json.dump(report, fobj) - subprocess.Popen(["python", "-m", "dvc.analytics", fobj.name]) + daemon(["analytics", fobj.name]) def is_enabled(): diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index 3953ab706..4fce0649b 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import os + from dvc.command.base import CmdBaseNoRepo from dvc.command.base import fix_subparsers @@ -10,7 +12,6 @@ class CmdDaemonBase(CmdBaseNoRepo): class CmdDaemonUpdater(CmdDaemonBase): def run(self): - import os from dvc.repo import Repo from dvc.updater import Updater @@ -22,6 +23,16 @@ class CmdDaemonUpdater(CmdDaemonBase): return 0 +class CmdDaemonAnalytics(CmdDaemonBase): + def run(self): + from dvc import analytics + + analytics.send(self.args.target) + os.path.remove(self.args.target) + + return 0 + + def add_parser(subparsers, parent_parser): DAEMON_HELP = "Service daemon." daemon_parser = subparsers.add_parser( @@ -45,3 +56,13 @@ def add_parser(subparsers, parent_parser): help=DAEMON_UPDATER_HELP, ) daemon_updater_parser.set_defaults(func=CmdDaemonUpdater) + + DAEMON_ANALYTICS_HELP = "Send dvc usage analytics." + daemon_analytics_parser = daemon_subparsers.add_parser( + "analytics", + parents=[parent_parser], + description=DAEMON_ANALYTICS_HELP, + help=DAEMON_ANALYTICS_HELP, + ) + daemon_analytics_parser.add_argument("target", help="Analytics file.") + daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics) From d958870edfddb49de17480172021063a5fc2c7fd Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Tue, 26 Nov 2019 10:45:19 -0600 Subject: [PATCH 008/118] analytics: exit_code -> return_code shout-out to @efiop for the suggestion --- dvc/analytics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 2a8190773..75ce5c58c 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -20,7 +20,7 @@ from dvc.utils import env2bool, is_binary, makedirs logger = logging.getLogger(__name__) -def collect_and_send_report(arguments=None, exit_code=None): +def collect_and_send_report(arguments=None, return_code=None): """ Query the system to fill a report and send it on a detached process. @@ -29,7 +29,7 @@ def collect_and_send_report(arguments=None, exit_code=None): """ report = { "cmd_class": arguments.func.__name__, - "cmd_return_code": exit_code, + "cmd_return_code": return_code, "dvc_version": __version__, "is_binary": is_binary(), "scm_class": scm_in_use(), From 34d3004779e279e568762d15e974b10c848e4ee0 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Tue, 26 Nov 2019 12:07:18 -0600 Subject: [PATCH 009/118] :face_palm: fix wrong module --- dvc/command/daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index 4fce0649b..3f5f545a1 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -28,7 +28,7 @@ class CmdDaemonAnalytics(CmdDaemonBase): from dvc import analytics analytics.send(self.args.target) - os.path.remove(self.args.target) + os.remove(self.args.target) return 0 From 5261ed7148e8877fdf3518e96bc846f0eb542306 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Tue, 26 Nov 2019 13:40:11 -0600 Subject: [PATCH 010/118] tests: analytics.find_or_create_user_id --- dvc/analytics.py | 2 +- tests/func/test_analytics.py | 1 + tests/unit/test_analytics.py | 11 +++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/func/test_analytics.py diff --git a/dvc/analytics.py b/dvc/analytics.py index 75ce5c58c..e17b6291c 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -119,7 +119,7 @@ def find_or_create_user_id(): try: with Lock(lockfile): try: - user_id = json.load(fname.read_text())["user_id"] + user_id = json.loads(fname.read_text())["user_id"] except (FileNotFoundError, json.JSONDecodeError, AttributeError): user_id = str(uuid.uuid4()) makedirs(fname.parent, exist_ok=True) diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/tests/func/test_analytics.py @@ -0,0 +1 @@ + diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 2fbe134d4..302bff381 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -1,4 +1,5 @@ import pytest +import mock from dvc import analytics @@ -22,3 +23,13 @@ def test_is_enabled(dvc_repo, config, result, monkeypatch): monkeypatch.delenv("DVC_TEST") assert result == analytics.is_enabled() + + +def test_find_or_create_user_id(tmp_path): + with mock.patch( + "dvc.config.Config.get_global_config_dir", return_value=tmp_path + ): + created = analytics.find_or_create_user_id() + found = analytics.find_or_create_user_id() + + assert created == found From 467b110929a7816ed3ca01d5d05b8669fdd54011 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Tue, 26 Nov 2019 18:32:02 -0600 Subject: [PATCH 011/118] py2: FileNotFoundError -> IOError --- dvc/analytics.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index e17b6291c..215c0c13f 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -5,6 +5,7 @@ import requests import sys import tempfile import uuid +import errno import distro @@ -120,7 +121,9 @@ def find_or_create_user_id(): with Lock(lockfile): try: user_id = json.loads(fname.read_text())["user_id"] - except (FileNotFoundError, json.JSONDecodeError, AttributeError): + except (IOError, json.JSONDecodeError, AttributeError) as exc: + if exc.errno != errno.ENOENT: + raise user_id = str(uuid.uuid4()) makedirs(fname.parent, exist_ok=True) fname.write_text(json.dumps({"user_id": user_id})) From 48b4cbad45f25589a1d5e9086a19119fd00cfad7 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Tue, 26 Nov 2019 18:32:39 -0600 Subject: [PATCH 012/118] :nail_care: change naming and docstring --- dvc/analytics.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 215c0c13f..0bcf31527 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -18,18 +18,16 @@ from dvc.repo import Repo from dvc.scm import SCM from dvc.utils import env2bool, is_binary, makedirs + logger = logging.getLogger(__name__) -def collect_and_send_report(arguments=None, return_code=None): +def collect_and_send_report(args=None, return_code=None): """ Query the system to fill a report and send it on a detached process. - - A temporary file is used as a mean of communication between the - current and detached process. """ report = { - "cmd_class": arguments.func.__name__, + "cmd_class": args.func.__name__ if hasattr(args, "func") else None, "cmd_return_code": return_code, "dvc_version": __version__, "is_binary": is_binary(), @@ -38,6 +36,8 @@ def collect_and_send_report(arguments=None, return_code=None): "user_id": find_or_create_user_id(), } + # A temporary file is used as a mean of communication between the + # current and detached process. with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: json.dump(report, fobj) daemon(["analytics", fobj.name]) From 6bc24039db8178dda1b77bf307ed22b079d725e6 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Tue, 26 Nov 2019 18:33:16 -0600 Subject: [PATCH 013/118] :nail_care: black --- tests/func/test_analytics.py | 1 - tests/unit/test_analytics.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 tests/func/test_analytics.py diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py deleted file mode 100644 index 8b1378917..000000000 --- a/tests/func/test_analytics.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 302bff381..5012d1617 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -32,4 +32,4 @@ def test_find_or_create_user_id(tmp_path): created = analytics.find_or_create_user_id() found = analytics.find_or_create_user_id() - assert created == found + assert created == found From f9952046fd376579fbb37906a402d9ab00a6fb2a Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Tue, 26 Nov 2019 19:29:04 -0600 Subject: [PATCH 014/118] tests: functional and unit tests for analytics --- dvc/analytics.py | 13 +++---------- dvc/main.py | 10 +++++++++- tests/func/test_analytics.py | 14 ++++++++++++++ tests/unit/test_analytics.py | 24 ++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 11 deletions(-) create mode 100644 tests/func/test_analytics.py diff --git a/dvc/analytics.py b/dvc/analytics.py index 0bcf31527..ebdfd9baf 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -11,7 +11,6 @@ import distro from dvc import __version__ from dvc.config import Config, to_bool -from dvc.daemon import daemon from dvc.exceptions import NotDvcRepoError from dvc.lock import Lock, LockError from dvc.repo import Repo @@ -22,11 +21,11 @@ from dvc.utils import env2bool, is_binary, makedirs logger = logging.getLogger(__name__) -def collect_and_send_report(args=None, return_code=None): +def collect(args=None, return_code=None): """ - Query the system to fill a report and send it on a detached process. + Query the system to fill a report. """ - report = { + return { "cmd_class": args.func.__name__ if hasattr(args, "func") else None, "cmd_return_code": return_code, "dvc_version": __version__, @@ -36,12 +35,6 @@ def collect_and_send_report(args=None, return_code=None): "user_id": find_or_create_user_id(), } - # A temporary file is used as a mean of communication between the - # current and detached process. - with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: - json.dump(report, fobj) - daemon(["analytics", fobj.name]) - def is_enabled(): if env2bool("DVC_TEST"): diff --git a/dvc/main.py b/dvc/main.py index 86e1a9e0d..566a46473 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -2,10 +2,12 @@ from __future__ import unicode_literals import logging +import json from dvc import analytics from dvc.cli import parse_args from dvc.config import ConfigError +from dvc.daemon import daemon from dvc.exceptions import DvcParserError from dvc.exceptions import NotDvcRepoError from dvc.external_repo import clean_repos @@ -82,6 +84,12 @@ def main(argv=None): logger.info(FOOTER) if analytics.is_enabled(): - analytics.collect_and_send_report(args, ret) + report = analytics.collect(args, ret) + + # Use a temporary file to pass the report between the current and + # detached process. + with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: + json.dump(report, fobj) + daemon(["analytics", fobj.name]) return ret diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py new file mode 100644 index 000000000..608fea00b --- /dev/null +++ b/tests/func/test_analytics.py @@ -0,0 +1,14 @@ +import mock +import json + +from dvc.main import main + + +@mock.patch("dvc.analytics.send") +def test_daemon_analytics(mock_send, tmp_path): + report = {"name": "dummy report"} + fname = tmp_path / "report" + fname.write_text(json.dumps(report)) + + assert 0 == main(["daemon", "analytics", str(fname)]) + assert mock_send.called diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 5012d1617..3800cf0ef 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -1,9 +1,33 @@ +import json import pytest import mock from dvc import analytics +def test_collect(): + report = analytics.collect(return_code=0) + + assert report["cmd_class"] == None + assert report["cmd_return_code"] == 0 + assert report["scm_class"] in ["Git", None] + assert type(report["is_binary"]) is bool + assert type(report["system_info"]) is dict + assert type(report["dvc_version"]) is str + assert type(report["user_id"]) is str + + +@mock.patch("requests.post") +def test_send(mock_post, tmp_path): + url = "https://analytics.dvc.org" + report = {"name": "dummy report"} + fname = tmp_path / "report" + + fname.write_text(json.dumps(report)) + analytics.send(fname) + mock_post.assert_called_with(url, json=report, timeout=5) + + @pytest.mark.parametrize( "config, result", [ From 2a5cac55700878a68f31e1e18ab854ed9a82e47d Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Tue, 26 Nov 2019 19:32:26 -0600 Subject: [PATCH 015/118] :nail_care: deepsource --- dvc/analytics.py | 3 +-- dvc/main.py | 1 + tests/unit/test_analytics.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index ebdfd9baf..4af122a9a 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -3,7 +3,6 @@ import logging import platform import requests import sys -import tempfile import uuid import errno @@ -124,4 +123,4 @@ def find_or_create_user_id(): return user_id except LockError: - logger.debug("Failed to acquire {lock}".format(lockfile)) + logger.debug("Failed to acquire {lockfile}".format(lockfile=lockfile)) diff --git a/dvc/main.py b/dvc/main.py index 566a46473..07080ae60 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import logging import json +import tempfile from dvc import analytics from dvc.cli import parse_args diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 3800cf0ef..e8acf1c5f 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -8,9 +8,9 @@ from dvc import analytics def test_collect(): report = analytics.collect(return_code=0) - assert report["cmd_class"] == None assert report["cmd_return_code"] == 0 assert report["scm_class"] in ["Git", None] + assert not report["cmd_class"] assert type(report["is_binary"]) is bool assert type(report["system_info"]) is dict assert type(report["dvc_version"]) is str From e9d56a969e9851d07b3c6a7e7a318970d701a418 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Tue, 26 Nov 2019 19:39:20 -0600 Subject: [PATCH 016/118] :nail_care: sort imports --- dvc/analytics.py | 2 +- dvc/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 4af122a9a..ac0a0a798 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -1,10 +1,10 @@ +import errno import json import logging import platform import requests import sys import uuid -import errno import distro diff --git a/dvc/main.py b/dvc/main.py index 07080ae60..2b74fe6e9 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -1,8 +1,8 @@ """Main entry point for dvc CLI.""" from __future__ import unicode_literals -import logging import json +import logging import tempfile from dvc import analytics From 787ea9cd31993458d0380f05bcf0a68bf68dde09 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Wed, 27 Nov 2019 12:37:11 -0600 Subject: [PATCH 017/118] tests: set temporary global config --- tests/unit/test_analytics.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index e8acf1c5f..2c443095c 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -5,7 +5,15 @@ import mock from dvc import analytics -def test_collect(): +@pytest.fixture +def tmp_global_config(tmp_path): + with mock.patch( + "dvc.config.Config.get_global_config_dir", return_value=tmp_path + ): + yield + + +def test_collect(tmp_global_config): report = analytics.collect(return_code=0) assert report["cmd_return_code"] == 0 @@ -49,11 +57,8 @@ def test_is_enabled(dvc_repo, config, result, monkeypatch): assert result == analytics.is_enabled() -def test_find_or_create_user_id(tmp_path): - with mock.patch( - "dvc.config.Config.get_global_config_dir", return_value=tmp_path - ): - created = analytics.find_or_create_user_id() - found = analytics.find_or_create_user_id() +def test_find_or_create_user_id(tmp_global_config): + created = analytics.find_or_create_user_id() + found = analytics.find_or_create_user_id() assert created == found From e69fffa7c74b5878dbbe257b74cb0cdee5ea0776 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Wed, 27 Nov 2019 18:36:53 -0600 Subject: [PATCH 018/118] py2 compat issues :shrug: --- dvc/analytics.py | 10 ++++------ dvc/config.py | 9 ++------- tests/func/test_analytics.py | 3 ++- tests/unit/test_analytics.py | 5 +++-- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index ac0a0a798..556dafe57 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -1,4 +1,3 @@ -import errno import json import logging import platform @@ -15,6 +14,7 @@ from dvc.lock import Lock, LockError from dvc.repo import Repo from dvc.scm import SCM from dvc.utils import env2bool, is_binary, makedirs +from dvc.utils.compat import str, FileNotFoundError logger = logging.getLogger(__name__) @@ -53,7 +53,7 @@ def is_enabled(): def send(path): url = "https://analytics.dvc.org" - with open(path) as fobj: + with open(str(path)) as fobj: report = json.load(fobj) requests.post(url, json=report, timeout=5) @@ -113,12 +113,10 @@ def find_or_create_user_id(): with Lock(lockfile): try: user_id = json.loads(fname.read_text())["user_id"] - except (IOError, json.JSONDecodeError, AttributeError) as exc: - if exc.errno != errno.ENOENT: - raise + except (FileNotFoundError, ValueError, AttributeError): user_id = str(uuid.uuid4()) makedirs(fname.parent, exist_ok=True) - fname.write_text(json.dumps({"user_id": user_id})) + fname.write_text(str(json.dumps({"user_id": user_id}))) return user_id diff --git a/dvc/config.py b/dvc/config.py index 8e9b11c2b..76de6cf84 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -342,13 +342,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes return ret def _load_configs(self): - system_config_file = os.path.join( - self.get_system_config_dir(), self.CONFIG - ) - - global_config_file = os.path.join( - self.get_global_config_dir(), self.CONFIG - ) + system_config_file = str(self.get_system_config_dir() / self.CONFIG) + global_config_file = str(self.get_global_config_dir() / self.CONFIG) self._system_config = configobj.ConfigObj(system_config_file) self._global_config = configobj.ConfigObj(global_config_file) diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py index 608fea00b..bd698703d 100644 --- a/tests/func/test_analytics.py +++ b/tests/func/test_analytics.py @@ -2,13 +2,14 @@ import mock import json from dvc.main import main +from dvc.utils.compat import str @mock.patch("dvc.analytics.send") def test_daemon_analytics(mock_send, tmp_path): report = {"name": "dummy report"} fname = tmp_path / "report" - fname.write_text(json.dumps(report)) + fname.write_text(str(json.dumps(report))) assert 0 == main(["daemon", "analytics", str(fname)]) assert mock_send.called diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 2c443095c..b0f3affaa 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -3,6 +3,7 @@ import pytest import mock from dvc import analytics +from dvc.utils.compat import builtin_str, str @pytest.fixture @@ -21,7 +22,7 @@ def test_collect(tmp_global_config): assert not report["cmd_class"] assert type(report["is_binary"]) is bool assert type(report["system_info"]) is dict - assert type(report["dvc_version"]) is str + assert type(report["dvc_version"]) is builtin_str assert type(report["user_id"]) is str @@ -31,7 +32,7 @@ def test_send(mock_post, tmp_path): report = {"name": "dummy report"} fname = tmp_path / "report" - fname.write_text(json.dumps(report)) + fname.write_text(str(json.dumps(report))) analytics.send(fname) mock_post.assert_called_with(url, json=report, timeout=5) From 656f986542afe332d5102d9a54e38387139e61d2 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Thu, 28 Nov 2019 17:47:14 -0600 Subject: [PATCH 019/118] :nail_care: correct wording "disenabled" -> "disabled" --- dvc/analytics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 556dafe57..a4c321a18 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -45,7 +45,7 @@ def is_enabled(): .get(Config.SECTION_CORE_ANALYTICS, "true") ) - logger.debug("Analytics is {}enabled.".format("" if enabled else "dis")) + logger.debug("Analytics is {}abled.".format("en" if enabled else "dis")) return enabled From 946dfc14004a1dd17241c30795e48d36fce4c615 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 2 Dec 2019 17:35:21 -0600 Subject: [PATCH 020/118] analytics: send report without loading it to memory shout-out to @Suor for the suggestion --- dvc/analytics.py | 5 ++--- tests/unit/test_analytics.py | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index a4c321a18..d658c9191 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -52,11 +52,10 @@ def is_enabled(): def send(path): url = "https://analytics.dvc.org" + headers = {"content-type": "application/json"} with open(str(path)) as fobj: - report = json.load(fobj) - - requests.post(url, json=report, timeout=5) + requests.post(url, data=fobj, headers=headers, timeout=5) def scm_in_use(): diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index b0f3affaa..e48e4972d 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -34,7 +34,8 @@ def test_send(mock_post, tmp_path): fname.write_text(str(json.dumps(report))) analytics.send(fname) - mock_post.assert_called_with(url, json=report, timeout=5) + assert mock_post.called + assert mock_post.call_args.args[0] == url @pytest.mark.parametrize( From 55960154c54338dbe68e547a3df39b7927f96175 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 2 Dec 2019 17:35:50 -0600 Subject: [PATCH 021/118] analytics: document why tmp_global_config is needed --- tests/unit/test_analytics.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index e48e4972d..b54b8132e 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -8,6 +8,9 @@ from dvc.utils.compat import builtin_str, str @pytest.fixture def tmp_global_config(tmp_path): + """ + Fixture to prevent modifying the actual global config + """ with mock.patch( "dvc.config.Config.get_global_config_dir", return_value=tmp_path ): From 2c8e1495b86e8cce017a831f9d87df016a421658 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 2 Dec 2019 21:27:31 -0600 Subject: [PATCH 022/118] analytics: use fspath instead of str --- dvc/analytics.py | 4 ++-- dvc/config.py | 6 +++--- dvc/lock.py | 6 +++--- tests/func/test_analytics.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index d658c9191..7939ba5c3 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -14,7 +14,7 @@ from dvc.lock import Lock, LockError from dvc.repo import Repo from dvc.scm import SCM from dvc.utils import env2bool, is_binary, makedirs -from dvc.utils.compat import str, FileNotFoundError +from dvc.utils.compat import str, FileNotFoundError, fspath_py35 logger = logging.getLogger(__name__) @@ -54,7 +54,7 @@ def send(path): url = "https://analytics.dvc.org" headers = {"content-type": "application/json"} - with open(str(path)) as fobj: + with open(fspath_py35(path)) as fobj: requests.post(url, data=fobj, headers=headers, timeout=5) diff --git a/dvc/config.py b/dvc/config.py index 76de6cf84..32c7a0977 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -13,7 +13,7 @@ from voluptuous import All, Any, Lower, Range, Coerce, Match from dvc.exceptions import DvcException from dvc.exceptions import NotDvcRepoError -from dvc.utils.compat import open, str, pathlib +from dvc.utils.compat import open, str, pathlib, fspath logger = logging.getLogger(__name__) @@ -342,8 +342,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes return ret def _load_configs(self): - system_config_file = str(self.get_system_config_dir() / self.CONFIG) - global_config_file = str(self.get_global_config_dir() / self.CONFIG) + system_config_file = fspath(self.get_system_config_dir() / self.CONFIG) + global_config_file = fspath(self.get_global_config_dir() / self.CONFIG) self._system_config = configobj.ConfigObj(system_config_file) self._global_config = configobj.ConfigObj(global_config_file) diff --git a/dvc/lock.py b/dvc/lock.py index 7ffe54fc3..1f3df8ebb 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -10,7 +10,7 @@ from funcy.py3 import lkeep from dvc.exceptions import DvcException from dvc.utils import makedirs -from dvc.utils.compat import is_py3, str +from dvc.utils.compat import is_py3, str, fspath DEFAULT_TIMEOUT = 5 @@ -58,7 +58,7 @@ if is_py3: # [1] https://github.com/iterative/dvc/issues/2582 self._hostname = socket.gethostname() - self._lockfile = str(lockfile) + self._lockfile = fspath(lockfile) self._lifetime = timedelta(days=365) # Lock for good by default self._separator = flufl.lock.SEP self._set_claimfile() @@ -111,7 +111,7 @@ else: """ def __init__(self, lockfile, tmp_dir=None): - self.lockfile = str(lockfile) + self.lockfile = fspath(lockfile) self._lock = None @property diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py index bd698703d..97a71d6d0 100644 --- a/tests/func/test_analytics.py +++ b/tests/func/test_analytics.py @@ -2,7 +2,7 @@ import mock import json from dvc.main import main -from dvc.utils.compat import str +from dvc.utils.compat import fspath @mock.patch("dvc.analytics.send") @@ -11,5 +11,5 @@ def test_daemon_analytics(mock_send, tmp_path): fname = tmp_path / "report" fname.write_text(str(json.dumps(report))) - assert 0 == main(["daemon", "analytics", str(fname)]) + assert 0 == main(["daemon", "analytics", fspath(fname)]) assert mock_send.called From ca485db24040587074f4dd7d4b101fe6ab585236 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 2 Dec 2019 21:36:57 -0600 Subject: [PATCH 023/118] analytics: define report schema and use it on tests --- dvc/analytics.py | 13 +++++++++++++ tests/unit/test_analytics.py | 9 +-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 7939ba5c3..2d0d328e7 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -6,6 +6,7 @@ import sys import uuid import distro +from voluptuous import Schema, Any from dvc import __version__ from dvc.config import Config, to_bool @@ -19,6 +20,18 @@ from dvc.utils.compat import str, FileNotFoundError, fspath_py35 logger = logging.getLogger(__name__) +report_schema = Schema( + { + "cmd_class": Any(str, None), + "cmd_return_code": Any(int, None), + "dvc_version": str, + "is_binary": bool, + "scm_class": Any("Git", None), + "user_id": str, + "system_info": dict, + } +) + def collect(args=None, return_code=None): """ diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index b54b8132e..953a681a3 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -19,14 +19,7 @@ def tmp_global_config(tmp_path): def test_collect(tmp_global_config): report = analytics.collect(return_code=0) - - assert report["cmd_return_code"] == 0 - assert report["scm_class"] in ["Git", None] - assert not report["cmd_class"] - assert type(report["is_binary"]) is bool - assert type(report["system_info"]) is dict - assert type(report["dvc_version"]) is builtin_str - assert type(report["user_id"]) is str + assert analytics.report_schema(report) @mock.patch("requests.post") From e0d47a138221a78a8dd05dbf7ec9927c0312baed Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 2 Dec 2019 21:57:06 -0600 Subject: [PATCH 024/118] :nail_care: formatting --- dvc/analytics.py | 6 +++--- dvc/lock.py | 2 +- tests/unit/test_analytics.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 2d0d328e7..4a4a4060b 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -15,7 +15,7 @@ from dvc.lock import Lock, LockError from dvc.repo import Repo from dvc.scm import SCM from dvc.utils import env2bool, is_binary, makedirs -from dvc.utils.compat import str, FileNotFoundError, fspath_py35 +from dvc.utils.compat import builtin_str, str, FileNotFoundError, fspath_py35 logger = logging.getLogger(__name__) @@ -24,10 +24,10 @@ report_schema = Schema( { "cmd_class": Any(str, None), "cmd_return_code": Any(int, None), - "dvc_version": str, + "dvc_version": Any(builtin_str, str), "is_binary": bool, "scm_class": Any("Git", None), - "user_id": str, + "user_id": Any(builtin_str, str), "system_info": dict, } ) diff --git a/dvc/lock.py b/dvc/lock.py index 1f3df8ebb..d4925e963 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -10,7 +10,7 @@ from funcy.py3 import lkeep from dvc.exceptions import DvcException from dvc.utils import makedirs -from dvc.utils.compat import is_py3, str, fspath +from dvc.utils.compat import is_py3, fspath DEFAULT_TIMEOUT = 5 diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 953a681a3..e62c6e7b5 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -3,7 +3,7 @@ import pytest import mock from dvc import analytics -from dvc.utils.compat import builtin_str, str +from dvc.utils.compat import str @pytest.fixture From 8b5ca253ec376aa3be484ba376236a218ba38368 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 2 Dec 2019 22:47:31 -0600 Subject: [PATCH 025/118] analytics: move report schema to tests --- dvc/analytics.py | 15 +-------------- tests/unit/test_analytics.py | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 4a4a4060b..7939ba5c3 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -6,7 +6,6 @@ import sys import uuid import distro -from voluptuous import Schema, Any from dvc import __version__ from dvc.config import Config, to_bool @@ -15,23 +14,11 @@ from dvc.lock import Lock, LockError from dvc.repo import Repo from dvc.scm import SCM from dvc.utils import env2bool, is_binary, makedirs -from dvc.utils.compat import builtin_str, str, FileNotFoundError, fspath_py35 +from dvc.utils.compat import str, FileNotFoundError, fspath_py35 logger = logging.getLogger(__name__) -report_schema = Schema( - { - "cmd_class": Any(str, None), - "cmd_return_code": Any(int, None), - "dvc_version": Any(builtin_str, str), - "is_binary": bool, - "scm_class": Any("Git", None), - "user_id": Any(builtin_str, str), - "system_info": dict, - } -) - def collect(args=None, return_code=None): """ diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index e62c6e7b5..7a864bfa3 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -2,8 +2,10 @@ import json import pytest import mock +from voluptuous import Schema, Any + from dvc import analytics -from dvc.utils.compat import str +from dvc.utils.compat import str, builtin_str @pytest.fixture @@ -18,8 +20,20 @@ def tmp_global_config(tmp_path): def test_collect(tmp_global_config): + schema = Schema( + { + "cmd_class": Any(str, None), + "cmd_return_code": Any(int, None), + "dvc_version": Any(builtin_str, str), + "is_binary": bool, + "scm_class": Any("Git", None), + "user_id": Any(builtin_str, str), + "system_info": dict, + } + ) + report = analytics.collect(return_code=0) - assert analytics.report_schema(report) + assert schema(report) @mock.patch("requests.post") From 8a32b28b543abb0eae550e0f28fefe6ec1cc06d0 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 2 Dec 2019 22:54:16 -0600 Subject: [PATCH 026/118] analytics: collect and send on daemon --- dvc/analytics.py | 13 +++++-------- dvc/command/daemon.py | 14 ++++++++++---- dvc/main.py | 12 +++--------- tests/func/test_analytics.py | 8 +------- tests/unit/test_analytics.py | 8 ++------ 5 files changed, 21 insertions(+), 34 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 7939ba5c3..6ea473571 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -14,18 +14,18 @@ from dvc.lock import Lock, LockError from dvc.repo import Repo from dvc.scm import SCM from dvc.utils import env2bool, is_binary, makedirs -from dvc.utils.compat import str, FileNotFoundError, fspath_py35 +from dvc.utils.compat import str, FileNotFoundError logger = logging.getLogger(__name__) -def collect(args=None, return_code=None): +def collect(cmd_class=None, return_code=None): """ Query the system to fill a report. """ return { - "cmd_class": args.func.__name__ if hasattr(args, "func") else None, + "cmd_class": cmd_class, "cmd_return_code": return_code, "dvc_version": __version__, "is_binary": is_binary(), @@ -50,12 +50,9 @@ def is_enabled(): return enabled -def send(path): +def send(report): url = "https://analytics.dvc.org" - headers = {"content-type": "application/json"} - - with open(fspath_py35(path)) as fobj: - requests.post(url, data=fobj, headers=headers, timeout=5) + requests.post(url, json=report, timeout=5) def scm_in_use(): diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index 3f5f545a1..6bcb15f54 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -25,10 +25,11 @@ class CmdDaemonUpdater(CmdDaemonBase): class CmdDaemonAnalytics(CmdDaemonBase): def run(self): + """Collect and send analytics""" from dvc import analytics - analytics.send(self.args.target) - os.remove(self.args.target) + report = analytics.collect(self.args.cmd_class, self.args.ret) + analytics.send(report) return 0 @@ -57,12 +58,17 @@ def add_parser(subparsers, parent_parser): ) daemon_updater_parser.set_defaults(func=CmdDaemonUpdater) - DAEMON_ANALYTICS_HELP = "Send dvc usage analytics." + DAEMON_ANALYTICS_HELP = "Collect and send dvc usage analytics." daemon_analytics_parser = daemon_subparsers.add_parser( "analytics", parents=[parent_parser], description=DAEMON_ANALYTICS_HELP, help=DAEMON_ANALYTICS_HELP, ) - daemon_analytics_parser.add_argument("target", help="Analytics file.") + daemon_analytics_parser.add_argument( + "cmd_class", help="Class called through main" + ) + daemon_analytics_parser.add_argument( + "ret", help="Return code from running such class" + ) daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics) diff --git a/dvc/main.py b/dvc/main.py index 2b74fe6e9..c88661714 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -1,9 +1,7 @@ """Main entry point for dvc CLI.""" from __future__ import unicode_literals -import json import logging -import tempfile from dvc import analytics from dvc.cli import parse_args @@ -85,12 +83,8 @@ def main(argv=None): logger.info(FOOTER) if analytics.is_enabled(): - report = analytics.collect(args, ret) - - # Use a temporary file to pass the report between the current and - # detached process. - with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: - json.dump(report, fobj) - daemon(["analytics", fobj.name]) + # Include the command class and return code on the analytics report + cmd_class = args.func.__name__ if hasattr(args, "func") else None + daemon(["analytics", cmd_class, ret]) return ret diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py index 97a71d6d0..31d62d56c 100644 --- a/tests/func/test_analytics.py +++ b/tests/func/test_analytics.py @@ -1,15 +1,9 @@ import mock -import json from dvc.main import main -from dvc.utils.compat import fspath @mock.patch("dvc.analytics.send") def test_daemon_analytics(mock_send, tmp_path): - report = {"name": "dummy report"} - fname = tmp_path / "report" - fname.write_text(str(json.dumps(report))) - - assert 0 == main(["daemon", "analytics", fspath(fname)]) + assert 0 == main(["daemon", "analytics", None, 0]) assert mock_send.called diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 7a864bfa3..c56898721 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -1,4 +1,3 @@ -import json import pytest import mock @@ -40,12 +39,9 @@ def test_collect(tmp_global_config): def test_send(mock_post, tmp_path): url = "https://analytics.dvc.org" report = {"name": "dummy report"} - fname = tmp_path / "report" - fname.write_text(str(json.dumps(report))) - analytics.send(fname) - assert mock_post.called - assert mock_post.call_args.args[0] == url + analytics.send(report) + mock_post.assert_called_with(url, json=report, timeout=5) @pytest.mark.parametrize( From 624cbb2d91a9b16a56e6e5dd56c355505661146f Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 2 Dec 2019 23:46:04 -0600 Subject: [PATCH 027/118] :nail_care: more specific comment about analytics --- dvc/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dvc/main.py b/dvc/main.py index c88661714..9d6469a73 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -83,7 +83,8 @@ def main(argv=None): logger.info(FOOTER) if analytics.is_enabled(): - # Include the command class and return code on the analytics report + # Collect and send analytics report in a separate process, + # including the command class and return code on the report cmd_class = args.func.__name__ if hasattr(args, "func") else None daemon(["analytics", cmd_class, ret]) From 285620d96e16bb8be5a359383d630a4a770c3059 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Tue, 3 Dec 2019 09:41:23 -0600 Subject: [PATCH 028/118] tests: mock analytics.collect while testing daemon --- tests/func/test_analytics.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py index 31d62d56c..e80964038 100644 --- a/tests/func/test_analytics.py +++ b/tests/func/test_analytics.py @@ -3,7 +3,9 @@ import mock from dvc.main import main +@mock.patch("dvc.analytics.collect") @mock.patch("dvc.analytics.send") -def test_daemon_analytics(mock_send, tmp_path): +def test_daemon_analytics(mock_collect, mock_send, tmp_path): assert 0 == main(["daemon", "analytics", None, 0]) + assert mock_collect.called assert mock_send.called From 02eaa9a14fa2994aa04936ae0d4c692c4d9067be Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Tue, 3 Dec 2019 18:38:11 -0600 Subject: [PATCH 029/118] py35: support for fspath(pathlib.Path()) --- dvc/utils/compat.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dvc/utils/compat.py b/dvc/utils/compat.py index 372c8d2c6..ab079c1d0 100644 --- a/dvc/utils/compat.py +++ b/dvc/utils/compat.py @@ -207,6 +207,12 @@ except ImportError: except AttributeError: if hasattr(path_type, "__fspath__"): raise + + # Support for pathlib.Path in Python 3.5, since it doesn't + # implement `__fspath__` + if isinstance(path, pathlib.Path): + return str(path) + else: raise TypeError( "expected str, bytes or os.PathLike object, " From ede010384b24a925a8fe003bfb8b8bec7fd5238d Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Wed, 4 Dec 2019 18:36:59 -0600 Subject: [PATCH 030/118] py2: use convert_to_unicode instead of str --- dvc/analytics.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 6ea473571..1da378932 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -15,6 +15,7 @@ from dvc.repo import Repo from dvc.scm import SCM from dvc.utils import env2bool, is_binary, makedirs from dvc.utils.compat import str, FileNotFoundError +from dvc.utils.compat import str, FileNotFoundError, convert_to_unicode logger = logging.getLogger(__name__) @@ -112,7 +113,8 @@ def find_or_create_user_id(): except (FileNotFoundError, ValueError, AttributeError): user_id = str(uuid.uuid4()) makedirs(fname.parent, exist_ok=True) - fname.write_text(str(json.dumps({"user_id": user_id}))) + data = convert_to_unicode(json.dumps({"user_id": user_id})) + fname.write_text(data) return user_id From 7a3aae1c7e8a5b276912e7d3f280e47b1c542c03 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Wed, 4 Dec 2019 19:43:00 -0600 Subject: [PATCH 031/118] tests: add unit test for analytics.system_info --- tests/unit/test_analytics.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index c56898721..4a2388c95 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -1,5 +1,7 @@ import pytest import mock +import platform +import json from voluptuous import Schema, Any @@ -65,6 +67,36 @@ def test_is_enabled(dvc_repo, config, result, monkeypatch): assert result == analytics.is_enabled() +def test_system_info(): + schema = Schema({"os": Any("windows", "mac", "linux")}) + + system = platform.system() + + if system == "Windows": + schema = schema.extend( + { + "windows_version_build": int, + "windows_version_major": int, + "windows_version_minor": int, + "windows_version_service_pack": str, + } + ) + + if system == "Darwin": + schema = schema.extend({"mac_version": str}) + + if system == "Linux": + schema = schema.extend( + { + "linux_distro": str, + "linux_distro_like": Any(str, None), + "linux_distro_version": Any(str, None), + } + ) + + assert schema(analytics.system_info()) + + def test_find_or_create_user_id(tmp_global_config): created = analytics.find_or_create_user_id() found = analytics.find_or_create_user_id() From 05a68684fd1780893338b78c25033925a319dc21 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Wed, 4 Dec 2019 19:44:12 -0600 Subject: [PATCH 032/118] tests: isolate global config from analytics tests fix #2806 --- tests/unit/test_analytics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 4a2388c95..b74989381 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -56,7 +56,7 @@ def test_send(mock_post, tmp_path): ({"analytics": "false", "unknown": "broken"}, False), ], ) -def test_is_enabled(dvc_repo, config, result, monkeypatch): +def test_is_enabled(dvc_repo, config, result, monkeypatch, tmp_global_config): configobj = dvc_repo.config._repo_config configobj["core"] = config configobj.write() From 71ead3f62e5f3f559b04cd776dd48c638ce63748 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Wed, 4 Dec 2019 19:46:40 -0600 Subject: [PATCH 033/118] analytics: use a tempfile for inter-process communication --- dvc/analytics.py | 41 ++++++++++++++++++++++++++++++------ dvc/command/daemon.py | 14 +++++------- dvc/main.py | 6 +----- tests/func/test_analytics.py | 23 +++++++++++++++----- tests/unit/test_analytics.py | 18 +++++++++------- 5 files changed, 69 insertions(+), 33 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 1da378932..6501ece8f 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -3,31 +3,57 @@ import logging import platform import requests import sys +import tempfile import uuid import distro from dvc import __version__ from dvc.config import Config, to_bool +from dvc.daemon import daemon from dvc.exceptions import NotDvcRepoError from dvc.lock import Lock, LockError from dvc.repo import Repo from dvc.scm import SCM from dvc.utils import env2bool, is_binary, makedirs -from dvc.utils.compat import str, FileNotFoundError from dvc.utils.compat import str, FileNotFoundError, convert_to_unicode logger = logging.getLogger(__name__) -def collect(cmd_class=None, return_code=None): +def collect_and_send_report(args=None, return_code=None): """ - Query the system to fill a report. + Collect information from the runtime/environment and the command + being executed into a report and send it over the network. + + To prevent analytics from blocking the execution of the main thread, + sending the report is done in a separate process. + + The inter-process communication happens through a file containing the + report as a JSON, where the _collector_ generates it and the _sender_ + removes it after sending it. + """ + report = runtime_info() + + # Include command execution information on the report + report.update( + { + "cmd_class": args.func.__name__ if hasattr(args, "func") else None, + "cmd_return_code": return_code, + } + ) + + with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: + json.dump(report, fobj) + daemon(["analytics", fobj.name]) + + +def runtime_info(): + """ + Gather information from the environment where DVC runs to fill a report. """ return { - "cmd_class": cmd_class, - "cmd_return_code": return_code, "dvc_version": __version__, "is_binary": is_binary(), "scm_class": scm_in_use(), @@ -53,7 +79,10 @@ def is_enabled(): def send(report): url = "https://analytics.dvc.org" - requests.post(url, json=report, timeout=5) + headers = {"content-type": "application/json"} + + with open(report, "rb") as fobj: + requests.post(url, data=fobj, headers=headers, timeout=5) def scm_in_use(): diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index 6bcb15f54..68d4b36f6 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -25,11 +25,12 @@ class CmdDaemonUpdater(CmdDaemonBase): class CmdDaemonAnalytics(CmdDaemonBase): def run(self): - """Collect and send analytics""" from dvc import analytics - report = analytics.collect(self.args.cmd_class, self.args.ret) + report = self.args.target + analytics.send(report) + os.remove(report) return 0 @@ -58,17 +59,12 @@ def add_parser(subparsers, parent_parser): ) daemon_updater_parser.set_defaults(func=CmdDaemonUpdater) - DAEMON_ANALYTICS_HELP = "Collect and send dvc usage analytics." + DAEMON_ANALYTICS_HELP = "Send dvc usage analytics." daemon_analytics_parser = daemon_subparsers.add_parser( "analytics", parents=[parent_parser], description=DAEMON_ANALYTICS_HELP, help=DAEMON_ANALYTICS_HELP, ) - daemon_analytics_parser.add_argument( - "cmd_class", help="Class called through main" - ) - daemon_analytics_parser.add_argument( - "ret", help="Return code from running such class" - ) + daemon_analytics_parser.add_argument("target", help="Analytics file.") daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics) diff --git a/dvc/main.py b/dvc/main.py index 9d6469a73..86e1a9e0d 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -6,7 +6,6 @@ import logging from dvc import analytics from dvc.cli import parse_args from dvc.config import ConfigError -from dvc.daemon import daemon from dvc.exceptions import DvcParserError from dvc.exceptions import NotDvcRepoError from dvc.external_repo import clean_repos @@ -83,9 +82,6 @@ def main(argv=None): logger.info(FOOTER) if analytics.is_enabled(): - # Collect and send analytics report in a separate process, - # including the command class and return code on the report - cmd_class = args.func.__name__ if hasattr(args, "func") else None - daemon(["analytics", cmd_class, ret]) + analytics.collect_and_send_report(args, ret) return ret diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py index e80964038..df7454965 100644 --- a/tests/func/test_analytics.py +++ b/tests/func/test_analytics.py @@ -1,11 +1,24 @@ import mock from dvc.main import main +from dvc.utils.compat import fspath -@mock.patch("dvc.analytics.collect") @mock.patch("dvc.analytics.send") -def test_daemon_analytics(mock_collect, mock_send, tmp_path): - assert 0 == main(["daemon", "analytics", None, 0]) - assert mock_collect.called - assert mock_send.called +@mock.patch("os.remove") +def test_daemon_analytics(mock_remove, mock_send, tmp_path): + report = fspath(tmp_path) + assert 0 == main(["daemon", "analytics", report]) + + mock_send.assert_called_with(report) + mock_remove.assert_called_with(report) + + +@mock.patch("dvc.daemon._spawn") +@mock.patch("dvc.analytics.is_enabled", return_value=True) +@mock.patch("dvc.analytics.runtime_info", return_value={}) +def test_main_analytics(mock_is_enabled, mock_daemon, mock_report, dvc_repo): + assert 0 == main(["add", "foo"]) + assert mock_is_enabled.called + assert mock_report.called + assert mock_daemon.called diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index b74989381..69754fd60 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -20,30 +20,32 @@ def tmp_global_config(tmp_path): yield -def test_collect(tmp_global_config): +def test_runtime_info(tmp_global_config): schema = Schema( { - "cmd_class": Any(str, None), - "cmd_return_code": Any(int, None), "dvc_version": Any(builtin_str, str), "is_binary": bool, - "scm_class": Any("Git", None), + "scm_class": Any("Git", "NoSCM"), "user_id": Any(builtin_str, str), "system_info": dict, } ) - report = analytics.collect(return_code=0) - assert schema(report) + assert schema(analytics.runtime_info()) @mock.patch("requests.post") def test_send(mock_post, tmp_path): url = "https://analytics.dvc.org" report = {"name": "dummy report"} + fname = tmp_path / "report" - analytics.send(report) - mock_post.assert_called_with(url, json=report, timeout=5) + with open(fname, "w") as fobj: + json.dump(report, fobj) + + analytics.send(fname) + assert mock_post.called + assert mock_post.call_args.args[0] == url @pytest.mark.parametrize( From af2f216c5e9b169ec728bbfd43fa3138b7b7e323 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Fri, 22 Nov 2019 20:40:46 +0100 Subject: [PATCH 034/118] get: copy regular files Allows `dvc get` to copy regular files or directories. fixes: #2515 --- dvc/repo/get.py | 52 ++++++++++++++++++++++++---------------- tests/basic_env.py | 4 +++- tests/func/test_get.py | 24 ++++++++++++++++++- tests/func/test_repro.py | 8 +++---- 4 files changed, 61 insertions(+), 27 deletions(-) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 21d01ed7e..ee4765678 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -1,5 +1,6 @@ import logging import os +import shutil import shortuuid @@ -29,32 +30,41 @@ def get(url, path, out=None, rev=None): # reflink and/or hardlink. Not using tempfile.TemporaryDirectory # because it will create a symlink to tmpfs, which defeats the purpose # and won't work with reflink/hardlink. - dpath = os.path.dirname(os.path.abspath(out)) + abspath = os.path.abspath(out) + dpath = os.path.dirname(abspath) tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid())) try: with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo: - # Note: we need to replace state, because in case of getting DVC - # dependency on CIFS or NFS filesystems, sqlite-based state - # will be unable to obtain lock - repo.state = StateNoop() + tmp_repo_path = os.path.join(repo.root_dir, path) + if os.path.exists(tmp_repo_path): + if os.path.isdir(tmp_repo_path): + shutil.copytree(tmp_repo_path, abspath) + else: + shutil.copy2(tmp_repo_path, abspath) + else: + # Note: we need to replace state, because in case of getting + # DVC dependency on CIFS or NFS filesystems, sqlite-based state + # will be unable to obtain lock + repo.state = StateNoop() - # Try any links possible to avoid data duplication. - # - # Not using symlink, because we need to remove cache after we are - # done, and to make that work we would have to copy data over - # anyway before removing the cache, so we might just copy it - # right away. - # - # Also, we can't use theoretical "move" link type here, because - # the same cache file might be used a few times in a directory. - repo.cache.local.cache_types = ["reflink", "hardlink", "copy"] + # Try any links possible to avoid data duplication. + # + # Not using symlink, because we need to remove cache after we + # are done, and to make that work we would have to copy data + # over anyway before removing the cache, so we might just copy + # it right away. + # + # Also, we can't use theoretical "move" link type here, because + # the same cache file might be used a few times in a directory. + repo.cache.local.cache_types = ["reflink", "hardlink", "copy"] - o = repo.find_out_by_relpath(path) - with repo.state: - repo.cloud.pull(o.get_used_cache()) - o.path_info = PathInfo(os.path.abspath(out)) - with o.repo.state: - o.checkout() + o = repo.find_out_by_relpath(path) + + with repo.state: + repo.cloud.pull(o.get_used_cache()) + o.path_info = PathInfo(os.path.abspath(out)) + with o.repo.state: + o.checkout() except NotDvcRepoError: raise UrlNotDvcRepoError(url) diff --git a/tests/basic_env.py b/tests/basic_env.py index 1fba67a55..a2f059df6 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -41,7 +41,8 @@ class TestDirFixture(object): # in tests, we replace foo with bar, so we need to make sure that when we # modify a file in our tests, its content length changes. BAR_CONTENTS = BAR + "r" - CODE = "code.py" + CODE_DIR = "code" + CODE = "code/code.py" CODE_CONTENTS = ( "import sys\nimport shutil\n" "shutil.copyfile(sys.argv[1], sys.argv[2])" @@ -90,6 +91,7 @@ class TestDirFixture(object): self._pushd(self._root_dir) self.create(self.FOO, self.FOO_CONTENTS) self.create(self.BAR, self.BAR_CONTENTS) + os.mkdir(self.CODE_DIR) self.create(self.CODE, self.CODE_CONTENTS) os.mkdir(self.DATA_DIR) os.mkdir(self.DATA_SUB_DIR) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 07551c5f3..7c2fccfe4 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -38,7 +38,29 @@ def test_get_repo_dir(erepo): trees_equal(src, dst) -def test_cache_type_is_properly_overridden(erepo): +def test_get_regular_file(repo_dir, erepo): + src = erepo.CODE + dst = erepo.CODE + + Repo.get(erepo.root_dir, src, dst) + + assert os.path.exists(dst) + assert os.path.isfile(dst) + assert filecmp.cmp(repo_dir.CODE, dst, shallow=False) + + +def test_get_regular_dir(repo_dir, erepo): + src = erepo.CODE_DIR + dst = erepo.CODE_DIR + + Repo.get(erepo.root_dir, src, dst) + + assert os.path.exists(dst) + assert os.path.isdir(dst) + trees_equal(src, dst) + + +def test_cache_type_is_properly_overridden(repo_dir, erepo): erepo.dvc.config.set( Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE, "symlink" ) diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index a85995fe9..5caf5c40e 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -738,9 +738,9 @@ class TestCmdReproChdirCwdBackwardCompatible(TestDvc): os.mkdir(dname) foo = os.path.join(dname, self.FOO) bar = os.path.join(dname, self.BAR) - code = os.path.join(dname, self.CODE) + code_dir = os.path.join(dname, self.CODE_DIR) shutil.copyfile(self.FOO, foo) - shutil.copyfile(self.CODE, code) + shutil.copytree(self.CODE_DIR, code_dir) ret = main( [ @@ -776,9 +776,9 @@ class TestCmdReproChdir(TestDvc): os.mkdir(dname) foo = os.path.join(dname, self.FOO) bar = os.path.join(dname, self.BAR) - code = os.path.join(dname, self.CODE) + code_dir = os.path.join(dname, self.CODE_DIR) shutil.copyfile(self.FOO, foo) - shutil.copyfile(self.CODE, code) + shutil.copytree(self.CODE_DIR, code_dir) ret = main( [ From 4d142569957ac103d11302b08cba711e23c64563 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Sat, 23 Nov 2019 14:57:56 +0100 Subject: [PATCH 035/118] fixup! get: copy regular files --- tests/basic_env.py | 13 ++++++++++--- tests/func/test_get.py | 10 +++++----- tests/func/test_repro.py | 8 ++++---- tests/func/test_tree.py | 9 ++++++--- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/tests/basic_env.py b/tests/basic_env.py index a2f059df6..d27130fbf 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -41,14 +41,19 @@ class TestDirFixture(object): # in tests, we replace foo with bar, so we need to make sure that when we # modify a file in our tests, its content length changes. BAR_CONTENTS = BAR + "r" - CODE_DIR = "code" - CODE = "code/code.py" + CODE = "code.py" CODE_CONTENTS = ( "import sys\nimport shutil\n" "shutil.copyfile(sys.argv[1], sys.argv[2])" ) UNICODE = "тест" UNICODE_CONTENTS = "проверка" + REGULAR_DIR = "lib" + REGULAR_FILE = os.path.join(REGULAR_DIR, "file.txt") + REGULAR_FILE_CONTENTS = """ + A file used to test retrieval of regular files tracked by git that + are not stored in a dvc remote backend. + """ def __init__(self): root_dir = self.mkdtemp() @@ -91,13 +96,14 @@ class TestDirFixture(object): self._pushd(self._root_dir) self.create(self.FOO, self.FOO_CONTENTS) self.create(self.BAR, self.BAR_CONTENTS) - os.mkdir(self.CODE_DIR) self.create(self.CODE, self.CODE_CONTENTS) os.mkdir(self.DATA_DIR) os.mkdir(self.DATA_SUB_DIR) self.create(self.DATA, self.DATA_CONTENTS) self.create(self.DATA_SUB, self.DATA_SUB_CONTENTS) self.create(self.UNICODE, self.UNICODE_CONTENTS) + os.mkdir(self.REGULAR_DIR) + self.create(self.REGULAR_FILE, self.REGULAR_FILE_CONTENTS) def tearDown(self): self._popd() @@ -139,6 +145,7 @@ class TestGitFixture(TestDirFixture): raise self.git.index.add([self.CODE]) + self.git.index.add([self.REGULAR_DIR]) self.git.index.commit("add code") def tearDown(self): diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 7c2fccfe4..e66f0d96b 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -39,19 +39,19 @@ def test_get_repo_dir(erepo): def test_get_regular_file(repo_dir, erepo): - src = erepo.CODE - dst = erepo.CODE + src = erepo.REGULAR_FILE + dst = erepo.REGULAR_FILE + "_imported" Repo.get(erepo.root_dir, src, dst) assert os.path.exists(dst) assert os.path.isfile(dst) - assert filecmp.cmp(repo_dir.CODE, dst, shallow=False) + assert filecmp.cmp(src, dst, shallow=False) def test_get_regular_dir(repo_dir, erepo): - src = erepo.CODE_DIR - dst = erepo.CODE_DIR + src = erepo.REGULAR_DIR + dst = erepo.REGULAR_DIR + "_imported" Repo.get(erepo.root_dir, src, dst) diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index 5caf5c40e..a85995fe9 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -738,9 +738,9 @@ class TestCmdReproChdirCwdBackwardCompatible(TestDvc): os.mkdir(dname) foo = os.path.join(dname, self.FOO) bar = os.path.join(dname, self.BAR) - code_dir = os.path.join(dname, self.CODE_DIR) + code = os.path.join(dname, self.CODE) shutil.copyfile(self.FOO, foo) - shutil.copytree(self.CODE_DIR, code_dir) + shutil.copyfile(self.CODE, code) ret = main( [ @@ -776,9 +776,9 @@ class TestCmdReproChdir(TestDvc): os.mkdir(dname) foo = os.path.join(dname, self.FOO) bar = os.path.join(dname, self.BAR) - code_dir = os.path.join(dname, self.CODE_DIR) + code = os.path.join(dname, self.CODE) shutil.copyfile(self.FOO, foo) - shutil.copytree(self.CODE_DIR, code_dir) + shutil.copyfile(self.CODE, code) ret = main( [ diff --git a/tests/func/test_tree.py b/tests/func/test_tree.py index c2ae316c2..656303525 100644 --- a/tests/func/test_tree.py +++ b/tests/func/test_tree.py @@ -115,9 +115,10 @@ class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir): [ ( self._root_dir, - ["data_dir"], + ["lib", "data_dir"], ["code.py", "bar", "тест", "foo"], ), + (join(self._root_dir, "lib"), [], ["file.txt"]), (join(self._root_dir, "data_dir"), ["data_sub_dir"], ["data"]), ( join(self._root_dir, "data_dir", "data_sub_dir"), @@ -151,9 +152,10 @@ class TestWalkInGit(AssertWalkEqualMixin, TestGit): [ ( self._root_dir, - ["data_dir"], + ["lib", "data_dir"], ["bar", "тест", "code.py", "foo"], ), + (join(self._root_dir, "lib"), [], ["file.txt"]), (join(self._root_dir, "data_dir"), ["data_sub_dir"], ["data"]), ( join(self._root_dir, "data_dir", "data_sub_dir"), @@ -181,13 +183,14 @@ class TestWalkInGit(AssertWalkEqualMixin, TestGit): self.assertWalkEqual( tree.walk("."), [ - (self._root_dir, ["data_dir"], ["code.py"]), + (self._root_dir, ["data_dir", "lib"], ["code.py"]), (join(self._root_dir, "data_dir"), ["data_sub_dir"], []), ( join(self._root_dir, "data_dir", "data_sub_dir"), [], ["data_sub"], ), + (join(self._root_dir, "lib"), [], ["file.txt"]), ], ) self.assertWalkEqual( From 97862a3d5fa2b5a979e4c1c8fdde280ad7368880 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Sat, 23 Nov 2019 15:26:32 +0100 Subject: [PATCH 036/118] Make tree.walk() output deterministic --- dvc/utils/__init__.py | 5 +++++ tests/func/test_tree.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 68ba41877..3a24a4f56 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -296,6 +296,11 @@ def dvc_walk(top, dvcignore, topdown=True, onerror=None, followlinks=False): for root, dirs, files in os.walk( top, topdown=topdown, onerror=onerror, followlinks=followlinks ): + # Dirs are returned in arbitrary order on different platforms. Force a + # specific order so that we can make deterministic assumptions on the + # directory order. + # https://stackoverflow.com/a/54773660/2966951 + dirs.sort() if dvcignore: dirs[:], files[:] = dvcignore(root, dirs, files) diff --git a/tests/func/test_tree.py b/tests/func/test_tree.py index 656303525..6aa2f5856 100644 --- a/tests/func/test_tree.py +++ b/tests/func/test_tree.py @@ -118,13 +118,13 @@ class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir): ["lib", "data_dir"], ["code.py", "bar", "тест", "foo"], ), - (join(self._root_dir, "lib"), [], ["file.txt"]), (join(self._root_dir, "data_dir"), ["data_sub_dir"], ["data"]), ( join(self._root_dir, "data_dir", "data_sub_dir"), [], ["data_sub"], ), + (join(self._root_dir, "lib"), [], ["file.txt"]), ], ) @@ -155,13 +155,13 @@ class TestWalkInGit(AssertWalkEqualMixin, TestGit): ["lib", "data_dir"], ["bar", "тест", "code.py", "foo"], ), - (join(self._root_dir, "lib"), [], ["file.txt"]), (join(self._root_dir, "data_dir"), ["data_sub_dir"], ["data"]), ( join(self._root_dir, "data_dir", "data_sub_dir"), [], ["data_sub"], ), + (join(self._root_dir, "lib"), [], ["file.txt"]), ], ) self.assertWalkEqual( From 8c3ffe816da1cc740cb499370c1925a696e39766 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Tue, 26 Nov 2019 17:07:21 +0100 Subject: [PATCH 037/118] fixup! Make tree.walk() output deterministic --- dvc/utils/__init__.py | 5 ----- tests/basic_env.py | 9 --------- tests/func/test_get.py | 13 ++++++++----- tests/func/test_tree.py | 9 +++------ 4 files changed, 11 insertions(+), 25 deletions(-) diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 3a24a4f56..68ba41877 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -296,11 +296,6 @@ def dvc_walk(top, dvcignore, topdown=True, onerror=None, followlinks=False): for root, dirs, files in os.walk( top, topdown=topdown, onerror=onerror, followlinks=followlinks ): - # Dirs are returned in arbitrary order on different platforms. Force a - # specific order so that we can make deterministic assumptions on the - # directory order. - # https://stackoverflow.com/a/54773660/2966951 - dirs.sort() if dvcignore: dirs[:], files[:] = dvcignore(root, dirs, files) diff --git a/tests/basic_env.py b/tests/basic_env.py index d27130fbf..1fba67a55 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -48,12 +48,6 @@ class TestDirFixture(object): ) UNICODE = "тест" UNICODE_CONTENTS = "проверка" - REGULAR_DIR = "lib" - REGULAR_FILE = os.path.join(REGULAR_DIR, "file.txt") - REGULAR_FILE_CONTENTS = """ - A file used to test retrieval of regular files tracked by git that - are not stored in a dvc remote backend. - """ def __init__(self): root_dir = self.mkdtemp() @@ -102,8 +96,6 @@ class TestDirFixture(object): self.create(self.DATA, self.DATA_CONTENTS) self.create(self.DATA_SUB, self.DATA_SUB_CONTENTS) self.create(self.UNICODE, self.UNICODE_CONTENTS) - os.mkdir(self.REGULAR_DIR) - self.create(self.REGULAR_FILE, self.REGULAR_FILE_CONTENTS) def tearDown(self): self._popd() @@ -145,7 +137,6 @@ class TestGitFixture(TestDirFixture): raise self.git.index.add([self.CODE]) - self.git.index.add([self.REGULAR_DIR]) self.git.index.commit("add code") def tearDown(self): diff --git a/tests/func/test_get.py b/tests/func/test_get.py index e66f0d96b..15034eef5 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -38,9 +38,10 @@ def test_get_repo_dir(erepo): trees_equal(src, dst) -def test_get_regular_file(repo_dir, erepo): - src = erepo.REGULAR_FILE - dst = erepo.REGULAR_FILE + "_imported" +def test_get_regular_file(erepo): + src = os.path.join(erepo.root_dir, "file") + dst = src + "_imported" + erepo.create(src, "hello") Repo.get(erepo.root_dir, src, dst) @@ -50,9 +51,11 @@ def test_get_regular_file(repo_dir, erepo): def test_get_regular_dir(repo_dir, erepo): - src = erepo.REGULAR_DIR - dst = erepo.REGULAR_DIR + "_imported" + src = os.path.join(erepo.root_dir, "directory") + dst = src + "_imported" + os.mkdir(src) + erepo.create(os.path.join(src, "file"), "hello") Repo.get(erepo.root_dir, src, dst) assert os.path.exists(dst) diff --git a/tests/func/test_tree.py b/tests/func/test_tree.py index 6aa2f5856..c2ae316c2 100644 --- a/tests/func/test_tree.py +++ b/tests/func/test_tree.py @@ -115,7 +115,7 @@ class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir): [ ( self._root_dir, - ["lib", "data_dir"], + ["data_dir"], ["code.py", "bar", "тест", "foo"], ), (join(self._root_dir, "data_dir"), ["data_sub_dir"], ["data"]), @@ -124,7 +124,6 @@ class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir): [], ["data_sub"], ), - (join(self._root_dir, "lib"), [], ["file.txt"]), ], ) @@ -152,7 +151,7 @@ class TestWalkInGit(AssertWalkEqualMixin, TestGit): [ ( self._root_dir, - ["lib", "data_dir"], + ["data_dir"], ["bar", "тест", "code.py", "foo"], ), (join(self._root_dir, "data_dir"), ["data_sub_dir"], ["data"]), @@ -161,7 +160,6 @@ class TestWalkInGit(AssertWalkEqualMixin, TestGit): [], ["data_sub"], ), - (join(self._root_dir, "lib"), [], ["file.txt"]), ], ) self.assertWalkEqual( @@ -183,14 +181,13 @@ class TestWalkInGit(AssertWalkEqualMixin, TestGit): self.assertWalkEqual( tree.walk("."), [ - (self._root_dir, ["data_dir", "lib"], ["code.py"]), + (self._root_dir, ["data_dir"], ["code.py"]), (join(self._root_dir, "data_dir"), ["data_sub_dir"], []), ( join(self._root_dir, "data_dir", "data_sub_dir"), [], ["data_sub"], ), - (join(self._root_dir, "lib"), [], ["file.txt"]), ], ) self.assertWalkEqual( From 26b62331c7ce591ad87fc76c1ddee1d2af588119 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Tue, 26 Nov 2019 17:51:47 +0100 Subject: [PATCH 038/118] CLI --help changes --- dvc/command/get.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dvc/command/get.py b/dvc/command/get.py index 77235ecef..0ec3321ce 100644 --- a/dvc/command/get.py +++ b/dvc/command/get.py @@ -33,7 +33,7 @@ class CmdGet(CmdBaseNoRepo): def add_parser(subparsers, parent_parser): - GET_HELP = "Download data from DVC repository." + GET_HELP = "Download files or directories from DVC repository." get_parser = subparsers.add_parser( "get", parents=[parent_parser], @@ -44,10 +44,10 @@ def add_parser(subparsers, parent_parser): get_parser.add_argument( "url", help="URL of Git repository with DVC project to download from." ) - get_parser.add_argument("path", help="Path to data within DVC repository.") get_parser.add_argument( - "-o", "--out", nargs="?", help="Destination path to put data to." + "path", help="Path to a file or directory within a DVC repository." ) + get_parser.add_argument("-o", "--out", nargs="?", help="Destination path.") get_parser.add_argument( "--rev", nargs="?", help="DVC repository git revision." ) From 7e1423d13c06ad80ca92a5e8081b220bc966abe7 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Thu, 28 Nov 2019 15:08:01 +0100 Subject: [PATCH 039/118] fix tests --- tests/func/test_get.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 15034eef5..f8143fc34 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -39,24 +39,28 @@ def test_get_repo_dir(erepo): def test_get_regular_file(erepo): - src = os.path.join(erepo.root_dir, "file") - dst = src + "_imported" - erepo.create(src, "hello") + src = os.path.join(erepo.root_dir, "some_file") + dst = os.path.join(os.getcwd(), "_imported") - Repo.get(erepo.root_dir, src, dst) + erepo.create(src, "hello") + erepo.dvc.scm.add([src]) + erepo.dvc.scm.commit("add a regular file") + Repo.get(erepo.root_dir, os.path.basename(src), os.path.basename(dst)) assert os.path.exists(dst) assert os.path.isfile(dst) assert filecmp.cmp(src, dst, shallow=False) -def test_get_regular_dir(repo_dir, erepo): - src = os.path.join(erepo.root_dir, "directory") - dst = src + "_imported" +def test_get_regular_dir(erepo): + src = os.path.join(erepo.root_dir, "some_directory") + src_file = os.path.join(src, "file.txt") + dst = os.path.join(os.getcwd(), "_imported") - os.mkdir(src) - erepo.create(os.path.join(src, "file"), "hello") - Repo.get(erepo.root_dir, src, dst) + erepo.create(src_file, "hello") + erepo.dvc.scm.add([src_file]) + erepo.dvc.scm.commit("add a regular dir") + Repo.get(erepo.root_dir, os.path.basename(src), os.path.basename(dst)) assert os.path.exists(dst) assert os.path.isdir(dst) From 1f5da3ffef8b2d6dde329e9826106db01bad706b Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Thu, 28 Nov 2019 15:22:22 +0100 Subject: [PATCH 040/118] fix CLI docs --- dvc/command/get.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dvc/command/get.py b/dvc/command/get.py index 0ec3321ce..e3172782d 100644 --- a/dvc/command/get.py +++ b/dvc/command/get.py @@ -33,7 +33,7 @@ class CmdGet(CmdBaseNoRepo): def add_parser(subparsers, parent_parser): - GET_HELP = "Download files or directories from DVC repository." + GET_HELP = "Download/copy files or directories from DVC repository." get_parser = subparsers.add_parser( "get", parents=[parent_parser], @@ -47,7 +47,12 @@ def add_parser(subparsers, parent_parser): get_parser.add_argument( "path", help="Path to a file or directory within a DVC repository." ) - get_parser.add_argument("-o", "--out", nargs="?", help="Destination path.") + get_parser.add_argument( + "-o", + "--out", + nargs="?", + help="Destination path to copy/download files to.", + ) get_parser.add_argument( "--rev", nargs="?", help="DVC repository git revision." ) From 74f385111797fcafa100df9ec1c3380bb3902ae2 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Mon, 2 Dec 2019 18:27:14 +0100 Subject: [PATCH 041/118] Disallow absolute paths in `dvc get` --- dvc/exceptions.py | 9 +++++++++ dvc/repo/get.py | 4 ++++ tests/func/test_get.py | 8 ++++++++ 3 files changed, 21 insertions(+) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index e5fb6f00d..1ced27032 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -268,6 +268,15 @@ class FileMissingError(DvcException): ) +class NoAbsolutePathError(DvcException): + def __init__(self, path, cause=None): + super(NoAbsolutePathError, self).__init__( + "The path '{}' is an absolute path. ".format(path) + + "Provide a relative path.", + cause=cause, + ) + + class DvcIgnoreInCollectedDirError(DvcException): def __init__(self, ignore_dirname): super(DvcIgnoreInCollectedDirError, self).__init__( diff --git a/dvc/repo/get.py b/dvc/repo/get.py index ee4765678..bb2482302 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -8,6 +8,7 @@ from dvc.exceptions import GetDVCFileError from dvc.exceptions import NotDvcRepoError from dvc.exceptions import OutputNotFoundError from dvc.exceptions import UrlNotDvcRepoError +from dvc.exceptions import NoAbsolutePathError from dvc.external_repo import external_repo from dvc.path_info import PathInfo from dvc.stage import Stage @@ -20,6 +21,9 @@ logger = logging.getLogger(__name__) @staticmethod def get(url, path, out=None, rev=None): + if os.path.isabs(path): + raise NoAbsolutePathError(path) + out = resolve_output(path, out) if Stage.is_valid_filename(out): diff --git a/tests/func/test_get.py b/tests/func/test_get.py index f8143fc34..ce0a7ab42 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -9,6 +9,7 @@ import pytest from dvc.config import Config from dvc.exceptions import GetDVCFileError from dvc.exceptions import UrlNotDvcRepoError +from dvc.exceptions import NoAbsolutePathError from dvc.repo import Repo from dvc.system import System from dvc.utils import makedirs @@ -106,6 +107,13 @@ def test_get_a_dvc_file(erepo): Repo.get(erepo.root_dir, "some_file.dvc") +# Prevent `Repo.get()` from copying any file on the filesystem +# https://github.com/iterative/dvc/pull/2837#discussion_r352123053 +def test_fails_with_absolute_paths(erepo): + with pytest.raises(NoAbsolutePathError): + Repo.get(erepo.root_dir, "/root/") + + @pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"]) def test_get_to_dir(dname, erepo): src = erepo.FOO From cadbf10e646b1ea6a2f13446228d27583595f5c2 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Mon, 2 Dec 2019 18:44:23 +0100 Subject: [PATCH 042/118] dont use absolute paths in tests --- tests/func/test_get.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index ce0a7ab42..c263acc3c 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -40,32 +40,33 @@ def test_get_repo_dir(erepo): def test_get_regular_file(erepo): - src = os.path.join(erepo.root_dir, "some_file") - dst = os.path.join(os.getcwd(), "_imported") + src = "some_file" + dst = "some_file_imported" - erepo.create(src, "hello") - erepo.dvc.scm.add([src]) + src_path = os.path.join(erepo.root_dir, src) + erepo.create(src_path, "hello") + erepo.dvc.scm.add([src_path]) erepo.dvc.scm.commit("add a regular file") - Repo.get(erepo.root_dir, os.path.basename(src), os.path.basename(dst)) + Repo.get(erepo.root_dir, src, dst) assert os.path.exists(dst) assert os.path.isfile(dst) - assert filecmp.cmp(src, dst, shallow=False) + assert filecmp.cmp(src_path, dst, shallow=False) def test_get_regular_dir(erepo): - src = os.path.join(erepo.root_dir, "some_directory") - src_file = os.path.join(src, "file.txt") - dst = os.path.join(os.getcwd(), "_imported") + src = "some_directory" + dst = "some_directory_imported" - erepo.create(src_file, "hello") - erepo.dvc.scm.add([src_file]) + src_file_path = os.path.join(erepo.root_dir, src, "file.txt") + erepo.create(src_file_path, "hello") + erepo.dvc.scm.add([src_file_path]) erepo.dvc.scm.commit("add a regular dir") - Repo.get(erepo.root_dir, os.path.basename(src), os.path.basename(dst)) + Repo.get(erepo.root_dir, src, dst) assert os.path.exists(dst) assert os.path.isdir(dst) - trees_equal(src, dst) + trees_equal(os.path.join(erepo.root_dir, src), dst) def test_cache_type_is_properly_overridden(repo_dir, erepo): From ed104978546eb015becd65bdd06d382ffc3cdb84 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Mon, 2 Dec 2019 20:26:42 +0100 Subject: [PATCH 043/118] Remove unused var --- tests/func/test_get.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index c263acc3c..11ab26faf 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -69,7 +69,7 @@ def test_get_regular_dir(erepo): trees_equal(os.path.join(erepo.root_dir, src), dst) -def test_cache_type_is_properly_overridden(repo_dir, erepo): +def test_cache_type_is_properly_overridden(erepo): erepo.dvc.config.set( Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE, "symlink" ) From 9ece4ed8add874d7a7d42bf8ae84bbdfcc0c2b19 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Tue, 3 Dec 2019 14:01:58 +0100 Subject: [PATCH 044/118] ensure absolute paths for dvc files work but not for git --- dvc/exceptions.py | 8 ++--- dvc/repo/get.py | 75 +++++++++++++++++++++++------------------- tests/func/test_get.py | 26 ++++++++++++--- 3 files changed, 66 insertions(+), 43 deletions(-) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 1ced27032..6909bcea0 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -268,12 +268,10 @@ class FileMissingError(DvcException): ) -class NoAbsolutePathError(DvcException): +class FileOutsideRepoError(DvcException): def __init__(self, path, cause=None): - super(NoAbsolutePathError, self).__init__( - "The path '{}' is an absolute path. ".format(path) - + "Provide a relative path.", - cause=cause, + super(FileOutsideRepoError, self).__init__( + "The path '{}' is outside the target repository.".format(path) ) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index bb2482302..24d1ae81a 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -8,7 +8,8 @@ from dvc.exceptions import GetDVCFileError from dvc.exceptions import NotDvcRepoError from dvc.exceptions import OutputNotFoundError from dvc.exceptions import UrlNotDvcRepoError -from dvc.exceptions import NoAbsolutePathError +from dvc.exceptions import NoOutputInExternalRepoError +from dvc.exceptions import FileOutsideRepoError from dvc.external_repo import external_repo from dvc.path_info import PathInfo from dvc.stage import Stage @@ -19,11 +20,24 @@ from dvc.utils.fs import remove logger = logging.getLogger(__name__) +def copy_git_file(repo, src, dst): + try: + # Assert that the file does indeed exist in git before attempting to + # copy it. + repo.scm.repo.head.commit.tree[src] # noqa + except KeyError: + raise FileOutsideRepoError(src) + + src_full_path = os.path.join(repo.root_dir, src) + dst_full_path = os.path.abspath(dst) + if os.path.isdir(src_full_path): + shutil.copytree(src_full_path, dst_full_path) + else: + shutil.copy2(src_full_path, dst_full_path) + + @staticmethod def get(url, path, out=None, rev=None): - if os.path.isabs(path): - raise NoAbsolutePathError(path) - out = resolve_output(path, out) if Stage.is_valid_filename(out): @@ -34,42 +48,35 @@ def get(url, path, out=None, rev=None): # reflink and/or hardlink. Not using tempfile.TemporaryDirectory # because it will create a symlink to tmpfs, which defeats the purpose # and won't work with reflink/hardlink. - abspath = os.path.abspath(out) - dpath = os.path.dirname(abspath) + dpath = os.path.dirname(os.path.abspath(out)) tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid())) try: with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo: - tmp_repo_path = os.path.join(repo.root_dir, path) - if os.path.exists(tmp_repo_path): - if os.path.isdir(tmp_repo_path): - shutil.copytree(tmp_repo_path, abspath) - else: - shutil.copy2(tmp_repo_path, abspath) - else: - # Note: we need to replace state, because in case of getting - # DVC dependency on CIFS or NFS filesystems, sqlite-based state - # will be unable to obtain lock - repo.state = StateNoop() + # Note: we need to replace state, because in case of getting DVC + # dependency on CIFS or NFS filesystems, sqlite-based state + # will be unable to obtain lock + repo.state = StateNoop() - # Try any links possible to avoid data duplication. - # - # Not using symlink, because we need to remove cache after we - # are done, and to make that work we would have to copy data - # over anyway before removing the cache, so we might just copy - # it right away. - # - # Also, we can't use theoretical "move" link type here, because - # the same cache file might be used a few times in a directory. - repo.cache.local.cache_types = ["reflink", "hardlink", "copy"] + # Try any links possible to avoid data duplication. + # + # Not using symlink, because we need to remove cache after we are + # done, and to make that work we would have to copy data over + # anyway before removing the cache, so we might just copy it + # right away. + # + # Also, we can't use theoretical "move" link type here, because + # the same cache file might be used a few times in a directory. + repo.cache.local.cache_types = ["reflink", "hardlink", "copy"] - o = repo.find_out_by_relpath(path) - - with repo.state: - repo.cloud.pull(o.get_used_cache()) - o.path_info = PathInfo(os.path.abspath(out)) - with o.repo.state: - o.checkout() + o = repo.find_out_by_relpath(path) + with repo.state: + repo.cloud.pull(o.get_used_cache()) + o.path_info = PathInfo(os.path.abspath(out)) + with o.repo.state: + o.checkout() + except NoOutputInExternalRepoError: + copy_git_file(repo, path, out) except NotDvcRepoError: raise UrlNotDvcRepoError(url) except OutputNotFoundError: diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 11ab26faf..cf35f4bc5 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -9,7 +9,7 @@ import pytest from dvc.config import Config from dvc.exceptions import GetDVCFileError from dvc.exceptions import UrlNotDvcRepoError -from dvc.exceptions import NoAbsolutePathError +from dvc.exceptions import FileOutsideRepoError from dvc.repo import Repo from dvc.system import System from dvc.utils import makedirs @@ -108,10 +108,28 @@ def test_get_a_dvc_file(erepo): Repo.get(erepo.root_dir, "some_file.dvc") -# Prevent `Repo.get()` from copying any file on the filesystem # https://github.com/iterative/dvc/pull/2837#discussion_r352123053 -def test_fails_with_absolute_paths(erepo): - with pytest.raises(NoAbsolutePathError): +def test_get_full_dvc_path(erepo): + external_data_dir = erepo.mkdtemp() + external_data = os.path.join(external_data_dir, "ext_data") + with open(external_data, "w+") as fobj: + fobj.write("ext_data") + + cur_dir = os.getcwd() + os.chdir(erepo.root_dir) + erepo.dvc.add(external_data) + erepo.dvc.scm.add(["ext_data.dvc"]) + erepo.dvc.scm.commit("add external data") + os.chdir(cur_dir) + + Repo.get(erepo.root_dir, external_data, "ext_data_imported") + assert os.path.isfile("ext_data_imported") + assert filecmp.cmp(external_data, "ext_data_imported", shallow=False) + + +# https://github.com/iterative/dvc/pull/2837#discussion_r352123053 +def test_fails_with_non_git_files(erepo): + with pytest.raises(FileOutsideRepoError): Repo.get(erepo.root_dir, "/root/") From f829338e1eecb94735bbb13f97661e07457428cb Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Tue, 3 Dec 2019 14:06:49 +0100 Subject: [PATCH 045/118] if instead of try / catch --- dvc/repo/get.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 24d1ae81a..d07251f82 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -21,11 +21,9 @@ logger = logging.getLogger(__name__) def copy_git_file(repo, src, dst): - try: - # Assert that the file does indeed exist in git before attempting to - # copy it. - repo.scm.repo.head.commit.tree[src] # noqa - except KeyError: + # Assert that the file does indeed exist in git before attempting to + # copy it. + if src not in repo.scm.repo.head.commit.tree: raise FileOutsideRepoError(src) src_full_path = os.path.join(repo.root_dir, src) From 97793b4fb3f1bb6563a9654799b4513e3db17221 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Tue, 3 Dec 2019 14:08:53 +0100 Subject: [PATCH 046/118] underscore private function --- dvc/repo/get.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index d07251f82..69ab2d538 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -20,7 +20,7 @@ from dvc.utils.fs import remove logger = logging.getLogger(__name__) -def copy_git_file(repo, src, dst): +def _copy_git_file(repo, src, dst): # Assert that the file does indeed exist in git before attempting to # copy it. if src not in repo.scm.repo.head.commit.tree: @@ -74,7 +74,7 @@ def get(url, path, out=None, rev=None): o.checkout() except NoOutputInExternalRepoError: - copy_git_file(repo, path, out) + _copy_git_file(repo, path, out) except NotDvcRepoError: raise UrlNotDvcRepoError(url) except OutputNotFoundError: From 9930e96390748e44b4c5d8de4a38e84ae66d5d48 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Tue, 3 Dec 2019 14:50:11 +0100 Subject: [PATCH 047/118] fixup! ensure absolute paths for dvc files work but not for git --- dvc/repo/get.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 69ab2d538..7ea599844 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -21,13 +21,12 @@ logger = logging.getLogger(__name__) def _copy_git_file(repo, src, dst): - # Assert that the file does indeed exist in git before attempting to - # copy it. - if src not in repo.scm.repo.head.commit.tree: - raise FileOutsideRepoError(src) - src_full_path = os.path.join(repo.root_dir, src) dst_full_path = os.path.abspath(dst) + + if repo.root_dir not in src_full_path: + raise FileOutsideRepoError(src) + if os.path.isdir(src_full_path): shutil.copytree(src_full_path, dst_full_path) else: From 38bed227c65463550e3fd624d6ff2f60cb6606ef Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Wed, 4 Dec 2019 17:28:01 +0100 Subject: [PATCH 048/118] fixup --- dvc/exceptions.py | 6 +++--- dvc/repo/get.py | 7 ++++--- tests/func/test_get.py | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 6909bcea0..237b0d2e5 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -268,9 +268,9 @@ class FileMissingError(DvcException): ) -class FileOutsideRepoError(DvcException): - def __init__(self, path, cause=None): - super(FileOutsideRepoError, self).__init__( +class PathOutsideRepoError(DvcException): + def __init__(self, path): + super(PathOutsideRepoError, self).__init__( "The path '{}' is outside the target repository.".format(path) ) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 7ea599844..f986b31d1 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -9,13 +9,14 @@ from dvc.exceptions import NotDvcRepoError from dvc.exceptions import OutputNotFoundError from dvc.exceptions import UrlNotDvcRepoError from dvc.exceptions import NoOutputInExternalRepoError -from dvc.exceptions import FileOutsideRepoError +from dvc.exceptions import PathOutsideRepoError from dvc.external_repo import external_repo from dvc.path_info import PathInfo from dvc.stage import Stage from dvc.state import StateNoop from dvc.utils import resolve_output from dvc.utils.fs import remove +from dvc.utils.fs import path_isin logger = logging.getLogger(__name__) @@ -24,8 +25,8 @@ def _copy_git_file(repo, src, dst): src_full_path = os.path.join(repo.root_dir, src) dst_full_path = os.path.abspath(dst) - if repo.root_dir not in src_full_path: - raise FileOutsideRepoError(src) + if not path_isin(src_full_path, repo.root_dir): + raise PathOutsideRepoError(src) if os.path.isdir(src_full_path): shutil.copytree(src_full_path, dst_full_path) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index cf35f4bc5..5d519194d 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -9,7 +9,7 @@ import pytest from dvc.config import Config from dvc.exceptions import GetDVCFileError from dvc.exceptions import UrlNotDvcRepoError -from dvc.exceptions import FileOutsideRepoError +from dvc.exceptions import PathOutsideRepoError from dvc.repo import Repo from dvc.system import System from dvc.utils import makedirs @@ -129,7 +129,7 @@ def test_get_full_dvc_path(erepo): # https://github.com/iterative/dvc/pull/2837#discussion_r352123053 def test_fails_with_non_git_files(erepo): - with pytest.raises(FileOutsideRepoError): + with pytest.raises(PathOutsideRepoError): Repo.get(erepo.root_dir, "/root/") From 3c2633e33ffe6eee3217c321f142ed4a221309c4 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Wed, 4 Dec 2019 22:59:55 +0100 Subject: [PATCH 049/118] Pass all tests --- dvc/exceptions.py | 7 ------ dvc/repo/get.py | 52 ++++++++++++++++++++---------------------- tests/func/test_get.py | 20 ++++++++++++++-- 3 files changed, 43 insertions(+), 36 deletions(-) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 237b0d2e5..e5fb6f00d 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -268,13 +268,6 @@ class FileMissingError(DvcException): ) -class PathOutsideRepoError(DvcException): - def __init__(self, path): - super(PathOutsideRepoError, self).__init__( - "The path '{}' is outside the target repository.".format(path) - ) - - class DvcIgnoreInCollectedDirError(DvcException): def __init__(self, ignore_dirname): super(DvcIgnoreInCollectedDirError, self).__init__( diff --git a/dvc/repo/get.py b/dvc/repo/get.py index f986b31d1..9409e2ad8 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -8,8 +8,6 @@ from dvc.exceptions import GetDVCFileError from dvc.exceptions import NotDvcRepoError from dvc.exceptions import OutputNotFoundError from dvc.exceptions import UrlNotDvcRepoError -from dvc.exceptions import NoOutputInExternalRepoError -from dvc.exceptions import PathOutsideRepoError from dvc.external_repo import external_repo from dvc.path_info import PathInfo from dvc.stage import Stage @@ -25,9 +23,6 @@ def _copy_git_file(repo, src, dst): src_full_path = os.path.join(repo.root_dir, src) dst_full_path = os.path.abspath(dst) - if not path_isin(src_full_path, repo.root_dir): - raise PathOutsideRepoError(src) - if os.path.isdir(src_full_path): shutil.copytree(src_full_path, dst_full_path) else: @@ -50,31 +45,34 @@ def get(url, path, out=None, rev=None): tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid())) try: with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo: - # Note: we need to replace state, because in case of getting DVC - # dependency on CIFS or NFS filesystems, sqlite-based state - # will be unable to obtain lock - repo.state = StateNoop() + full_path = os.path.join(repo.root_dir, path) + path_in_repo = path_isin(full_path, repo.root_dir) + if os.path.exists(full_path) and path_in_repo: + _copy_git_file(repo, path, out) + else: + # Note: we need to replace state, because in case of getting + # DVC dependency on CIFS or NFS filesystems, sqlite-based state + # will be unable to obtain lock + repo.state = StateNoop() - # Try any links possible to avoid data duplication. - # - # Not using symlink, because we need to remove cache after we are - # done, and to make that work we would have to copy data over - # anyway before removing the cache, so we might just copy it - # right away. - # - # Also, we can't use theoretical "move" link type here, because - # the same cache file might be used a few times in a directory. - repo.cache.local.cache_types = ["reflink", "hardlink", "copy"] + # Try any links possible to avoid data duplication. + # + # Not using symlink, because we need to remove cache after we + # are done, and to make that work we would have to copy data + # over anyway before removing the cache, so we might just copy + # it right away. + # + # Also, we can't use theoretical "move" link type here, because + # the same cache file might be used a few times in a directory. + repo.cache.local.cache_types = ["reflink", "hardlink", "copy"] - o = repo.find_out_by_relpath(path) - with repo.state: - repo.cloud.pull(o.get_used_cache()) - o.path_info = PathInfo(os.path.abspath(out)) - with o.repo.state: - o.checkout() + o = repo.find_out_by_relpath(path) + with repo.state: + repo.cloud.pull(o.get_used_cache()) + o.path_info = PathInfo(os.path.abspath(out)) + with o.repo.state: + o.checkout() - except NoOutputInExternalRepoError: - _copy_git_file(repo, path, out) except NotDvcRepoError: raise UrlNotDvcRepoError(url) except OutputNotFoundError: diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 5d519194d..d9e098c8e 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -3,13 +3,14 @@ from __future__ import unicode_literals import filecmp import logging import os +import tempfile import pytest from dvc.config import Config from dvc.exceptions import GetDVCFileError from dvc.exceptions import UrlNotDvcRepoError -from dvc.exceptions import PathOutsideRepoError +from dvc.exceptions import NoOutputInExternalRepoError from dvc.repo import Repo from dvc.system import System from dvc.utils import makedirs @@ -127,9 +128,24 @@ def test_get_full_dvc_path(erepo): assert filecmp.cmp(external_data, "ext_data_imported", shallow=False) +def test_non_cached_output(request, erepo): + os.chdir(erepo.root_dir) + erepo.dvc.run( + outs_no_cache=["non_cached_file"], cmd="echo hello > non_cached_file" + ) + erepo.dvc.scm.add(["non_cached_file", "non_cached_file.dvc"]) + erepo.dvc.scm.commit("add non-cached output") + new_dir = tempfile.mkdtemp(prefix=request.node.name + "_") + os.chdir(new_dir) + Repo.get(erepo.root_dir, "non_cached_file") + assert os.path.isfile("non_cached_file") + src = os.path.join(erepo.root_dir, "non_cached_file") + assert filecmp.cmp(src, "non_cached_file", shallow=False) + + # https://github.com/iterative/dvc/pull/2837#discussion_r352123053 def test_fails_with_non_git_files(erepo): - with pytest.raises(PathOutsideRepoError): + with pytest.raises(NoOutputInExternalRepoError): Repo.get(erepo.root_dir, "/root/") From 20e7696da366218237b9b88667353ac0bec99623 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Wed, 4 Dec 2019 23:14:17 +0100 Subject: [PATCH 050/118] return early --- dvc/repo/get.py | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 9409e2ad8..0219d4170 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -49,29 +49,30 @@ def get(url, path, out=None, rev=None): path_in_repo = path_isin(full_path, repo.root_dir) if os.path.exists(full_path) and path_in_repo: _copy_git_file(repo, path, out) - else: - # Note: we need to replace state, because in case of getting - # DVC dependency on CIFS or NFS filesystems, sqlite-based state - # will be unable to obtain lock - repo.state = StateNoop() + return - # Try any links possible to avoid data duplication. - # - # Not using symlink, because we need to remove cache after we - # are done, and to make that work we would have to copy data - # over anyway before removing the cache, so we might just copy - # it right away. - # - # Also, we can't use theoretical "move" link type here, because - # the same cache file might be used a few times in a directory. - repo.cache.local.cache_types = ["reflink", "hardlink", "copy"] + # Note: we need to replace state, because in case of getting DVC + # dependency on CIFS or NFS filesystems, sqlite-based state + # will be unable to obtain lock + repo.state = StateNoop() - o = repo.find_out_by_relpath(path) - with repo.state: - repo.cloud.pull(o.get_used_cache()) - o.path_info = PathInfo(os.path.abspath(out)) - with o.repo.state: - o.checkout() + # Try any links possible to avoid data duplication. + # + # Not using symlink, because we need to remove cache after we are + # done, and to make that work we would have to copy data over + # anyway before removing the cache, so we might just copy it + # right away. + # + # Also, we can't use theoretical "move" link type here, because + # the same cache file might be used a few times in a directory. + repo.cache.local.cache_types = ["reflink", "hardlink", "copy"] + + o = repo.find_out_by_relpath(path) + with repo.state: + repo.cloud.pull(o.get_used_cache()) + o.path_info = PathInfo(os.path.abspath(out)) + with o.repo.state: + o.checkout() except NotDvcRepoError: raise UrlNotDvcRepoError(url) From b17d1c5b04d1a1ad78f13f8fbc79e21e3d7fa611 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Thu, 5 Dec 2019 00:33:23 +0100 Subject: [PATCH 051/118] fixup --- dvc/repo/get.py | 4 +--- tests/func/test_get.py | 9 ++++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 0219d4170..2d1e04c44 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -14,7 +14,6 @@ from dvc.stage import Stage from dvc.state import StateNoop from dvc.utils import resolve_output from dvc.utils.fs import remove -from dvc.utils.fs import path_isin logger = logging.getLogger(__name__) @@ -46,8 +45,7 @@ def get(url, path, out=None, rev=None): try: with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo: full_path = os.path.join(repo.root_dir, path) - path_in_repo = path_isin(full_path, repo.root_dir) - if os.path.exists(full_path) and path_in_repo: + if os.path.exists(full_path) and not os.path.isabs(path): _copy_git_file(repo, path, out) return diff --git a/tests/func/test_get.py b/tests/func/test_get.py index d9e098c8e..2dbd62daf 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import filecmp import logging import os -import tempfile import pytest @@ -128,18 +127,18 @@ def test_get_full_dvc_path(erepo): assert filecmp.cmp(external_data, "ext_data_imported", shallow=False) -def test_non_cached_output(request, erepo): +def test_non_cached_output(tmp_path, erepo): os.chdir(erepo.root_dir) erepo.dvc.run( outs_no_cache=["non_cached_file"], cmd="echo hello > non_cached_file" ) erepo.dvc.scm.add(["non_cached_file", "non_cached_file.dvc"]) erepo.dvc.scm.commit("add non-cached output") - new_dir = tempfile.mkdtemp(prefix=request.node.name + "_") - os.chdir(new_dir) + os.chdir(tmp_path) Repo.get(erepo.root_dir, "non_cached_file") - assert os.path.isfile("non_cached_file") + src = os.path.join(erepo.root_dir, "non_cached_file") + assert os.path.isfile("non_cached_file") assert filecmp.cmp(src, "non_cached_file", shallow=False) From 9ae4ab88bf790d463c541d7fe8752bba131e758c Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Thu, 5 Dec 2019 18:55:22 +0100 Subject: [PATCH 052/118] fix error message for git files --- dvc/repo/get.py | 26 ++++++++++++++++++++------ tests/func/test_get.py | 12 +++++++++++- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 2d1e04c44..5902575c6 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -8,6 +8,7 @@ from dvc.exceptions import GetDVCFileError from dvc.exceptions import NotDvcRepoError from dvc.exceptions import OutputNotFoundError from dvc.exceptions import UrlNotDvcRepoError +from dvc.exceptions import FileMissingError from dvc.external_repo import external_repo from dvc.path_info import PathInfo from dvc.stage import Stage @@ -18,14 +19,28 @@ from dvc.utils.fs import remove logger = logging.getLogger(__name__) +def _is_git_file(repo, path): + if not os.path.isabs(path): + try: + output = repo.find_out_by_relpath(path) + if not output.use_cache: + return True + except OutputNotFoundError: + return True + return False + + def _copy_git_file(repo, src, dst): src_full_path = os.path.join(repo.root_dir, src) dst_full_path = os.path.abspath(dst) - if os.path.isdir(src_full_path): - shutil.copytree(src_full_path, dst_full_path) - else: - shutil.copy2(src_full_path, dst_full_path) + try: + if os.path.isdir(src_full_path): + shutil.copytree(src_full_path, dst_full_path) + else: + shutil.copy2(src_full_path, dst_full_path) + except FileNotFoundError: + raise FileMissingError(src) @staticmethod @@ -44,8 +59,7 @@ def get(url, path, out=None, rev=None): tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid())) try: with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo: - full_path = os.path.join(repo.root_dir, path) - if os.path.exists(full_path) and not os.path.isabs(path): + if _is_git_file(repo, path): _copy_git_file(repo, path, out) return diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 2dbd62daf..82c19b217 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -10,6 +10,7 @@ from dvc.config import Config from dvc.exceptions import GetDVCFileError from dvc.exceptions import UrlNotDvcRepoError from dvc.exceptions import NoOutputInExternalRepoError +from dvc.exceptions import FileMissingError from dvc.repo import Repo from dvc.system import System from dvc.utils import makedirs @@ -143,11 +144,20 @@ def test_non_cached_output(tmp_path, erepo): # https://github.com/iterative/dvc/pull/2837#discussion_r352123053 -def test_fails_with_non_git_files(erepo): +def test_fails_with_files_outside_repo(erepo): with pytest.raises(NoOutputInExternalRepoError): Repo.get(erepo.root_dir, "/root/") +def test_fails_with_non_existing_files(erepo): + fname = "file_does_not_exist" + with pytest.raises( + FileMissingError, + match="Can't find '{}' neither locally nor on remote".format(fname), + ): + Repo.get(erepo.root_dir, fname) + + @pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"]) def test_get_to_dir(dname, erepo): src = erepo.FOO From 7debaca22e8d3e4b99c6ddd54a373ed6d99b3f9d Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Thu, 5 Dec 2019 21:02:18 +0100 Subject: [PATCH 053/118] fixup! fix error message for git files --- dvc/repo/get.py | 1 + tests/func/test_get.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 5902575c6..8f82fc8e9 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -15,6 +15,7 @@ from dvc.stage import Stage from dvc.state import StateNoop from dvc.utils import resolve_output from dvc.utils.fs import remove +from dvc.utils.compat import FileNotFoundError logger = logging.getLogger(__name__) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 82c19b217..930afd155 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -15,6 +15,7 @@ from dvc.repo import Repo from dvc.system import System from dvc.utils import makedirs from dvc.utils.compat import fspath +from dvc.utils import fspath_py35 from tests.utils import trees_equal @@ -135,7 +136,7 @@ def test_non_cached_output(tmp_path, erepo): ) erepo.dvc.scm.add(["non_cached_file", "non_cached_file.dvc"]) erepo.dvc.scm.commit("add non-cached output") - os.chdir(tmp_path) + os.chdir(fspath_py35(tmp_path)) Repo.get(erepo.root_dir, "non_cached_file") src = os.path.join(erepo.root_dir, "non_cached_file") From 7e8bdbd6035221c4ebae2058b21011f9b7ad6e7a Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Thu, 5 Dec 2019 19:37:15 -0600 Subject: [PATCH 054/118] remove pathlib / fspath changes related to the patch --- dvc/analytics.py | 21 ++++++++++++++------- dvc/config.py | 19 ++++++++++++------- dvc/lock.py | 6 +++--- dvc/utils/compat.py | 6 ------ 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 6501ece8f..0be3ed91a 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -1,5 +1,6 @@ import json import logging +import os import platform import requests import sys @@ -16,7 +17,7 @@ from dvc.lock import Lock, LockError from dvc.repo import Repo from dvc.scm import SCM from dvc.utils import env2bool, is_binary, makedirs -from dvc.utils.compat import str, FileNotFoundError, convert_to_unicode +from dvc.utils.compat import str, FileNotFoundError logger = logging.getLogger(__name__) @@ -132,18 +133,24 @@ def find_or_create_user_id(): IDs are generated randomly with UUID. """ config_dir = Config.get_global_config_dir() - fname = config_dir / "user_id" - lockfile = fname.with_suffix(".lock") + fname = os.path.join(config_dir, "user_id") + lockfile = os.path.join(config_dir, "user_id.lock") + + # Since the `fname` and `lockfile` are under the global config, + # we need to make sure such directory exist already + makedirs(config_dir, exist_ok=True) try: with Lock(lockfile): try: - user_id = json.loads(fname.read_text())["user_id"] + with open(fname, "r") as fobj: + user_id = json.load(fobj)["user_id"] + except (FileNotFoundError, ValueError, AttributeError): user_id = str(uuid.uuid4()) - makedirs(fname.parent, exist_ok=True) - data = convert_to_unicode(json.dumps({"user_id": user_id})) - fname.write_text(data) + + with open(fname, "w") as fobj: + json.dump({"user_id": user_id}, fobj) return user_id diff --git a/dvc/config.py b/dvc/config.py index 32c7a0977..62823deb6 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -13,7 +13,7 @@ from voluptuous import All, Any, Lower, Range, Coerce, Match from dvc.exceptions import DvcException from dvc.exceptions import NotDvcRepoError -from dvc.utils.compat import open, str, pathlib, fspath +from dvc.utils.compat import open, str logger = logging.getLogger(__name__) @@ -284,8 +284,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes """ from appdirs import user_config_dir - return pathlib.Path( - user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) + return user_config_dir( + appname=Config.APPNAME, appauthor=Config.APPAUTHOR ) @staticmethod @@ -297,8 +297,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes """ from appdirs import site_config_dir - return pathlib.Path( - site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) + return site_config_dir( + appname=Config.APPNAME, appauthor=Config.APPAUTHOR ) @staticmethod @@ -342,8 +342,13 @@ class Config(object): # pylint: disable=too-many-instance-attributes return ret def _load_configs(self): - system_config_file = fspath(self.get_system_config_dir() / self.CONFIG) - global_config_file = fspath(self.get_global_config_dir() / self.CONFIG) + system_config_file = os.path.join( + self.get_system_config_dir(), self.CONFIG + ) + + global_config_file = os.path.join( + self.get_global_config_dir(), self.CONFIG + ) self._system_config = configobj.ConfigObj(system_config_file) self._global_config = configobj.ConfigObj(global_config_file) diff --git a/dvc/lock.py b/dvc/lock.py index d4925e963..147118c0f 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -10,7 +10,7 @@ from funcy.py3 import lkeep from dvc.exceptions import DvcException from dvc.utils import makedirs -from dvc.utils.compat import is_py3, fspath +from dvc.utils.compat import is_py3 DEFAULT_TIMEOUT = 5 @@ -58,7 +58,7 @@ if is_py3: # [1] https://github.com/iterative/dvc/issues/2582 self._hostname = socket.gethostname() - self._lockfile = fspath(lockfile) + self._lockfile = lockfile self._lifetime = timedelta(days=365) # Lock for good by default self._separator = flufl.lock.SEP self._set_claimfile() @@ -111,7 +111,7 @@ else: """ def __init__(self, lockfile, tmp_dir=None): - self.lockfile = fspath(lockfile) + self.lockfile = lockfile self._lock = None @property diff --git a/dvc/utils/compat.py b/dvc/utils/compat.py index ab079c1d0..372c8d2c6 100644 --- a/dvc/utils/compat.py +++ b/dvc/utils/compat.py @@ -207,12 +207,6 @@ except ImportError: except AttributeError: if hasattr(path_type, "__fspath__"): raise - - # Support for pathlib.Path in Python 3.5, since it doesn't - # implement `__fspath__` - if isinstance(path, pathlib.Path): - return str(path) - else: raise TypeError( "expected str, bytes or os.PathLike object, " From bc2471cd30b817adaf71d0244a450ef617d12ac4 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Thu, 5 Dec 2019 19:56:45 -0600 Subject: [PATCH 055/118] tests: adjust scm_class schema --- tests/unit/test_analytics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 69754fd60..11678617a 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -25,7 +25,7 @@ def test_runtime_info(tmp_global_config): { "dvc_version": Any(builtin_str, str), "is_binary": bool, - "scm_class": Any("Git", "NoSCM"), + "scm_class": Any("Git", None), "user_id": Any(builtin_str, str), "system_info": dict, } From c82d9dcbc104592c6c2525fb401cfce76aa5b450 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Thu, 5 Dec 2019 23:18:44 -0600 Subject: [PATCH 056/118] compat: bring back unicode literals --- dvc/analytics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dvc/analytics.py b/dvc/analytics.py index 0be3ed91a..f0bc086d7 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import json import logging import os From 9e6e43093582ebd85c0626bb37eb6fee6c0735ee Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 6 Dec 2019 16:56:12 +0200 Subject: [PATCH 057/118] import-url: add directories to examples https://github.com/iterative/dvc/pull/2894 --- dvc/command/imp_url.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dvc/command/imp_url.py b/dvc/command/imp_url.py index 46b24f8aa..e072bc1e3 100644 --- a/dvc/command/imp_url.py +++ b/dvc/command/imp_url.py @@ -44,12 +44,17 @@ def add_parser(subparsers, parent_parser): "url", help="Supported urls:\n" "/path/to/file\n" + "/path/to/directory\n" "C:\\\\path\\to\\file\n" + "C:\\\\path\\to\\directory\n" "https://example.com/path/to/file\n" "s3://bucket/path/to/file\n" + "s3://bucket/path/to/directory\n" "gs://bucket/path/to/file\n" + "gs://bucket/path/to/directory\n" "hdfs://example.com/path/to/file\n" "ssh://example.com:/path/to/file\n" + "ssh://example.com:/path/to/directory\n" "remote://myremote/path/to/file (see `dvc remote`)", ) import_parser.add_argument( From f704e5468bf949e8b6789e2b047ef04fc91f4c1f Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Fri, 6 Dec 2019 10:20:30 -0600 Subject: [PATCH 058/118] tests: stringify tmp_global_config since it doesnt return a pathlike object --- tests/unit/test_analytics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 11678617a..46a40ae89 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -15,7 +15,7 @@ def tmp_global_config(tmp_path): Fixture to prevent modifying the actual global config """ with mock.patch( - "dvc.config.Config.get_global_config_dir", return_value=tmp_path + "dvc.config.Config.get_global_config_dir", return_value=str(tmp_path) ): yield From 659eb04705b2788693c8723f144e461b9b13a281 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Fri, 6 Dec 2019 16:00:05 +0100 Subject: [PATCH 059/118] test: write a usage text for dir_helpers --- tests/dir_helpers.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 1d9789790..050925e66 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -1,3 +1,47 @@ +""" +The goal of this module is making dvc functional tests setup a breeze. This +includes a temporary dir, initializing git and dvc repos and bootstraping some +file structure. + +The cornerstone of these fixtures is `tmp_dir`, which creates a temporary dir +and changes path to it, it might be combined with `scm` and `dvc` to initialize +empty git and dvc repos. `tmp_dir` returns a Path instance, which should save +you from using `open()`, `os` and `os.path` utils many times: + + (tmp_dir / "some_file").write_text("some text") + # ... + assert "some text" == (tmp_dir / "some_file").read_text() + assert (tmp_dir / "some_file").exists() + +Additionally it provides `.gen()`, `.scm_gen()` and `.dvc_gen()` methoda to +bootstrap a required file structure in a single call: + + # Generate a dir with files + tmp_dir.gen({"dir": {"file": "file text", "second_file": "..."}}) + + # Generate a single file, dirs will be created along the way + tmp_dir.gen("dir/file", "file text") + + # Generate + git add + tmp_dir.scm_gen({"file1": "...", ...}) + + # Generate + git add + git commit + tmp_dir.scm_gen({"file1": "...", ...}, commit="add files") + + # Generate + dvc add + tmp_dir.dvc_gen({"file1": "...", ...}) + + # Generate + dvc add + git commit -am "..." + # This commits stages to git not the generated files. + tmp_dir.dvc_gen({"file1": "...", ...}, commit="add files") + +Making it easier to bootstrap things has a supergoal of incentivizing a move +from global repo template to creating everything inplace, which: + + - makes all path references local to test, enhancing readability + - allows using telling filenames, e.g. "git_tracked_file" instead of "foo" + - does not create whatever is not needed +""" from __future__ import unicode_literals import os From 5bec7007614e44545fbae34ff6bc5018ea32492b Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Fri, 6 Dec 2019 18:35:03 +0100 Subject: [PATCH 060/118] test: accept Paths in tmp_dir.gen() --- tests/dir_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 050925e66..ac47f65a9 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -88,7 +88,7 @@ class TmpDir(pathlib.Path): ) def gen(self, struct, text=""): - if isinstance(struct, basestring): + if isinstance(struct, (basestring, pathlib.PurePath)): struct = {struct: text} self._gen(struct) From 811855ade22655d29fcc281914d7bb1c9983575f Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Thu, 5 Dec 2019 22:54:10 +0100 Subject: [PATCH 061/118] fixes --- dvc/exceptions.py | 6 ++++++ dvc/repo/get.py | 47 +++++++++++++++++++++--------------------- tests/func/test_get.py | 10 ++++----- 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index e5fb6f00d..fbb8c7eaa 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -268,6 +268,12 @@ class FileMissingError(DvcException): ) +class PathOutsideRepoError(DvcException): + def __init__(self, path, repo): + msg = "The path '{}' does not exist in the target repository '{}'." + super(PathOutsideRepoError, self).__init__(msg.format(path, repo)) + + class DvcIgnoreInCollectedDirError(DvcException): def __init__(self, ignore_dirname): super(DvcIgnoreInCollectedDirError, self).__init__( diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 8f82fc8e9..9185a1ef7 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -8,7 +8,7 @@ from dvc.exceptions import GetDVCFileError from dvc.exceptions import NotDvcRepoError from dvc.exceptions import OutputNotFoundError from dvc.exceptions import UrlNotDvcRepoError -from dvc.exceptions import FileMissingError +from dvc.exceptions import PathOutsideRepoError from dvc.external_repo import external_repo from dvc.path_info import PathInfo from dvc.stage import Stage @@ -20,18 +20,7 @@ from dvc.utils.compat import FileNotFoundError logger = logging.getLogger(__name__) -def _is_git_file(repo, path): - if not os.path.isabs(path): - try: - output = repo.find_out_by_relpath(path) - if not output.use_cache: - return True - except OutputNotFoundError: - return True - return False - - -def _copy_git_file(repo, src, dst): +def _copy_git_file(repo, src, dst, repo_url): src_full_path = os.path.join(repo.root_dir, src) dst_full_path = os.path.abspath(dst) @@ -41,7 +30,7 @@ def _copy_git_file(repo, src, dst): else: shutil.copy2(src_full_path, dst_full_path) except FileNotFoundError: - raise FileMissingError(src) + raise PathOutsideRepoError(src, repo_url) @staticmethod @@ -60,10 +49,6 @@ def get(url, path, out=None, rev=None): tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid())) try: with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo: - if _is_git_file(repo, path): - _copy_git_file(repo, path, out) - return - # Note: we need to replace state, because in case of getting DVC # dependency on CIFS or NFS filesystems, sqlite-based state # will be unable to obtain lock @@ -80,12 +65,28 @@ def get(url, path, out=None, rev=None): # the same cache file might be used a few times in a directory. repo.cache.local.cache_types = ["reflink", "hardlink", "copy"] - o = repo.find_out_by_relpath(path) + output = None + output_error = None + + try: + output = repo.find_out_by_relpath(path) + except OutputNotFoundError as ex: + output_error = ex + + is_git_file = output_error and not os.path.isabs(path) + is_not_cached = output and not output.use_cache + + if is_git_file or is_not_cached: + return _copy_git_file(repo, path, out, url) + + if output_error: + raise output_error + with repo.state: - repo.cloud.pull(o.get_used_cache()) - o.path_info = PathInfo(os.path.abspath(out)) - with o.repo.state: - o.checkout() + repo.cloud.pull(output.get_used_cache()) + output.path_info = PathInfo(os.path.abspath(out)) + with output.repo.state: + output.checkout() except NotDvcRepoError: raise UrlNotDvcRepoError(url) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 930afd155..f838985fb 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -10,7 +10,7 @@ from dvc.config import Config from dvc.exceptions import GetDVCFileError from dvc.exceptions import UrlNotDvcRepoError from dvc.exceptions import NoOutputInExternalRepoError -from dvc.exceptions import FileMissingError +from dvc.exceptions import PathOutsideRepoError from dvc.repo import Repo from dvc.system import System from dvc.utils import makedirs @@ -152,10 +152,10 @@ def test_fails_with_files_outside_repo(erepo): def test_fails_with_non_existing_files(erepo): fname = "file_does_not_exist" - with pytest.raises( - FileMissingError, - match="Can't find '{}' neither locally nor on remote".format(fname), - ): + err = "The path '{}' does not exist in the target repository '{}'".format( + fname, erepo.root_dir + ) + with pytest.raises(PathOutsideRepoError, match=err): Repo.get(erepo.root_dir, fname) From da064e2ede7cc041ac16842b6993d88b18743da0 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Fri, 6 Dec 2019 12:28:06 -0600 Subject: [PATCH 062/118] analytics: remove the report after sending it --- dvc/analytics.py | 9 +++++++++ dvc/command/daemon.py | 8 ++------ tests/func/test_analytics.py | 4 +--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index f0bc086d7..624b6c0c2 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -81,12 +81,21 @@ def is_enabled(): def send(report): + """ + Side effect: Removes the report after sending it. + + The report is generated and stored in a temporary file, see: + `collect_and_send_report`. Sending happens on another process, + thus, the need of removing such file afterwards. + """ url = "https://analytics.dvc.org" headers = {"content-type": "application/json"} with open(report, "rb") as fobj: requests.post(url, data=fobj, headers=headers, timeout=5) + os.remove(report) + def scm_in_use(): try: diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index 68d4b36f6..da9210584 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -1,7 +1,5 @@ from __future__ import unicode_literals -import os - from dvc.command.base import CmdBaseNoRepo from dvc.command.base import fix_subparsers @@ -12,6 +10,7 @@ class CmdDaemonBase(CmdBaseNoRepo): class CmdDaemonUpdater(CmdDaemonBase): def run(self): + import os from dvc.repo import Repo from dvc.updater import Updater @@ -27,10 +26,7 @@ class CmdDaemonAnalytics(CmdDaemonBase): def run(self): from dvc import analytics - report = self.args.target - - analytics.send(report) - os.remove(report) + analytics.send(self.args.target) return 0 diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py index df7454965..863b0e329 100644 --- a/tests/func/test_analytics.py +++ b/tests/func/test_analytics.py @@ -5,13 +5,11 @@ from dvc.utils.compat import fspath @mock.patch("dvc.analytics.send") -@mock.patch("os.remove") -def test_daemon_analytics(mock_remove, mock_send, tmp_path): +def test_daemon_analytics(mock_send, tmp_path): report = fspath(tmp_path) assert 0 == main(["daemon", "analytics", report]) mock_send.assert_called_with(report) - mock_remove.assert_called_with(report) @mock.patch("dvc.daemon._spawn") From d59bceb71afbe22c80102b5101a933235a19f42d Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Fri, 6 Dec 2019 20:20:25 +0100 Subject: [PATCH 063/118] fixes --- dvc/repo/get.py | 13 ++++++------- tests/func/test_get.py | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 9185a1ef7..534047504 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -24,11 +24,12 @@ def _copy_git_file(repo, src, dst, repo_url): src_full_path = os.path.join(repo.root_dir, src) dst_full_path = os.path.abspath(dst) + if os.path.isdir(src_full_path): + shutil.copytree(src_full_path, dst_full_path) + return + try: - if os.path.isdir(src_full_path): - shutil.copytree(src_full_path, dst_full_path) - else: - shutil.copy2(src_full_path, dst_full_path) + shutil.copy2(src_full_path, dst_full_path) except FileNotFoundError: raise PathOutsideRepoError(src, repo_url) @@ -80,7 +81,7 @@ def get(url, path, out=None, rev=None): return _copy_git_file(repo, path, out, url) if output_error: - raise output_error + raise OutputNotFoundError(path) with repo.state: repo.cloud.pull(output.get_used_cache()) @@ -90,7 +91,5 @@ def get(url, path, out=None, rev=None): except NotDvcRepoError: raise UrlNotDvcRepoError(url) - except OutputNotFoundError: - raise OutputNotFoundError(path) finally: remove(tmp_dir) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index f838985fb..c7147447e 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -9,7 +9,7 @@ import pytest from dvc.config import Config from dvc.exceptions import GetDVCFileError from dvc.exceptions import UrlNotDvcRepoError -from dvc.exceptions import NoOutputInExternalRepoError +from dvc.exceptions import OutputNotFoundError from dvc.exceptions import PathOutsideRepoError from dvc.repo import Repo from dvc.system import System @@ -146,7 +146,7 @@ def test_non_cached_output(tmp_path, erepo): # https://github.com/iterative/dvc/pull/2837#discussion_r352123053 def test_fails_with_files_outside_repo(erepo): - with pytest.raises(NoOutputInExternalRepoError): + with pytest.raises(OutputNotFoundError): Repo.get(erepo.root_dir, "/root/") From 5a8033415da073dbccd5fb1b15e2bf3603208881 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Fri, 6 Dec 2019 13:54:17 -0600 Subject: [PATCH 064/118] tests: use str, builtin_str in schema --- dvc/analytics.py | 2 -- tests/unit/test_analytics.py | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 624b6c0c2..088b24eb8 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import json import logging import os diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 46a40ae89..60e7a565b 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -9,6 +9,9 @@ from dvc import analytics from dvc.utils.compat import str, builtin_str +string = Any(str, builtin_str) + + @pytest.fixture def tmp_global_config(tmp_path): """ @@ -23,10 +26,10 @@ def tmp_global_config(tmp_path): def test_runtime_info(tmp_global_config): schema = Schema( { - "dvc_version": Any(builtin_str, str), + "dvc_version": string, "is_binary": bool, "scm_class": Any("Git", None), - "user_id": Any(builtin_str, str), + "user_id": string, "system_info": dict, } ) @@ -38,7 +41,7 @@ def test_runtime_info(tmp_global_config): def test_send(mock_post, tmp_path): url = "https://analytics.dvc.org" report = {"name": "dummy report"} - fname = tmp_path / "report" + fname = str(tmp_path / "report") with open(fname, "w") as fobj: json.dump(report, fobj) @@ -80,19 +83,19 @@ def test_system_info(): "windows_version_build": int, "windows_version_major": int, "windows_version_minor": int, - "windows_version_service_pack": str, + "windows_version_service_pack": string, } ) if system == "Darwin": - schema = schema.extend({"mac_version": str}) + schema = schema.extend({"mac_version": string}) if system == "Linux": schema = schema.extend( { - "linux_distro": str, - "linux_distro_like": Any(str, None), - "linux_distro_version": Any(str, None), + "linux_distro": string, + "linux_distro_like": string, + "linux_distro_version": string, } ) From 29b34bacc2b963d9868cf0b84215f4ca49b67f9a Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Fri, 6 Dec 2019 21:55:54 +0100 Subject: [PATCH 065/118] fixup! fixes --- dvc/repo/get.py | 3 ++- tests/func/test_get.py | 8 ++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 534047504..7fbe6a5d7 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -78,7 +78,8 @@ def get(url, path, out=None, rev=None): is_not_cached = output and not output.use_cache if is_git_file or is_not_cached: - return _copy_git_file(repo, path, out, url) + _copy_git_file(repo, path, out, url) + return if output_error: raise OutputNotFoundError(path) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index c7147447e..11f70ecc3 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -151,12 +151,8 @@ def test_fails_with_files_outside_repo(erepo): def test_fails_with_non_existing_files(erepo): - fname = "file_does_not_exist" - err = "The path '{}' does not exist in the target repository '{}'".format( - fname, erepo.root_dir - ) - with pytest.raises(PathOutsideRepoError, match=err): - Repo.get(erepo.root_dir, fname) + with pytest.raises(PathOutsideRepoError): + Repo.get(erepo.root_dir, "file_does_not_exist") @pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"]) From 2145daa130c6c5323f001863d92e8997a2d6c429 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Fri, 6 Dec 2019 16:23:58 -0600 Subject: [PATCH 066/118] analytics: define private methods --- dvc/analytics.py | 34 +++++++++++++++++----------------- tests/func/test_analytics.py | 2 +- tests/unit/test_analytics.py | 8 ++++---- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 088b24eb8..2826a95f7 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -35,7 +35,7 @@ def collect_and_send_report(args=None, return_code=None): report as a JSON, where the _collector_ generates it and the _sender_ removes it after sending it. """ - report = runtime_info() + report = _runtime_info() # Include command execution information on the report report.update( @@ -50,19 +50,6 @@ def collect_and_send_report(args=None, return_code=None): daemon(["analytics", fobj.name]) -def runtime_info(): - """ - Gather information from the environment where DVC runs to fill a report. - """ - return { - "dvc_version": __version__, - "is_binary": is_binary(), - "scm_class": scm_in_use(), - "system_info": system_info(), - "user_id": find_or_create_user_id(), - } - - def is_enabled(): if env2bool("DVC_TEST"): return False @@ -95,7 +82,7 @@ def send(report): os.remove(report) -def scm_in_use(): +def _scm_in_use(): try: scm = SCM(root_dir=Repo.find_root()) return type(scm).__name__ @@ -103,7 +90,20 @@ def scm_in_use(): pass -def system_info(): +def _runtime_info(): + """ + Gather information from the environment where DVC runs to fill a report. + """ + return { + "dvc_version": __version__, + "is_binary": is_binary(), + "scm_class": _scm_in_use(), + "system_info": _system_info(), + "user_id": _find_or_create_user_id(), + } + + +def _system_info(): system = platform.system() if system == "Windows": @@ -131,7 +131,7 @@ def system_info(): return {"os": system.lower()} -def find_or_create_user_id(): +def _find_or_create_user_id(): """ The user's ID is stored on a file under the global config directory. diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py index 863b0e329..f7cc0bfed 100644 --- a/tests/func/test_analytics.py +++ b/tests/func/test_analytics.py @@ -14,7 +14,7 @@ def test_daemon_analytics(mock_send, tmp_path): @mock.patch("dvc.daemon._spawn") @mock.patch("dvc.analytics.is_enabled", return_value=True) -@mock.patch("dvc.analytics.runtime_info", return_value={}) +@mock.patch("dvc.analytics._runtime_info", return_value={}) def test_main_analytics(mock_is_enabled, mock_daemon, mock_report, dvc_repo): assert 0 == main(["add", "foo"]) assert mock_is_enabled.called diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 60e7a565b..7d4138564 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -34,7 +34,7 @@ def test_runtime_info(tmp_global_config): } ) - assert schema(analytics.runtime_info()) + assert schema(analytics._runtime_info()) @mock.patch("requests.post") @@ -99,11 +99,11 @@ def test_system_info(): } ) - assert schema(analytics.system_info()) + assert schema(analytics._system_info()) def test_find_or_create_user_id(tmp_global_config): - created = analytics.find_or_create_user_id() - found = analytics.find_or_create_user_id() + created = analytics._find_or_create_user_id() + found = analytics._find_or_create_user_id() assert created == found From c6c67ace20b2a8978b5379c61444deb0196f2ce6 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Fri, 6 Dec 2019 23:46:00 +0100 Subject: [PATCH 067/118] test: create empty dirs via tmp_dir.gen() --- tests/dir_helpers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 1d9789790..f58c03ed8 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -55,7 +55,10 @@ class TmpDir(pathlib.Path): path = (prefix or self) / name if isinstance(contents, dict): - self._gen(contents, prefix=path) + if not contents: + makedirs(path, exist_ok=True) + else: + self._gen(contents, prefix=path) else: makedirs(path.parent, exist_ok=True) if is_py2 and isinstance(contents, str): From 42096c5bfc37608d04c6e4a4cacb8d677af22443 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Fri, 6 Dec 2019 23:46:20 +0100 Subject: [PATCH 068/118] test: migrate test_ignore to new helpers --- tests/func/test_ignore.py | 221 +++++++++++++------------------------- 1 file changed, 75 insertions(+), 146 deletions(-) diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index b09a06a78..84b8af966 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -1,186 +1,115 @@ -import itertools +# encoding: utf-8 +from __future__ import unicode_literals import os import shutil - import pytest from dvc.exceptions import DvcIgnoreInCollectedDirError -from dvc.ignore import DvcIgnore -from dvc.ignore import DvcIgnoreDirs -from dvc.ignore import DvcIgnoreFilter -from dvc.ignore import DvcIgnorePatterns +from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns from dvc.scm.tree import WorkingTree -from dvc.utils.compat import cast_bytes +from dvc.utils import walk_files +from dvc.utils.compat import fspath from dvc.utils.fs import get_mtime_and_size -from tests.basic_env import TestDvc -from tests.utils import to_posixpath -class TestDvcIgnore(TestDvc): - def setUp(self): - super(TestDvcIgnore, self).setUp() +def test_ignore_in_child_dir(tmp_dir, dvc): + tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}}) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - def _get_all_paths(self): - - paths = [] - for root, dirs, files in self.dvc.tree.walk( - self.dvc.root_dir, dvcignore=self.dvc.dvcignore - ): - for dname in dirs: - paths.append(os.path.join(root, dname)) - - for fname in files: - paths.append(os.path.join(root, fname)) - - return paths - - def test_ignore_in_child_dir(self): - ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE) - with open(ignore_file, "w") as fobj: - fobj.write("data_dir/data") - - forbidden_path = os.path.join(self.dvc.root_dir, self.DATA) - all_paths = self._get_all_paths() - - self.assertNotIn(forbidden_path, all_paths) - - def test_ignore_in_child_dir_unicode(self): - ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE) - with open(ignore_file, "wb") as fobj: - fobj.write(cast_bytes(self.UNICODE, "utf-8")) - - forbidden_path = os.path.join(self.dvc.root_dir, self.UNICODE) - all_paths = self._get_all_paths() - - self.assertNotIn(forbidden_path, all_paths) - - def test_ignore_in_parent_dir(self): - ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE) - with open(ignore_file, "w") as fobj: - fobj.write("data_dir/data") - - os.chdir(self.DATA_DIR) - - forbidden_path = os.path.join(self.dvc.root_dir, self.DATA) - all_paths = self._get_all_paths() - - self.assertNotIn(forbidden_path, all_paths) + assert set(walk_files("dir", dvc.dvcignore)) == {"dir/other"} -def test_metadata_unchanged_when_moving_ignored_file(dvc_repo, repo_dir): - new_data_path = repo_dir.DATA_SUB + "_new" +def test_ignore_in_parent_dir(tmp_dir, dvc, monkeypatch): + tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}}) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE) - repo_dir.create( - ignore_file, - "\n".join( - [to_posixpath(repo_dir.DATA_SUB), to_posixpath(new_data_path)] - ), - ) - - mtime_sig, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) - - shutil.move(repo_dir.DATA_SUB, new_data_path) - - new_mtime_sig, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) - - assert new_mtime_sig == mtime_sig - assert new_size == size + monkeypatch.chdir("dir") + assert set(walk_files(".", dvc.dvcignore)) == {"./other"} -def test_mtime_changed_when_moving_non_ignored_file(dvc_repo, repo_dir): - new_data_path = repo_dir.DATA_SUB + "_new" - mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) +def test_ignore_unicode(tmp_dir, dvc): + tmp_dir.gen({"dir": {"тест": "проверка", "other": "text"}}) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест") - shutil.move(repo_dir.DATA_SUB, new_data_path) - new_mtime, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) - - assert new_mtime != mtime - assert new_size == size + assert set(walk_files("dir", dvc.dvcignore)) == {"dir/other"} -def test_metadata_unchanged_on_ignored_file_deletion(dvc_repo, repo_dir): - ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE) - repo_dir.create(ignore_file, to_posixpath(repo_dir.DATA_SUB)) +def test_rename_ignored_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}}) - mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*") + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) - os.remove(repo_dir.DATA_SUB) - new_mtime, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) + shutil.move("dir/ignored", "dir/ignored_new") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) - assert new_mtime == mtime - assert new_size == size + assert new_mtime == mtime and new_size == size -def test_metadata_changed_on_non_ignored_file_deletion(dvc_repo, repo_dir): - mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) +def test_rename_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) - os.remove(repo_dir.DATA_SUB) - new_mtime_sig, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) + shutil.move("dir/foo", "dir/foo_new") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) - assert new_mtime_sig != mtime - assert new_size != size + assert new_mtime != mtime and new_size == size -def test_should_raise_on_dvcignore_in_out_dir(dvc_repo, repo_dir): - ignore_file = os.path.join(repo_dir.DATA_DIR, DvcIgnore.DVCIGNORE_FILE) - repo_dir.create(ignore_file, "") +def test_remove_ignored_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}}) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") + + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) + + os.remove("dir/ignored") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) + + assert new_mtime == mtime and new_size == size + + +def test_remove_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) + + os.remove("dir/foo") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) + + assert new_mtime != mtime and new_size != size + + +def test_dvcignore_in_out_dir(tmp_dir, dvc): + tmp_dir.gen({"dir": {"foo": "foo", DvcIgnore.DVCIGNORE_FILE: ""}}) with pytest.raises(DvcIgnoreInCollectedDirError): - dvc_repo.add(repo_dir.DATA_DIR) + dvc.add("dir") -@pytest.mark.parametrize("dname", [TestDvc.DATA_DIR, TestDvc.DATA_SUB_DIR]) -def test_ignore_collecting_dvcignores(repo_dir, dname): - top_ignore_file = os.path.join( - repo_dir.root_dir, os.path.dirname(dname), DvcIgnore.DVCIGNORE_FILE - ) - repo_dir.create(top_ignore_file, os.path.basename(dname)) +# @efiop why do we need to parametrize this? +@pytest.mark.parametrize("dname", ["dir", "dir/subdir"]) +def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname): + tmp_dir.gen({"dir": {"subdir": {}}}) - ignore_file = os.path.join( - repo_dir.root_dir, dname, DvcIgnore.DVCIGNORE_FILE - ) - repo_dir.create(ignore_file, repo_dir.FOO) + top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE) + top_ignore_file.write_text(os.path.basename(dname)) - assert DvcIgnoreFilter( - repo_dir.root_dir, WorkingTree(repo_dir.root_dir) - ).ignores == { + ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE + ignore_file.write_text("foo") + + assert dvc.dvcignore.ignores == { DvcIgnoreDirs([".git", ".hg", ".dvc"]), - DvcIgnorePatterns(top_ignore_file, WorkingTree(repo_dir.root_dir)), + DvcIgnorePatterns(fspath(top_ignore_file), WorkingTree(dvc.root_dir)), } -def test_ignore_on_branch(git, dvc_repo, repo_dir): - dvc_repo.add(repo_dir.DATA_DIR) - dvc_repo.scm.commit("add data dir") +def test_ignore_on_branch(tmp_dir, scm, dvc): + tmp_dir.scm_gen({"foo": "foo", "bar": "bar"}, commit="add files") - branch_name = "branch_one" - dvc_repo.scm.checkout(branch_name, create_new=True) + scm.checkout("branch", create_new=True) + tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore") - repo_dir.create(DvcIgnore.DVCIGNORE_FILE, to_posixpath(repo_dir.DATA_SUB)) - dvc_repo.scm.add([DvcIgnore.DVCIGNORE_FILE]) - git.index.commit("add ignore") + scm.checkout("master") + assert set(walk_files(".", dvc.dvcignore)) == {"./foo", "./bar"} - dvc_repo.scm.checkout("master") - - git_tree = dvc_repo.scm.get_tree(branch_name) - branch_data_files = set( - itertools.chain.from_iterable( - [ - files - for _, _, files in dvc_repo.tree.walk( - repo_dir.DATA_DIR, - dvcignore=DvcIgnoreFilter(repo_dir.root_dir, git_tree), - ) - ] - ) - ) - assert branch_data_files == {"data"} + dvc.tree = scm.get_tree("branch") + assert set(walk_files(".", dvc.dvcignore)) == {"./bar"} From 77fcd75b7da5c7d29e6489071e500087748d3e46 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Sat, 7 Dec 2019 13:16:42 +0100 Subject: [PATCH 069/118] test: use utf-8 in tmp_dir.gen() --- tests/dir_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index f58c03ed8..336fd3474 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -64,7 +64,7 @@ class TmpDir(pathlib.Path): if is_py2 and isinstance(contents, str): path.write_bytes(contents) else: - path.write_text(contents) + path.write_text(contents, encoding="utf-8") def dvc_gen(self, struct, text="", commit=None): paths = self.gen(struct, text) From 79902988fcead014e9d532c82c1310085f087707 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Sat, 7 Dec 2019 13:17:08 +0100 Subject: [PATCH 070/118] test: fix files set comparisons in dvcignore tests under windows --- tests/func/test_ignore.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 84b8af966..b08d57415 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -11,27 +11,24 @@ from dvc.utils import walk_files from dvc.utils.compat import fspath from dvc.utils.fs import get_mtime_and_size +from tests.utils import to_posixpath -def test_ignore_in_child_dir(tmp_dir, dvc): + +def test_ignore(tmp_dir, dvc, monkeypatch): tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - assert set(walk_files("dir", dvc.dvcignore)) == {"dir/other"} - - -def test_ignore_in_parent_dir(tmp_dir, dvc, monkeypatch): - tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}}) - tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") + assert _files_set("dir", dvc.dvcignore) == {"dir/other"} monkeypatch.chdir("dir") - assert set(walk_files(".", dvc.dvcignore)) == {"./other"} + assert _files_set(".", dvc.dvcignore) == {"./other"} def test_ignore_unicode(tmp_dir, dvc): tmp_dir.gen({"dir": {"тест": "проверка", "other": "text"}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест") - assert set(walk_files("dir", dvc.dvcignore)) == {"dir/other"} + assert _files_set("dir", dvc.dvcignore) == {"dir/other"} def test_rename_ignored_file(tmp_dir, dvc): @@ -109,7 +106,11 @@ def test_ignore_on_branch(tmp_dir, scm, dvc): tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore") scm.checkout("master") - assert set(walk_files(".", dvc.dvcignore)) == {"./foo", "./bar"} + assert _files_set(".", dvc.dvcignore) == {"./foo", "./bar"} dvc.tree = scm.get_tree("branch") - assert set(walk_files(".", dvc.dvcignore)) == {"./bar"} + assert _files_set(".", dvc.dvcignore) == {"./bar"} + + +def _files_set(root, dvcignore): + return {to_posixpath(f) for f in walk_files(root, dvcignore)} From 0dd310d3873ceb55096652f3f08eab90a6e381f1 Mon Sep 17 00:00:00 2001 From: Mayur Kulkarni Date: Sat, 7 Dec 2019 18:39:32 +0530 Subject: [PATCH 071/118] external_repo: fix for local file import fail If the URL is local dvc repo, the import/get should fetch from the source project cache. Fixes #2599 --- dvc/external_repo.py | 49 +++++++++++++++++++++++++++++++- tests/func/test_external_repo.py | 9 ++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 0555f2d75..cbd33b250 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -5,9 +5,10 @@ import tempfile from contextlib import contextmanager from distutils.dir_util import copy_tree +from dvc.remote import RemoteConfig from funcy import retry -from dvc.config import NoRemoteError +from dvc.config import NoRemoteError, ConfigError from dvc.exceptions import RemoteNotSpecifiedInExternalRepoError from dvc.exceptions import NoOutputInExternalRepoError from dvc.exceptions import OutputNotFoundError @@ -62,16 +63,29 @@ def _external_repo(url=None, rev=None, cache_dir=None): # Adjust new clone/copy to fit rev and cache_dir repo = Repo(new_path) + # Adjust original repo for pointing remote towards its' cache + original_repo = Repo(url) + rconfig = RemoteConfig(original_repo.config) try: if rev is not None: repo.scm.checkout(rev) + if not _is_local(url) and not _remote_config_exists(rconfig): + # check if the URL is local and no default remote + # add default remote pointing to the original repo's cache location + rconfig.add("upstream", + original_repo.cache.local.cache_dir, + default=True) + original_repo.scm.add([original_repo.config.config_file]) + original_repo.scm.commit("add remote") + if cache_dir is not None: cache_config = CacheConfig(repo.config) cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL) finally: # Need to close/reopen repo to force config reread repo.close() + original_repo.close() REPO_CACHE[key] = new_path return new_path @@ -100,3 +114,36 @@ def _clone_repo(url, path): git = Git.clone(url, path) git.close() + + +def _remote_config_exists(rconfig): + """ + Checks if default remote config is present. + Args: + rconfig: a remote config + + Returns: + True if the remote config exists, else False + """ + try: + default = rconfig.get_default() + except ConfigError: + default = None + return True if default else False + + +def _is_local(url): + """ + Checks if the URL is local or not. + Args: + url: url + + Returns: + True, if the URL is local else False + """ + remote_urls = {"azure://", "gs://", "http://", "https://", + "oss://", "s3://", "hdfs://"} + for remote_url in remote_urls: + if url.startswith(remote_url): + return False + return True diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index de2b8b20b..d21e056a9 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -1,5 +1,6 @@ import os +from dvc.repo import Repo from mock import patch from dvc.external_repo import external_repo @@ -26,3 +27,11 @@ def test_external_repo(erepo): assert path_isin(repo.cache.local.cache_dir, repo.root_dir) assert mock.call_count == 1 + + +def test_external_repo_import_without_remote(erepo, dvc_repo): + src = erepo.CODE + dst = dvc_repo.root_dir + + Repo.get(erepo.root_dir, src, dst) + assert os.path.exists(dst + "/" + erepo.CODE) From 6c12a2e555ab75ecd81a37723047e969e0178519 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Sat, 7 Dec 2019 15:50:03 +0100 Subject: [PATCH 072/118] test: fix unicode test_ignore issue under Windows Py2 --- tests/dir_helpers.py | 4 ++-- tests/func/test_ignore.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 336fd3474..17379d3a6 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -6,7 +6,7 @@ import pytest from funcy.py3 import lmap, retry from dvc.utils import makedirs -from dvc.utils.compat import basestring, is_py2, pathlib, fspath, fspath_py35 +from dvc.utils.compat import basestring, pathlib, fspath, fspath_py35, bytes __all__ = ["tmp_dir", "scm", "dvc", "repo_template", "run_copy", "erepo_dir"] @@ -61,7 +61,7 @@ class TmpDir(pathlib.Path): self._gen(contents, prefix=path) else: makedirs(path.parent, exist_ok=True) - if is_py2 and isinstance(contents, str): + if isinstance(contents, bytes): path.write_bytes(contents) else: path.write_text(contents, encoding="utf-8") diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index b08d57415..ee4c5f7a1 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -25,7 +25,12 @@ def test_ignore(tmp_dir, dvc, monkeypatch): def test_ignore_unicode(tmp_dir, dvc): - tmp_dir.gen({"dir": {"тест": "проверка", "other": "text"}}) + tmp_dir.gen({"dir": {"other": "text"}}) + # Path() doesn't handle unicode paths in Windows/Python 2 + # I don't know to debug it further, waiting till Python 2 EOL + with open("dir/тест", "wb") as fd: + fd.write("проверка".encode("utf-8")) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест") assert _files_set("dir", dvc.dvcignore) == {"dir/other"} From c50bd17bb48fcf07b76fa32ac7bda709e15f36d5 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Sat, 7 Dec 2019 21:13:06 -0600 Subject: [PATCH 073/118] analytics: collect execution info only when available --- dvc/analytics.py | 13 ++++++------- tests/func/test_analytics.py | 6 ++---- tests/unit/test_analytics.py | 25 +++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 2826a95f7..39e6234ad 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -37,13 +37,12 @@ def collect_and_send_report(args=None, return_code=None): """ report = _runtime_info() - # Include command execution information on the report - report.update( - { - "cmd_class": args.func.__name__ if hasattr(args, "func") else None, - "cmd_return_code": return_code, - } - ) + # Include command execution information on the report only when available. + if args and hasattr(args, "func"): + report.update({"cmd_class": args.func.__name__}) + + if return_code is not None: + report.update({"cmd_return_code": return_code}) with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: json.dump(report, fobj) diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py index f7cc0bfed..6004c6412 100644 --- a/tests/func/test_analytics.py +++ b/tests/func/test_analytics.py @@ -12,11 +12,9 @@ def test_daemon_analytics(mock_send, tmp_path): mock_send.assert_called_with(report) -@mock.patch("dvc.daemon._spawn") +@mock.patch("dvc.analytics.collect_and_send_report") @mock.patch("dvc.analytics.is_enabled", return_value=True) -@mock.patch("dvc.analytics._runtime_info", return_value={}) -def test_main_analytics(mock_is_enabled, mock_daemon, mock_report, dvc_repo): +def test_main_analytics(mock_is_enabled, mock_report, dvc_repo): assert 0 == main(["add", "foo"]) assert mock_is_enabled.called assert mock_report.called - assert mock_daemon.called diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 7d4138564..ece2501a9 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -6,6 +6,7 @@ import json from voluptuous import Schema, Any from dvc import analytics +from dvc.cli import parse_args from dvc.utils.compat import str, builtin_str @@ -23,6 +24,30 @@ def tmp_global_config(tmp_path): yield +@mock.patch("dvc.daemon._spawn") +@mock.patch("json.dump") +def test_collect_and_send_report(mock_json, mock_daemon, tmp_global_config): + analytics.collect_and_send_report() + report = mock_json.call_args[0][0] + + with pytest.raises(KeyError): + report["cmd_class"] + + with pytest.raises(KeyError): + report["cmd_return_code"] + + args = parse_args(['add', 'foo']) + return_code = 0 + + analytics.collect_and_send_report(args, return_code) + report = mock_json.call_args[0][0] + + assert report["cmd_class"] == "CmdAdd" + assert report["cmd_return_code"] == return_code + + assert mock_daemon.call_count == 2 + + def test_runtime_info(tmp_global_config): schema = Schema( { From 146e75be3ee0c49d24c94e171dcdac625c2d4979 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Sat, 7 Dec 2019 21:15:11 -0600 Subject: [PATCH 074/118] analytics: raise error when collecting a not supported os --- dvc/analytics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 39e6234ad..26243d609 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -127,7 +127,8 @@ def _system_info(): "linux_distro_version": distro.version(), } - return {"os": system.lower()} + # We don't collect data for any other system. + raise NotImplementedError def _find_or_create_user_id(): From c9f958dd001f145d09a48ac8b31568d9b0e1f261 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Sat, 7 Dec 2019 21:15:38 -0600 Subject: [PATCH 075/118] analytics: AttributeError -> KeyError --- dvc/analytics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 26243d609..2a9e86352 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -155,7 +155,7 @@ def _find_or_create_user_id(): with open(fname, "r") as fobj: user_id = json.load(fobj)["user_id"] - except (FileNotFoundError, ValueError, AttributeError): + except (FileNotFoundError, ValueError, KeyError): user_id = str(uuid.uuid4()) with open(fname, "w") as fobj: From 3244f10e7ff8922640993ce03cedb721bd53b30b Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Sat, 7 Dec 2019 21:15:47 -0600 Subject: [PATCH 076/118] :nail_care: add dot to the end of the comment --- dvc/analytics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 2a9e86352..1ae02f60a 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -146,7 +146,7 @@ def _find_or_create_user_id(): lockfile = os.path.join(config_dir, "user_id.lock") # Since the `fname` and `lockfile` are under the global config, - # we need to make sure such directory exist already + # we need to make sure such directory exist already. makedirs(config_dir, exist_ok=True) try: From 6e4c3d6acd1f6ee0891c25543188455c4a12c6f6 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Sat, 7 Dec 2019 21:18:28 -0600 Subject: [PATCH 077/118] tests: require keys on analytics report schema --- tests/unit/test_analytics.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index ece2501a9..a7163c48a 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -56,7 +56,8 @@ def test_runtime_info(tmp_global_config): "scm_class": Any("Git", None), "user_id": string, "system_info": dict, - } + }, + required=True, ) assert schema(analytics._runtime_info()) @@ -98,7 +99,7 @@ def test_is_enabled(dvc_repo, config, result, monkeypatch, tmp_global_config): def test_system_info(): - schema = Schema({"os": Any("windows", "mac", "linux")}) + schema = Schema({"os": Any("windows", "mac", "linux")}, required=True) system = platform.system() From 5dd63008783f3a6fe3dbaa9244b6b5acb8aa656d Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Sat, 7 Dec 2019 21:19:54 -0600 Subject: [PATCH 078/118] :nail_care: black --- tests/func/test_analytics.py | 2 +- tests/unit/test_analytics.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py index 6004c6412..a2eccfd62 100644 --- a/tests/func/test_analytics.py +++ b/tests/func/test_analytics.py @@ -14,7 +14,7 @@ def test_daemon_analytics(mock_send, tmp_path): @mock.patch("dvc.analytics.collect_and_send_report") @mock.patch("dvc.analytics.is_enabled", return_value=True) -def test_main_analytics(mock_is_enabled, mock_report, dvc_repo): +def test_main_analytics(mock_is_enabled, mock_report, dvc_repo): assert 0 == main(["add", "foo"]) assert mock_is_enabled.called assert mock_report.called diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index a7163c48a..43eb37dcb 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -36,7 +36,7 @@ def test_collect_and_send_report(mock_json, mock_daemon, tmp_global_config): with pytest.raises(KeyError): report["cmd_return_code"] - args = parse_args(['add', 'foo']) + args = parse_args(["add", "foo"]) return_code = 0 analytics.collect_and_send_report(args, return_code) From 9e24be14f0cfa588db1fa1b18085581d742958cd Mon Sep 17 00:00:00 2001 From: Mayur Kularni Date: Sun, 8 Dec 2019 14:02:44 +0530 Subject: [PATCH 079/118] incorporated feedback --- dvc/external_repo.py | 47 ++++++++++++++++---------------------------- tests/conftest.py | 6 ------ 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index cbd33b250..1216e836c 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -63,21 +63,24 @@ def _external_repo(url=None, rev=None, cache_dir=None): # Adjust new clone/copy to fit rev and cache_dir repo = Repo(new_path) - # Adjust original repo for pointing remote towards its' cache - original_repo = Repo(url) - rconfig = RemoteConfig(original_repo.config) + original_repo = None try: if rev is not None: repo.scm.checkout(rev) - - if not _is_local(url) and not _remote_config_exists(rconfig): - # check if the URL is local and no default remote - # add default remote pointing to the original repo's cache location - rconfig.add("upstream", - original_repo.cache.local.cache_dir, - default=True) - original_repo.scm.add([original_repo.config.config_file]) - original_repo.scm.commit("add remote") + # check if the URL is local and no default remote + # add default remote pointing to the original repo's cache location + if not os.path.isdir(url): + # Adjust original repo for pointing remote towards its' cache + original_repo = Repo(url) + rconfig = RemoteConfig(original_repo.config) + if not _remote_config_exists(rconfig): + rconfig.add( + "upstream", + original_repo.cache.local.cache_dir, + default=True, + ) + original_repo.scm.add([original_repo.config.config_file]) + original_repo.scm.commit("add remote") if cache_dir is not None: cache_config = CacheConfig(repo.config) @@ -85,7 +88,8 @@ def _external_repo(url=None, rev=None, cache_dir=None): finally: # Need to close/reopen repo to force config reread repo.close() - original_repo.close() + if original_repo: + original_repo.close() REPO_CACHE[key] = new_path return new_path @@ -130,20 +134,3 @@ def _remote_config_exists(rconfig): except ConfigError: default = None return True if default else False - - -def _is_local(url): - """ - Checks if the URL is local or not. - Args: - url: url - - Returns: - True, if the URL is local else False - """ - remote_urls = {"azure://", "gs://", "http://", "https://", - "oss://", "s3://", "hdfs://"} - for remote_url in remote_urls: - if url.startswith(remote_url): - return False - return True diff --git a/tests/conftest.py b/tests/conftest.py index 0cdb204b8..730b6d32d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,6 @@ from git.exc import GitCommandNotFound from .basic_env import TestDirFixture from .basic_env import TestDvcGitFixture from .basic_env import TestGitFixture -from dvc.remote.config import RemoteConfig from dvc.remote.ssh.connection import SSHConnection from dvc.repo import Repo as DvcRepo from dvc.utils.compat import cast_bytes_py2 @@ -148,11 +147,6 @@ def erepo(repo_dir): repo.dvc.scm.add([stage_foo.path, stage_bar.path, stage_data_dir.path]) repo.dvc.scm.commit("init repo") - rconfig = RemoteConfig(repo.dvc.config) - rconfig.add("upstream", repo.dvc.cache.local.cache_dir, default=True) - repo.dvc.scm.add([repo.dvc.config.config_file]) - repo.dvc.scm.commit("add remote") - repo.create("version", "master") repo.dvc.add("version") repo.dvc.scm.add([".gitignore", "version.dvc"]) From 779a0a1b15c9722263695f032ea95ce757291b60 Mon Sep 17 00:00:00 2001 From: Mayur Kularni Date: Sun, 8 Dec 2019 14:35:51 +0530 Subject: [PATCH 080/118] codeclimate recommendation fix --- dvc/external_repo.py | 5 ++--- tests/func/test_external_repo.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 1216e836c..2b24735b6 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -4,10 +4,9 @@ import os import tempfile from contextlib import contextmanager from distutils.dir_util import copy_tree - -from dvc.remote import RemoteConfig from funcy import retry +from dvc.remote import RemoteConfig from dvc.config import NoRemoteError, ConfigError from dvc.exceptions import RemoteNotSpecifiedInExternalRepoError from dvc.exceptions import NoOutputInExternalRepoError @@ -133,4 +132,4 @@ def _remote_config_exists(rconfig): default = rconfig.get_default() except ConfigError: default = None - return True if default else False + return bool(default) diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index d21e056a9..e6a8ee7bc 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -1,8 +1,8 @@ import os -from dvc.repo import Repo from mock import patch +from dvc.repo import Repo from dvc.external_repo import external_repo from dvc.scm.git import Git from dvc.utils.fs import path_isin From e1c5f6c5c062052d615eb3766bb81abcc2de3179 Mon Sep 17 00:00:00 2001 From: Mayur Kulkarni Date: Sat, 7 Dec 2019 18:39:32 +0530 Subject: [PATCH 081/118] external_repo: fix for local file import fail If the URL is local dvc repo, the import/get should fetch from the source project cache. Fixes #2599 --- dvc/external_repo.py | 50 +++++++++++++++++++++++++++++++- tests/func/test_external_repo.py | 9 ++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 8f53da4ef..61b4a85f3 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -5,10 +5,12 @@ import tempfile from contextlib import contextmanager from distutils.dir_util import copy_tree +from dvc.remote import RemoteConfig from funcy import retry -from dvc.config import NoRemoteError + from dvc.exceptions import NoRemoteInExternalRepoError +from dvc.config import NoRemoteError, ConfigError from dvc.exceptions import NoOutputInExternalRepoError from dvc.exceptions import OutputNotFoundError from dvc.utils.fs import remove @@ -68,13 +70,26 @@ def _external_repo(url=None, rev=None, cache_dir=None): _git_checkout(new_path, rev) repo = Repo(new_path) + # Adjust original repo for pointing remote towards its' cache + original_repo = Repo(url) + rconfig = RemoteConfig(original_repo.config) try: + if not _is_local(url) and not _remote_config_exists(rconfig): + # check if the URL is local and no default remote + # add default remote pointing to the original repo's cache location + rconfig.add("upstream", + original_repo.cache.local.cache_dir, + default=True) + original_repo.scm.add([original_repo.config.config_file]) + original_repo.scm.commit("add remote") + if cache_dir is not None: cache_config = CacheConfig(repo.config) cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL) finally: # Need to close/reopen repo to force config reread repo.close() + original_repo.close() REPO_CACHE[key] = new_path return new_path @@ -113,3 +128,36 @@ def _clone_repo(url, path): git = Git.clone(url, path) git.close() + + +def _remote_config_exists(rconfig): + """ + Checks if default remote config is present. + Args: + rconfig: a remote config + + Returns: + True if the remote config exists, else False + """ + try: + default = rconfig.get_default() + except ConfigError: + default = None + return True if default else False + + +def _is_local(url): + """ + Checks if the URL is local or not. + Args: + url: url + + Returns: + True, if the URL is local else False + """ + remote_urls = {"azure://", "gs://", "http://", "https://", + "oss://", "s3://", "hdfs://"} + for remote_url in remote_urls: + if url.startswith(remote_url): + return False + return True diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index de2b8b20b..d21e056a9 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -1,5 +1,6 @@ import os +from dvc.repo import Repo from mock import patch from dvc.external_repo import external_repo @@ -26,3 +27,11 @@ def test_external_repo(erepo): assert path_isin(repo.cache.local.cache_dir, repo.root_dir) assert mock.call_count == 1 + + +def test_external_repo_import_without_remote(erepo, dvc_repo): + src = erepo.CODE + dst = dvc_repo.root_dir + + Repo.get(erepo.root_dir, src, dst) + assert os.path.exists(dst + "/" + erepo.CODE) From ca20a1c93aed85562f4b8e11fb7dde42c3cee73f Mon Sep 17 00:00:00 2001 From: Mayur Kularni Date: Sun, 8 Dec 2019 14:02:44 +0530 Subject: [PATCH 082/118] incorporated feedback --- dvc/external_repo.py | 43 ++++++++++++++----------------------------- tests/conftest.py | 6 ------ 2 files changed, 14 insertions(+), 35 deletions(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 61b4a85f3..e4f6586b1 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -70,18 +70,21 @@ def _external_repo(url=None, rev=None, cache_dir=None): _git_checkout(new_path, rev) repo = Repo(new_path) - # Adjust original repo for pointing remote towards its' cache - original_repo = Repo(url) - rconfig = RemoteConfig(original_repo.config) try: - if not _is_local(url) and not _remote_config_exists(rconfig): - # check if the URL is local and no default remote - # add default remote pointing to the original repo's cache location - rconfig.add("upstream", - original_repo.cache.local.cache_dir, - default=True) - original_repo.scm.add([original_repo.config.config_file]) - original_repo.scm.commit("add remote") + # check if the URL is local and no default remote + # add default remote pointing to the original repo's cache location + if not os.path.isdir(url): + # Adjust original repo for pointing remote towards its' cache + original_repo = Repo(url) + rconfig = RemoteConfig(original_repo.config) + if not _remote_config_exists(rconfig): + rconfig.add( + "upstream", + original_repo.cache.local.cache_dir, + default=True, + ) + original_repo.scm.add([original_repo.config.config_file]) + original_repo.scm.commit("add remote") if cache_dir is not None: cache_config = CacheConfig(repo.config) @@ -89,7 +92,6 @@ def _external_repo(url=None, rev=None, cache_dir=None): finally: # Need to close/reopen repo to force config reread repo.close() - original_repo.close() REPO_CACHE[key] = new_path return new_path @@ -144,20 +146,3 @@ def _remote_config_exists(rconfig): except ConfigError: default = None return True if default else False - - -def _is_local(url): - """ - Checks if the URL is local or not. - Args: - url: url - - Returns: - True, if the URL is local else False - """ - remote_urls = {"azure://", "gs://", "http://", "https://", - "oss://", "s3://", "hdfs://"} - for remote_url in remote_urls: - if url.startswith(remote_url): - return False - return True diff --git a/tests/conftest.py b/tests/conftest.py index 46b059d5e..3eca524c0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,6 @@ import pytest from git import Repo from git.exc import GitCommandNotFound -from dvc.remote.config import RemoteConfig from dvc.remote.ssh.connection import SSHConnection from dvc.repo import Repo as DvcRepo from dvc.utils.compat import cast_bytes_py2 @@ -148,11 +147,6 @@ def erepo(repo_dir): repo.dvc.scm.add([stage_foo.path, stage_bar.path, stage_data_dir.path]) repo.dvc.scm.commit("init repo") - rconfig = RemoteConfig(repo.dvc.config) - rconfig.add("upstream", repo.dvc.cache.local.cache_dir, default=True) - repo.dvc.scm.add([repo.dvc.config.config_file]) - repo.dvc.scm.commit("add remote") - repo.create("version", "master") repo.dvc.add("version") repo.dvc.scm.add([".gitignore", "version.dvc"]) From c6050efbe3ebde6d07ceb8c762df586ba7f1f670 Mon Sep 17 00:00:00 2001 From: Mayur Kularni Date: Sun, 8 Dec 2019 14:35:51 +0530 Subject: [PATCH 083/118] codeclimate recommendation fix --- dvc/external_repo.py | 6 ++---- tests/func/test_external_repo.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index e4f6586b1..abe4c6d04 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -4,12 +4,10 @@ import os import tempfile from contextlib import contextmanager from distutils.dir_util import copy_tree - -from dvc.remote import RemoteConfig from funcy import retry - from dvc.exceptions import NoRemoteInExternalRepoError +from dvc.remote import RemoteConfig from dvc.config import NoRemoteError, ConfigError from dvc.exceptions import NoOutputInExternalRepoError from dvc.exceptions import OutputNotFoundError @@ -145,4 +143,4 @@ def _remote_config_exists(rconfig): default = rconfig.get_default() except ConfigError: default = None - return True if default else False + return bool(default) diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index d21e056a9..e6a8ee7bc 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -1,8 +1,8 @@ import os -from dvc.repo import Repo from mock import patch +from dvc.repo import Repo from dvc.external_repo import external_repo from dvc.scm.git import Git from dvc.utils.fs import path_isin From 610e43a0ab95b87bb293834f12c920fb39de2a1e Mon Sep 17 00:00:00 2001 From: Mayur Kularni Date: Sun, 8 Dec 2019 18:48:20 +0530 Subject: [PATCH 084/118] removed original repo ops --- dvc/external_repo.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index abe4c6d04..b85baa927 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -71,18 +71,17 @@ def _external_repo(url=None, rev=None, cache_dir=None): try: # check if the URL is local and no default remote # add default remote pointing to the original repo's cache location - if not os.path.isdir(url): - # Adjust original repo for pointing remote towards its' cache - original_repo = Repo(url) - rconfig = RemoteConfig(original_repo.config) + if os.path.isdir(url): + rconfig = RemoteConfig(repo.config) if not _remote_config_exists(rconfig): rconfig.add( "upstream", - original_repo.cache.local.cache_dir, + repo.cache.local.cache_dir, default=True, + level=Config.LEVEL_LOCAL, ) - original_repo.scm.add([original_repo.config.config_file]) - original_repo.scm.commit("add remote") + repo.scm.add([repo.config.config_file]) + repo.scm.commit("add remote") if cache_dir is not None: cache_config = CacheConfig(repo.config) From 84616e8e5ed894faef00ef16f6bff4e1e38b3f4d Mon Sep 17 00:00:00 2001 From: Mayur Kularni Date: Sun, 8 Dec 2019 20:25:24 +0530 Subject: [PATCH 085/118] removed commit logic --- dvc/external_repo.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 026c5bb13..3f55f128a 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -68,9 +68,8 @@ def _external_repo(url=None, rev=None, cache_dir=None): _git_checkout(new_path, rev) repo = Repo(new_path) - original_repo = None try: - # check if the URL is local and no default remote + # check if the URL is local and no default remote is present # add default remote pointing to the original repo's cache location if os.path.isdir(url): rconfig = RemoteConfig(repo.config) @@ -81,8 +80,6 @@ def _external_repo(url=None, rev=None, cache_dir=None): default=True, level=Config.LEVEL_LOCAL, ) - repo.scm.add([repo.config.config_file]) - repo.scm.commit("add remote") if cache_dir is not None: cache_config = CacheConfig(repo.config) @@ -90,8 +87,6 @@ def _external_repo(url=None, rev=None, cache_dir=None): finally: # Need to close/reopen repo to force config reread repo.close() - if original_repo: - original_repo.close() REPO_CACHE[key] = new_path return new_path From 5b9daea2b3885383d6332a7b676bfbd04626fc1a Mon Sep 17 00:00:00 2001 From: Mayur Kularni Date: Mon, 9 Dec 2019 12:11:49 +0530 Subject: [PATCH 086/118] fixed test case --- dvc/external_repo.py | 4 +++- tests/func/test_external_repo.py | 9 ++++++--- tests/func/test_get.py | 1 - 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 3f55f128a..ed4b8b39e 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -74,12 +74,14 @@ def _external_repo(url=None, rev=None, cache_dir=None): if os.path.isdir(url): rconfig = RemoteConfig(repo.config) if not _remote_config_exists(rconfig): + original_repo = Repo(url) rconfig.add( "upstream", - repo.cache.local.cache_dir, + original_repo.cache.local.cache_dir, default=True, level=Config.LEVEL_LOCAL, ) + original_repo.close() if cache_dir is not None: cache_config = CacheConfig(repo.config) diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index e6a8ee7bc..4c34505a0 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -1,3 +1,4 @@ +import filecmp import os from mock import patch @@ -30,8 +31,10 @@ def test_external_repo(erepo): def test_external_repo_import_without_remote(erepo, dvc_repo): - src = erepo.CODE - dst = dvc_repo.root_dir + src = erepo.FOO + dst = os.path.join(dvc_repo.root_dir, "foo") Repo.get(erepo.root_dir, src, dst) - assert os.path.exists(dst + "/" + erepo.CODE) + assert os.path.exists(dst) + assert os.path.isfile(dst) + assert filecmp.cmp(erepo.FOO, dst, shallow=False) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 07551c5f3..8b99547f0 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -101,7 +101,6 @@ def test_get_from_non_dvc_master(erepo, tmp_path, monkeypatch, caplog): with caplog.at_level(logging.INFO, logger="dvc"): Repo.get(erepo._root_dir, erepo.FOO, out=imported_file, rev="branch") - assert caplog.text == "" assert filecmp.cmp( os.path.join(erepo._root_dir, erepo.FOO), imported_file, shallow=False ) From 0cd6106189e373b54efa2d8f5b917f230dfabda1 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Mon, 9 Dec 2019 11:24:41 +0300 Subject: [PATCH 087/118] Update tests/dir_helpers.py Co-Authored-By: Mr. Outis --- tests/dir_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index ac47f65a9..2645e5e29 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -13,7 +13,7 @@ you from using `open()`, `os` and `os.path` utils many times: assert "some text" == (tmp_dir / "some_file").read_text() assert (tmp_dir / "some_file").exists() -Additionally it provides `.gen()`, `.scm_gen()` and `.dvc_gen()` methoda to +Additionally it provides `.gen()`, `.scm_gen()` and `.dvc_gen()` methods to bootstrap a required file structure in a single call: # Generate a dir with files From a59041aeab54ef5be2b943ed93a2454bf3a9010f Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Mon, 9 Dec 2019 11:33:35 +0300 Subject: [PATCH 088/118] test: update dir_helpers usage Co-Authored-By: Mr. Outis --- tests/dir_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 2645e5e29..74e920a01 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -40,7 +40,7 @@ from global repo template to creating everything inplace, which: - makes all path references local to test, enhancing readability - allows using telling filenames, e.g. "git_tracked_file" instead of "foo" - - does not create whatever is not needed + - does not create unnecessary files """ from __future__ import unicode_literals From 1da133b9ed62964decce674fd6446e5ed52ee6e2 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Mon, 9 Dec 2019 11:34:10 +0300 Subject: [PATCH 089/118] test: update dir_helpers usage Co-Authored-By: Mr. Outis --- tests/dir_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 74e920a01..105b6d968 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -1,6 +1,6 @@ """ The goal of this module is making dvc functional tests setup a breeze. This -includes a temporary dir, initializing git and dvc repos and bootstraping some +includes a temporary dir, initializing git and dvc repos and bootstrapping some file structure. The cornerstone of these fixtures is `tmp_dir`, which creates a temporary dir From b27136884551a7dd86a9173764423fddfd122ab2 Mon Sep 17 00:00:00 2001 From: Mayur Kularni Date: Mon, 9 Dec 2019 14:14:49 +0530 Subject: [PATCH 090/118] Moved `remote_config_exists` into `RemoteConfig` --- dvc/external_repo.py | 20 ++------------------ dvc/remote/config.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index ed4b8b39e..4e9c50e98 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -8,7 +8,7 @@ from funcy import retry from dvc.exceptions import NoRemoteInExternalRepoError from dvc.remote import RemoteConfig -from dvc.config import NoRemoteError, ConfigError +from dvc.config import NoRemoteError from dvc.exceptions import NoOutputInExternalRepoError from dvc.exceptions import OutputNotFoundError from dvc.utils.fs import remove @@ -73,7 +73,7 @@ def _external_repo(url=None, rev=None, cache_dir=None): # add default remote pointing to the original repo's cache location if os.path.isdir(url): rconfig = RemoteConfig(repo.config) - if not _remote_config_exists(rconfig): + if not rconfig.remote_config_exists(): original_repo = Repo(url) rconfig.add( "upstream", @@ -127,19 +127,3 @@ def _clone_repo(url, path): git = Git.clone(url, path) git.close() - - -def _remote_config_exists(rconfig): - """ - Checks if default remote config is present. - Args: - rconfig: a remote config - - Returns: - True if the remote config exists, else False - """ - try: - default = rconfig.get_default() - except ConfigError: - default = None - return bool(default) diff --git a/dvc/remote/config.py b/dvc/remote/config.py index dccad52ba..e1dc2dfb9 100644 --- a/dvc/remote/config.py +++ b/dvc/remote/config.py @@ -156,3 +156,18 @@ class RemoteConfig(object): return self.config.get( Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level ) + + def remote_config_exists(self): + """ + Checks if default remote config is present. + Args: + rconfig: a remote config + + Returns: + True if the remote config exists, else False + """ + try: + default = self.get_default() + except ConfigError: + default = None + return bool(default) From 24fc8beaeb2ec577c204fe2d14de4d27f4084b23 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 9 Dec 2019 15:27:54 +0200 Subject: [PATCH 091/118] Update test_ignore.py --- tests/func/test_ignore.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index ee4c5f7a1..97fac4644 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -87,7 +87,6 @@ def test_dvcignore_in_out_dir(tmp_dir, dvc): dvc.add("dir") -# @efiop why do we need to parametrize this? @pytest.mark.parametrize("dname", ["dir", "dir/subdir"]) def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname): tmp_dir.gen({"dir": {"subdir": {}}}) From c4565ce2acdcad0765f7f3ba1c3c7030eee2362f Mon Sep 17 00:00:00 2001 From: pawel Date: Thu, 5 Dec 2019 18:14:04 +0100 Subject: [PATCH 092/118] add: test: convert to dir helpers 3 TestAddUnsupportedFile TestAdd TestAddDirectory --- tests/func/test_add.py | 76 ++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 43 deletions(-) diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 00ba50122..300cccc45 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -31,57 +31,47 @@ from tests.utils import get_gitignore_content from tests.utils import spy -class TestAdd(TestDvc): - def test(self): - md5 = file_md5(self.FOO)[0] +def test_add(tmp_dir, dvc): + stage, = tmp_dir.dvc_gen({"foo": "foo"}) + md5, _ = file_md5("foo") - stages = self.dvc.add(self.FOO) - self.assertEqual(len(stages), 1) - stage = stages[0] - self.assertTrue(stage is not None) + assert stage is not None - self.assertIsInstance(stage, Stage) - self.assertTrue(os.path.isfile(stage.path)) - self.assertEqual(len(stage.outs), 1) - self.assertEqual(len(stage.deps), 0) - self.assertEqual(stage.cmd, None) - self.assertEqual(stage.outs[0].info["md5"], md5) - self.assertEqual(stage.md5, "ee343f2482f53efffc109be83cc976ac") - - def test_unicode(self): - fname = "\xe1" - - with open(fname, "w") as fobj: - fobj.write("something") - - stage = self.dvc.add(fname)[0] - - self.assertTrue(os.path.isfile(stage.path)) + assert isinstance(stage, Stage) + assert os.path.isfile(stage.path) + assert len(stage.outs) == 1 + assert len(stage.deps) == 0 + assert stage.cmd is None + assert stage.outs[0].info["md5"] == md5 + assert stage.md5 == "ee343f2482f53efffc109be83cc976ac" -class TestAddUnsupportedFile(TestDvc): - def test(self): - with self.assertRaises(DvcException): - self.dvc.add("unsupported://unsupported") +def test_add_unicode(tmp_dir, dvc): + with open("\xe1", "wb") as fd: + fd.write("something".encode("utf-8")) + + stage, = dvc.add("\xe1") + + assert os.path.isfile(stage.path) -class TestAddDirectory(TestDvc): - def test(self): - dname = "directory" - os.mkdir(dname) - self.create(os.path.join(dname, "file"), "file") - stages = self.dvc.add(dname) - self.assertEqual(len(stages), 1) - stage = stages[0] - self.assertTrue(stage is not None) - self.assertEqual(len(stage.deps), 0) - self.assertEqual(len(stage.outs), 1) +def test_add_unsupported_file(dvc): + with pytest.raises(DvcException): + dvc.add("unsupported://unsupported") - md5 = stage.outs[0].info["md5"] - dir_info = self.dvc.cache.local.load_dir_cache(md5) - for info in dir_info: - self.assertTrue("\\" not in info["relpath"]) +def test_add_directory(tmp_dir, dvc): + stage, = tmp_dir.dvc_gen({"dir": {"file": "file"}}) + + assert stage is not None + assert len(stage.deps) == 0 + assert len(stage.outs) == 1 + + md5 = stage.outs[0].info["md5"] + + dir_info = dvc.cache.local.load_dir_cache(md5) + for info in dir_info: + assert "\\" not in info["relpath"] class TestAddDirectoryRecursive(TestDvc): From 93280c937b9160003afb0d2f3fd473c03d6d9673 Mon Sep 17 00:00:00 2001 From: Anton Abrosimov Date: Mon, 9 Dec 2019 19:14:53 +0300 Subject: [PATCH 093/118] build_deb: Debian directory appended. Issue #2800 --- debian/changelog | 5 +++++ debian/compat | 1 + debian/control | 33 +++++++++++++++++++++++++++++++++ debian/copyright | 11 +++++++++++ debian/rules | 6 ++++++ debian/source/format | 1 + debian/watch | 4 ++++ 7 files changed, 61 insertions(+) create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/copyright create mode 100755 debian/rules create mode 100644 debian/source/format create mode 100644 debian/watch diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 000000000..ebe497cee --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +dvc (0.75.b1) experimental; urgency=low + + * First native Debian package version. + + -- Dmitry Petrov Sat, 07 Dec 2019 09:49:28 +0300 diff --git a/debian/compat b/debian/compat new file mode 100644 index 000000000..f599e28b8 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +10 diff --git a/debian/control b/debian/control new file mode 100644 index 000000000..2ddf61df8 --- /dev/null +++ b/debian/control @@ -0,0 +1,33 @@ +Source: dvc +Maintainer: Dmitry Petrov +Section: python +Priority: optional +Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9) +Standards-Version: 4.0.0 +Homepage: http://dvc.org +Vcs-Git: https://github.com/iterative/dvc.git +Vcs-browser: https://github.com/iterative/dvc + +Package: dvc +Architecture: all +Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2) +Recommends: python3-arrow (=0.14.0), + python3-paramiko (>=2.5.0), + python3-gssapi (>= 2.5.0), + python3-boto3 (>=1.9.86), + python3-pydrive (=1.3.1), + python3-azure-storage-blob (=12.1.0), + python3-oss2 (=2.6.1), + python3-google-cloud-storage (=1.19.0) +Description: Git for data scientists - manage your code and data together + Data Version Control or DVC is an open-source tool for data science and machine learning projects. + . + Key features: + . + 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services. + . + 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID. + . + 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built. + . + 4. Helps manage experiments with Git tags/branches and metrics tracking. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 000000000..523903a1c --- /dev/null +++ b/debian/copyright @@ -0,0 +1,11 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: DVC +Upstream-Contact: Dmitry Petrov +Source: https://github.com/iterative/dvc + +Files: * +Copyright: Dmitry Petrov +License: Apache 2.0 + On Debian systems, the full text of the Apache 2.0 + License can be found in the file + `/usr/share/common-licenses/Apache-2.0`. diff --git a/debian/rules b/debian/rules new file mode 100755 index 000000000..60ae08041 --- /dev/null +++ b/debian/rules @@ -0,0 +1,6 @@ +#!/usr/bin/make -f + +export PYBUILD_NAME=dvc +%: + dh $@ --with python3 --buildsystem=pybuild + diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 000000000..89ae9db8f --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/debian/watch b/debian/watch new file mode 100644 index 000000000..497d9f3d8 --- /dev/null +++ b/debian/watch @@ -0,0 +1,4 @@ +# please also check http://pypi.debian.net/dvc/watch +version=3 +opts=uversionmangle=s/(rc|a|b|c)/~$1/ \ +http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz))) \ No newline at end of file From 9607e5f873e0495872dd289aa40ea81b348bbdc1 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Mon, 9 Dec 2019 21:02:49 +0300 Subject: [PATCH 094/118] get: refactor, clean up and fix `dvc get` implementation Things changed: - both no output and no git file result into PathMissingError now - fail on missing cache or cache download error loudly - removed sqlite bypass, since we use nolock now anyway Refactoring: - streamlined logic and made it similar to Repo.open() one - moved exclusive exception classes to dvc.repo.get --- dvc/exceptions.py | 14 ------- dvc/repo/get.py | 89 +++++++++++++++++++++++------------------- tests/func/test_get.py | 14 +++---- 3 files changed, 54 insertions(+), 63 deletions(-) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index fbb8c7eaa..92c84ed0e 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -268,12 +268,6 @@ class FileMissingError(DvcException): ) -class PathOutsideRepoError(DvcException): - def __init__(self, path, repo): - msg = "The path '{}' does not exist in the target repository '{}'." - super(PathOutsideRepoError, self).__init__(msg.format(path, repo)) - - class DvcIgnoreInCollectedDirError(DvcException): def __init__(self, ignore_dirname): super(DvcIgnoreInCollectedDirError, self).__init__( @@ -289,14 +283,6 @@ class UrlNotDvcRepoError(DvcException): ) -class GetDVCFileError(DvcException): - def __init__(self): - super(GetDVCFileError, self).__init__( - "the given path is a DVC-file, you must specify a data file " - "or a directory" - ) - - class GitHookAlreadyExistsError(DvcException): def __init__(self, hook_name): super(GitHookAlreadyExistsError, self).__init__( diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 7fbe6a5d7..60c4c2e4a 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -1,37 +1,40 @@ import logging import os import shutil +from dvc.utils.compat import FileNotFoundError import shortuuid -from dvc.exceptions import GetDVCFileError -from dvc.exceptions import NotDvcRepoError -from dvc.exceptions import OutputNotFoundError -from dvc.exceptions import UrlNotDvcRepoError -from dvc.exceptions import PathOutsideRepoError +from dvc.exceptions import ( + DvcException, + NotDvcRepoError, + OutputNotFoundError, + UrlNotDvcRepoError, +) from dvc.external_repo import external_repo from dvc.path_info import PathInfo from dvc.stage import Stage -from dvc.state import StateNoop from dvc.utils import resolve_output from dvc.utils.fs import remove -from dvc.utils.compat import FileNotFoundError logger = logging.getLogger(__name__) -def _copy_git_file(repo, src, dst, repo_url): - src_full_path = os.path.join(repo.root_dir, src) - dst_full_path = os.path.abspath(dst) +class GetDVCFileError(DvcException): + def __init__(self): + super(GetDVCFileError, self).__init__( + "the given path is a DVC-file, you must specify a data file " + "or a directory" + ) - if os.path.isdir(src_full_path): - shutil.copytree(src_full_path, dst_full_path) - return - try: - shutil.copy2(src_full_path, dst_full_path) - except FileNotFoundError: - raise PathOutsideRepoError(src, repo_url) +class PathMissingError(DvcException): + def __init__(self, path, repo): + msg = ( + "The path '{}' does not exist in the target repository '{}'" + " neighther as an output nor a git-handled file." + ) + super(PathMissingError, self).__init__(msg.format(path, repo)) @staticmethod @@ -50,11 +53,6 @@ def get(url, path, out=None, rev=None): tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid())) try: with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo: - # Note: we need to replace state, because in case of getting DVC - # dependency on CIFS or NFS filesystems, sqlite-based state - # will be unable to obtain lock - repo.state = StateNoop() - # Try any links possible to avoid data duplication. # # Not using symlink, because we need to remove cache after we are @@ -66,31 +64,40 @@ def get(url, path, out=None, rev=None): # the same cache file might be used a few times in a directory. repo.cache.local.cache_types = ["reflink", "hardlink", "copy"] - output = None - output_error = None - try: output = repo.find_out_by_relpath(path) - except OutputNotFoundError as ex: - output_error = ex + except OutputNotFoundError: + output = None - is_git_file = output_error and not os.path.isabs(path) - is_not_cached = output and not output.use_cache + if output and output.use_cache: + _get_cached(repo, output, out) + else: + # Either an uncached out with absolute path or a user error + if os.path.isabs(path): + raise FileNotFoundError - if is_git_file or is_not_cached: - _copy_git_file(repo, path, out, url) - return - - if output_error: - raise OutputNotFoundError(path) - - with repo.state: - repo.cloud.pull(output.get_used_cache()) - output.path_info = PathInfo(os.path.abspath(out)) - with output.repo.state: - output.checkout() + _copy(os.path.join(repo.root_dir, path), out) + except (OutputNotFoundError, FileNotFoundError): + raise PathMissingError(path, url) except NotDvcRepoError: raise UrlNotDvcRepoError(url) finally: remove(tmp_dir) + + +def _get_cached(repo, output, out): + with repo.state: + repo.cloud.pull(output.get_used_cache()) + output.path_info = PathInfo(os.path.abspath(out)) + failed = output.checkout() + # This might happen when pull haven't really pulled all the files + if failed: + raise FileNotFoundError + + +def _copy(src, dst): + if os.path.isdir(src): + shutil.copytree(src, dst) + else: + shutil.copy2(src, dst) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 11f70ecc3..a48e76bd9 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -7,10 +7,8 @@ import os import pytest from dvc.config import Config -from dvc.exceptions import GetDVCFileError from dvc.exceptions import UrlNotDvcRepoError -from dvc.exceptions import OutputNotFoundError -from dvc.exceptions import PathOutsideRepoError +from dvc.repo.get import GetDVCFileError, PathMissingError from dvc.repo import Repo from dvc.system import System from dvc.utils import makedirs @@ -145,14 +143,14 @@ def test_non_cached_output(tmp_path, erepo): # https://github.com/iterative/dvc/pull/2837#discussion_r352123053 -def test_fails_with_files_outside_repo(erepo): - with pytest.raises(OutputNotFoundError): +def test_absolute_file_outside_repo(erepo): + with pytest.raises(PathMissingError): Repo.get(erepo.root_dir, "/root/") -def test_fails_with_non_existing_files(erepo): - with pytest.raises(PathOutsideRepoError): - Repo.get(erepo.root_dir, "file_does_not_exist") +def test_unknown_path(erepo): + with pytest.raises(PathMissingError): + Repo.get(erepo.root_dir, "a_non_existing_file") @pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"]) From 267ac9b3ffe891748f215c9e5ba4703075cd7faa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Redzy=C5=84ski?= Date: Mon, 9 Dec 2019 19:27:28 +0100 Subject: [PATCH 095/118] test: move: convert to dir helpers --- tests/func/test_move.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/func/test_move.py b/tests/func/test_move.py index eef8a603c..416ea3682 100644 --- a/tests/func/test_move.py +++ b/tests/func/test_move.py @@ -195,19 +195,21 @@ class TestMoveFileInsideDirectory(TestDvc): self.assertTrue(os.path.exists(dvc_fullpath)) -def test_move_should_save_stage_info(dvc_repo, repo_dir): - dvc_repo.add(repo_dir.DATA_DIR) +def test_move_should_save_stage_info(tmp_dir, dvc): + tmp_dir.dvc_gen({"old_name": {"file1": "file1"}}) - dvc_repo.move(repo_dir.DATA_DIR, "new_name") + dvc.move("old_name", "new_name") - assert dvc_repo.status() == {} + assert dvc.status() == {} -def test_should_move_to_dir_on_non_default_stage_file(dvc_repo, repo_dir): +def test_should_move_to_dir_on_non_default_stage_file(tmp_dir, dvc): stage_file_name = "stage.dvc" + tmp_dir.gen({"file": "file_content"}) - dvc_repo.add(repo_dir.FOO, fname=stage_file_name) + dvc.add("file", fname=stage_file_name) + os.mkdir("directory") - dvc_repo.move(repo_dir.FOO, repo_dir.DATA_DIR) + dvc.move("file", "directory") - assert os.path.exists(os.path.join(repo_dir.DATA_DIR, repo_dir.FOO)) + assert os.path.exists(os.path.join("directory", "file")) From 9c2c2b307dc44e5d3b9fc0f306d2f0a79a7b1d11 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 10 Dec 2019 01:02:50 +0200 Subject: [PATCH 096/118] s3: dynamically import botocore https://github.com/iterative/dvc/issues/2927 --- dvc/remote/s3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 8dd2bdf48..0b8237549 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -5,7 +5,6 @@ import logging import os import threading -from botocore.exceptions import ClientError from funcy import cached_property, wrap_prop from dvc.config import Config @@ -209,6 +208,8 @@ class RemoteS3(RemoteBASE): return self._list_paths(self.path_info) def isfile(self, path_info): + from botocore.exceptions import ClientError + if path_info.path.endswith("/"): return False From a1ae3578c081db079bf0f715e5b70c10180e8b36 Mon Sep 17 00:00:00 2001 From: Mayur Kularni Date: Tue, 10 Dec 2019 11:49:19 +0530 Subject: [PATCH 097/118] Fix caplog test case and move call to CmdRemoteAdd --- dvc/command/remote.py | 4 ++++ dvc/external_repo.py | 18 ++++++++++-------- dvc/remote/config.py | 9 ++++----- tests/func/test_external_repo.py | 12 ------------ tests/func/test_get.py | 1 + 5 files changed, 19 insertions(+), 25 deletions(-) diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 8c6dbc3ef..683c040f4 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -19,6 +19,10 @@ class CmdRemoteConfig(CmdConfig): class CmdRemoteAdd(CmdRemoteConfig): def run(self): + if self.args.default: + logger.info( + "Setting '{}' as a default remote.".format(self.args.name) + ) self.remote_config.add( self.args.name, self.args.url, diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 4e9c50e98..9aca3c2f9 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -73,15 +73,17 @@ def _external_repo(url=None, rev=None, cache_dir=None): # add default remote pointing to the original repo's cache location if os.path.isdir(url): rconfig = RemoteConfig(repo.config) - if not rconfig.remote_config_exists(): + if not rconfig.remote_remote_set(): original_repo = Repo(url) - rconfig.add( - "upstream", - original_repo.cache.local.cache_dir, - default=True, - level=Config.LEVEL_LOCAL, - ) - original_repo.close() + try: + rconfig.add( + "aut-generated-upstream", + original_repo.cache.local.cache_dir, + default=True, + level=Config.LEVEL_LOCAL, + ) + finally: + original_repo.close() if cache_dir is not None: cache_config = CacheConfig(repo.config) diff --git a/dvc/remote/config.py b/dvc/remote/config.py index e1dc2dfb9..b4d12baaa 100644 --- a/dvc/remote/config.py +++ b/dvc/remote/config.py @@ -105,7 +105,6 @@ class RemoteConfig(object): level=level, ) if default: - logger.info("Setting '{}' as a default remote.".format(name)) self.config.set( Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, @@ -157,7 +156,7 @@ class RemoteConfig(object): Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level ) - def remote_config_exists(self): + def remote_remote_set(self): """ Checks if default remote config is present. Args: @@ -167,7 +166,7 @@ class RemoteConfig(object): True if the remote config exists, else False """ try: - default = self.get_default() + self.get_default() + return True except ConfigError: - default = None - return bool(default) + return False diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 4c34505a0..de2b8b20b 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -1,9 +1,7 @@ -import filecmp import os from mock import patch -from dvc.repo import Repo from dvc.external_repo import external_repo from dvc.scm.git import Git from dvc.utils.fs import path_isin @@ -28,13 +26,3 @@ def test_external_repo(erepo): assert path_isin(repo.cache.local.cache_dir, repo.root_dir) assert mock.call_count == 1 - - -def test_external_repo_import_without_remote(erepo, dvc_repo): - src = erepo.FOO - dst = os.path.join(dvc_repo.root_dir, "foo") - - Repo.get(erepo.root_dir, src, dst) - assert os.path.exists(dst) - assert os.path.isfile(dst) - assert filecmp.cmp(erepo.FOO, dst, shallow=False) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 8b99547f0..07551c5f3 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -101,6 +101,7 @@ def test_get_from_non_dvc_master(erepo, tmp_path, monkeypatch, caplog): with caplog.at_level(logging.INFO, logger="dvc"): Repo.get(erepo._root_dir, erepo.FOO, out=imported_file, rev="branch") + assert caplog.text == "" assert filecmp.cmp( os.path.join(erepo._root_dir, erepo.FOO), imported_file, shallow=False ) From 40c392d3123c5a00d67f406de8de41df7d7cf418 Mon Sep 17 00:00:00 2001 From: Mayur Kularni Date: Tue, 10 Dec 2019 15:31:58 +0530 Subject: [PATCH 098/118] type and doc fix --- dvc/external_repo.py | 2 +- dvc/remote/config.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 9aca3c2f9..fe5792f13 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -73,7 +73,7 @@ def _external_repo(url=None, rev=None, cache_dir=None): # add default remote pointing to the original repo's cache location if os.path.isdir(url): rconfig = RemoteConfig(repo.config) - if not rconfig.remote_remote_set(): + if not rconfig.default_remote_set(): original_repo = Repo(url) try: rconfig.add( diff --git a/dvc/remote/config.py b/dvc/remote/config.py index b4d12baaa..53735f5c1 100644 --- a/dvc/remote/config.py +++ b/dvc/remote/config.py @@ -156,14 +156,14 @@ class RemoteConfig(object): Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level ) - def remote_remote_set(self): + def default_remote_set(self): """ Checks if default remote config is present. Args: rconfig: a remote config Returns: - True if the remote config exists, else False + True if the remote config is set, else False """ try: self.get_default() From e6285f70c57017d05ca160015093a72e48756d9f Mon Sep 17 00:00:00 2001 From: Mayur Kularni Date: Tue, 10 Dec 2019 15:38:54 +0530 Subject: [PATCH 099/118] typo in doc fix --- dvc/remote/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/remote/config.py b/dvc/remote/config.py index 53735f5c1..2e9e20446 100644 --- a/dvc/remote/config.py +++ b/dvc/remote/config.py @@ -163,7 +163,7 @@ class RemoteConfig(object): rconfig: a remote config Returns: - True if the remote config is set, else False + True if the default remote config is set, else False """ try: self.get_default() From 9eb1ac2d959929814cc8e245da779f6b6fd311f2 Mon Sep 17 00:00:00 2001 From: Veselin Borisov Date: Wed, 27 Nov 2019 19:09:43 +0200 Subject: [PATCH 100/118] remote: Extend S3 remote support for custom ACL Besides AWS boto3 extra_arg `ACL` (`x-amz-acl` header), there four more that could be used to grant specific permissions to specific grantees on the uploaded object. These headers are: - `x-amz-grant-full-control` with boto3's extra_arg `GrantFullControl` - `x-amz-grant-read` with boto3's extra_arg `GrantRead` - `x-amz-grant-read-acp` with boto3's extra_arg `GrantReadACP` - `x-amz-grant-write-acp` with boto3's extra_arg `GrantWriteACP` As `ACL` (`x-amz-acl` header) is limited to be used with predefined canned ACLs, adding support for the aformentioned headers will help in applying more customized persmissions solutions according to different project needs. References: - https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#permissions - https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS - https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html Fixes #(2930). --- dvc/config.py | 8 +++++++ dvc/remote/s3.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/dvc/config.py b/dvc/config.py index 25b091c67..722d549d8 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -171,6 +171,10 @@ class Config(object): # pylint: disable=too-many-instance-attributes SECTION_AWS_USE_SSL = "use_ssl" SECTION_AWS_SSE = "sse" SECTION_AWS_ACL = "acl" + SECTION_AWS_GRANT_READ = "grant_read" + SECTION_AWS_GRANT_READ_ACP = "grant_read_acp" + SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp" + SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control" # gcp specific options SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH @@ -210,6 +214,10 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional(SECTION_AWS_USE_SSL, default=True): Bool, SECTION_AWS_SSE: str, SECTION_AWS_ACL: str, + SECTION_AWS_GRANT_READ: str, + SECTION_AWS_GRANT_READ_ACP: str, + SECTION_AWS_GRANT_WRITE_ACP: str, + SECTION_AWS_GRANT_FULL_CONTROL: str, SECTION_GCP_PROJECTNAME: str, SECTION_CACHE_TYPE: supported_cache_type, Optional(SECTION_CACHE_PROTECTED, default=False): Bool, diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 0b8237549..071b621e4 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -53,6 +53,8 @@ class RemoteS3(RemoteBASE): if self.acl: self.extra_args["ACL"] = self.acl + self._append_aws_grants_to_extra_args(config) + shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH) if shared_creds: os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds) @@ -302,3 +304,56 @@ class RemoteS3(RemoteBASE): continue yield path_info.replace(path=fname) + + def _append_aws_grants_to_extra_args(self, config): + # Keys for extra_args can be one of the following list: + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS + """ + ALLOWED_UPLOAD_ARGS = [ + 'ACL', 'CacheControl', 'ContentDisposition', 'ContentEncoding', + 'ContentLanguage', 'ContentType', 'Expires', 'GrantFullControl', + 'GrantRead', 'GrantReadACP', 'GrantWriteACP', 'Metadata', + 'RequestPayer', 'ServerSideEncryption', 'StorageClass', + 'SSECustomerAlgorithm', 'SSECustomerKey', 'SSECustomerKeyMD5', + 'SSEKMSKeyId', 'WebsiteRedirectLocation' + ] + """ + overwriting_acl_grantees = False + + self.grant_full_control = config.get( + Config.SECTION_AWS_GRANT_FULL_CONTROL, "" + ) + if self.grant_full_control: + self.extra_args["GrantFullControl"] = self.grant_full_control + overwriting_acl_grantees = True + + self.grant_read = config.get(Config.SECTION_AWS_GRANT_READ, "") + if self.grant_read: + self.extra_args["GrantRead"] = self.grant_read + overwriting_acl_grantees = True + + self.grant_read_acp = config.get(Config.SECTION_AWS_GRANT_READ_ACP, "") + if self.grant_read_acp: + self.extra_args["GrantReadACP"] = self.grant_read_acp + overwriting_acl_grantees = True + + self.grant_write_acp = config.get( + Config.SECTION_AWS_GRANT_WRITE_ACP, "" + ) + if self.grant_write_acp: + self.extra_args["GrantWriteACP"] = self.grant_write_acp + overwriting_acl_grantees = True + + if overwriting_acl_grantees: + overwriting_fields = [ + Config.SECTION_AWS_GRANT_FULL_CONTROL, + Config.SECTION_AWS_GRANT_READ, + Config.SECTION_AWS_GRANT_READ_ACP, + Config.SECTION_AWS_GRANT_WRITE_ACP, + ] + warning_message = [ + f"Using one of the [{', '.join(overwriting_fields)}] will", + "overwrite default ACL Grantees list. Default ACL Grantees", + "list include Owner with FULL_CONTROL permissions.", + ] + logger.warning(" ".join(warning_message)) From a160a8ea7a3cc15b2a84de73755cbf70bc27f2c0 Mon Sep 17 00:00:00 2001 From: Veselin Borisov Date: Tue, 10 Dec 2019 13:44:13 +0200 Subject: [PATCH 101/118] fixup! remote: Extend S3 remote support for custom ACL --- dvc/remote/s3.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 071b621e4..2594a8a63 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -352,7 +352,9 @@ class RemoteS3(RemoteBASE): Config.SECTION_AWS_GRANT_WRITE_ACP, ] warning_message = [ - f"Using one of the [{', '.join(overwriting_fields)}] will", + "Using one of the [{0}] will".format( + ", ".join(overwriting_fields) + ), "overwrite default ACL Grantees list. Default ACL Grantees", "list include Owner with FULL_CONTROL permissions.", ] From 8a7cf747e9e8f26c427f92604b913dad5a211067 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 10 Dec 2019 23:04:39 +0200 Subject: [PATCH 102/118] dvc: bump to 0.75.1 --- dvc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/version.py b/dvc/version.py index 8a3a2f0ce..ea6c7b8a0 100644 --- a/dvc/version.py +++ b/dvc/version.py @@ -7,7 +7,7 @@ import os import subprocess -_BASE_VERSION = "0.75.0" +_BASE_VERSION = "0.75.1" def _generate_version(base_version): From 6296e12f4877b6f81e4338de8303489cb98f2e5b Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 11 Dec 2019 06:24:49 +0200 Subject: [PATCH 103/118] setup: update pyarrow version https://github.com/apache/arrow/pull/4907 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0ee7cda27..d53fac5f6 100644 --- a/setup.py +++ b/setup.py @@ -104,7 +104,7 @@ ssh = ["paramiko>=2.5.0"] # requirements, including kerberos itself. Once all the wheels are available, # we can start shipping it by default. ssh_gssapi = ["paramiko[gssapi]>=2.5.0"] -hdfs = ["pyarrow==0.14.0"] +hdfs = ["pyarrow==0.15.1"] all_remotes = gs + s3 + azure + ssh + oss + gdrive if os.name != "nt" or sys.version_info[0] != 2: From e1e0cbd0383c0a8227f82447b29cde54c81fa2e9 Mon Sep 17 00:00:00 2001 From: Mayur Kularni Date: Wed, 11 Dec 2019 10:54:59 +0530 Subject: [PATCH 104/118] moved default check out of `RemoteConfig` --- dvc/external_repo.py | 23 ++++++++++++++++++++--- dvc/remote/config.py | 15 --------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index fe5792f13..758d8f421 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -4,11 +4,12 @@ import os import tempfile from contextlib import contextmanager from distutils.dir_util import copy_tree + from funcy import retry +from dvc.config import NoRemoteError, ConfigError from dvc.exceptions import NoRemoteInExternalRepoError from dvc.remote import RemoteConfig -from dvc.config import NoRemoteError from dvc.exceptions import NoOutputInExternalRepoError from dvc.exceptions import OutputNotFoundError from dvc.utils.fs import remove @@ -73,11 +74,11 @@ def _external_repo(url=None, rev=None, cache_dir=None): # add default remote pointing to the original repo's cache location if os.path.isdir(url): rconfig = RemoteConfig(repo.config) - if not rconfig.default_remote_set(): + if not _default_remote_set(rconfig): original_repo = Repo(url) try: rconfig.add( - "aut-generated-upstream", + "auto-generated-upstream", original_repo.cache.local.cache_dir, default=True, level=Config.LEVEL_LOCAL, @@ -129,3 +130,19 @@ def _clone_repo(url, path): git = Git.clone(url, path) git.close() + + +def _default_remote_set(rconfig): + """ + Checks if default remote config is present. + Args: + rconfig: a remote config + + Returns: + True if the default remote config is set, else False + """ + try: + rconfig.get_default() + return True + except ConfigError: + return False diff --git a/dvc/remote/config.py b/dvc/remote/config.py index 2e9e20446..bbda7fc44 100644 --- a/dvc/remote/config.py +++ b/dvc/remote/config.py @@ -155,18 +155,3 @@ class RemoteConfig(object): return self.config.get( Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level ) - - def default_remote_set(self): - """ - Checks if default remote config is present. - Args: - rconfig: a remote config - - Returns: - True if the default remote config is set, else False - """ - try: - self.get_default() - return True - except ConfigError: - return False From 26ef77cf96e04f6622f1c7e7e00a40af9b61e3c1 Mon Sep 17 00:00:00 2001 From: pawel Date: Tue, 10 Dec 2019 19:56:18 +0100 Subject: [PATCH 105/118] checkout: test: refactor to dir helpers --- tests/func/test_checkout.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index f2249bee1..24e114e2e 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -475,11 +475,14 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc): ) -def test_checkout_no_checksum(repo_dir, dvc_repo): - stage = dvc_repo.run(outs=[repo_dir.FOO], no_exec=True, cmd="somecmd") +def test_checkout_no_checksum(tmp_dir, dvc): + tmp_dir.gen("file", "file content") + stage = dvc.run(outs=["file"], no_exec=True, cmd="somecmd") + with pytest.raises(CheckoutError): - dvc_repo.checkout([stage.path], force=True) - assert not os.path.exists(repo_dir.FOO) + dvc.checkout([stage.path], force=True) + + assert not os.path.exists("file") @pytest.mark.skip From a1643d576d3a2b36fa6778fbd1b21529df2610d6 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Wed, 11 Dec 2019 20:19:02 +0700 Subject: [PATCH 106/118] test: fix run_copy fixture when used with scm --- tests/dir_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 158f545fb..3ccf021c9 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -229,7 +229,7 @@ def run_copy(tmp_dir, dvc, request): # Do we need this? if "scm" in request.fixturenames: request.getfixturevalue("scm") - tmp_dir.git_add("copy.py", commit="add copy.py") + tmp_dir.scm_add("copy.py", commit="add copy.py") def run_copy(src, dst, **run_kwargs): return dvc.run( From b8cf906edb1ece967c2a43076a3fec7ff19a4c7a Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Wed, 11 Dec 2019 20:19:38 +0700 Subject: [PATCH 107/118] test: stop erepo_dir from undoing parent chdir --- tests/dir_helpers.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 3ccf021c9..4487e79b4 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -250,27 +250,27 @@ def erepo_dir(tmp_path_factory, monkeypatch): path = TmpDir(fspath_py35(tmp_path_factory.mktemp("erepo"))) # Chdir for git and dvc to work locally - monkeypatch.chdir(fspath_py35(path)) + with monkeypatch.context() as m: + m.chdir(fspath_py35(path)) - _git_init() - path.dvc = Repo.init() - path.scm = path.dvc.scm - path.dvc_gen(REPO_TEMPLATE, commit="init repo") + _git_init() + path.dvc = Repo.init() + path.scm = path.dvc.scm + path.dvc_gen(REPO_TEMPLATE, commit="init repo") - rconfig = RemoteConfig(path.dvc.config) - rconfig.add("upstream", path.dvc.cache.local.cache_dir, default=True) - path.scm_add([path.dvc.config.config_file], commit="add remote") + rconfig = RemoteConfig(path.dvc.config) + rconfig.add("upstream", path.dvc.cache.local.cache_dir, default=True) + path.scm_add([path.dvc.config.config_file], commit="add remote") - path.dvc_gen("version", "master") - path.scm_add([".gitignore", "version.dvc"], commit="master") + path.dvc_gen("version", "master") + path.scm_add([".gitignore", "version.dvc"], commit="master") - path.scm.checkout("branch", create_new=True) - (path / "version").unlink() # For mac ??? - path.dvc_gen("version", "branch") - path.scm_add([".gitignore", "version.dvc"], commit="branch") + path.scm.checkout("branch", create_new=True) + (path / "version").unlink() # For mac ??? + path.dvc_gen("version", "branch") + path.scm_add([".gitignore", "version.dvc"], commit="branch") - path.scm.checkout("master") - path.dvc.close() - monkeypatch.undo() # Undo chdir + path.scm.checkout("master") + path.dvc.close() return path From 5f826dd2e66d50aa82299f49e4b2cb9a72ecb370 Mon Sep 17 00:00:00 2001 From: pawel Date: Tue, 10 Dec 2019 14:41:10 +0100 Subject: [PATCH 108/118] commit: test: refactor to dir helpers --- tests/func/test_commit.py | 48 ++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/tests/func/test_commit.py b/tests/func/test_commit.py index c3238e6ad..8d6a8e57d 100644 --- a/tests/func/test_commit.py +++ b/tests/func/test_commit.py @@ -7,40 +7,37 @@ from dvc.utils.stage import dump_stage_file from dvc.utils.stage import load_stage_file -def test_commit_recursive(dvc_repo, repo_dir): - stages = dvc_repo.add(repo_dir.DATA_DIR, recursive=True, no_commit=True) +def test_commit_recursive(tmp_dir, dvc): + tmp_dir.gen({"dir": {"file": "text1", "subdir": {"file2": "text2"}}}) + stages = dvc.add("dir", recursive=True, no_commit=True) + assert len(stages) == 2 + assert dvc.status() != {} - assert dvc_repo.status() != {} - - dvc_repo.commit(repo_dir.DATA_DIR, recursive=True) - - assert dvc_repo.status() == {} + dvc.commit("dir", recursive=True) + assert dvc.status() == {} -def test_commit_force(dvc_repo, repo_dir): - stages = dvc_repo.add(repo_dir.FOO, no_commit=True) - assert len(stages) == 1 - stage = stages[0] +def test_commit_force(tmp_dir, dvc): + tmp_dir.gen({"dir": {"file": "text1", "file2": "text2"}}) + stage, = dvc.add("dir", no_commit=True) - with dvc_repo.state: + with dvc.state: assert stage.outs[0].changed_cache() - with open(repo_dir.FOO, "a") as fobj: - fobj.write(repo_dir.FOO_CONTENTS) + tmp_dir.gen("dir/file", "file content modified") - with dvc_repo.state: + with dvc.state: assert stage.outs[0].changed_cache() with pytest.raises(StageCommitError): - dvc_repo.commit(stage.path) + dvc.commit(stage.path) - with dvc_repo.state: + with dvc.state: assert stage.outs[0].changed_cache() - dvc_repo.commit(stage.path, force=True) - - assert dvc_repo.status([stage.path]) == {} + dvc.commit(stage.path, force=True) + assert dvc.status([stage.path]) == {} def test_commit_with_deps(tmp_dir, dvc, run_copy): @@ -63,16 +60,15 @@ def test_commit_with_deps(tmp_dir, dvc, run_copy): assert not stage.outs[0].changed_cache() -def test_commit_changed_md5(dvc_repo, repo_dir): - stages = dvc_repo.add(repo_dir.FOO, no_commit=True) - assert len(stages) == 1 - stage = stages[0] +def test_commit_changed_md5(tmp_dir, dvc): + tmp_dir.gen({"file": "file content"}) + stage, = dvc.add("file", no_commit=True) stage_file_content = load_stage_file(stage.path) stage_file_content["md5"] = "1111111111" dump_stage_file(stage.path, stage_file_content) with pytest.raises(StageCommitError): - dvc_repo.commit(stage.path) + dvc.commit(stage.path) - dvc_repo.commit(stage.path, force=True) + dvc.commit(stage.path, force=True) From ed95000bb4bb4dc3b829236cf3fd38b9d7663cf5 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Wed, 11 Dec 2019 17:37:37 +0000 Subject: [PATCH 109/118] remove plugs (incompatible with classic) Details: --- snap/snapcraft.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 1f8a0b39f..b1e9ed058 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -6,13 +6,6 @@ grade: stable confinement: classic # for same reasons as snap `git-ubuntu` base: core18 license: Apache-2.0 -plugs: - config-dvc: - interface: system-files - read: - - /etc/host.conf - - /etc/resolv.conf - - /etc/hosts layout: /etc/dvc: bind: $SNAP_DATA/etc/dvc @@ -48,18 +41,7 @@ apps: environment: GIT_PYTHON_GIT_EXECUTABLE: $SNAP/usr/bin/git XDG_CONFIG_DIRS: /etc - plugs: - - config-dvc - - home - - mount-observe - - network-control - - removable-media git: command: usr/bin/git environment: GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 - plugs: - - home - - mount-observe - - network-control - - removable-media From 4c2bc6bf4853ce86b33c49a103af10353727ce0b Mon Sep 17 00:00:00 2001 From: Veselin Borisov Date: Thu, 12 Dec 2019 12:41:38 +0200 Subject: [PATCH 110/118] fixup! remote: Extend S3 remote support for custom ACL --- tests/unit/remote/test_s3.py | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tests/unit/remote/test_s3.py diff --git a/tests/unit/remote/test_s3.py b/tests/unit/remote/test_s3.py new file mode 100644 index 000000000..53e96b1b6 --- /dev/null +++ b/tests/unit/remote/test_s3.py @@ -0,0 +1,40 @@ +from unittest import TestCase + +from dvc.remote.s3 import RemoteS3 + + +class TestRemoteS3(TestCase): + bucket_name = "bucket-name" + prefix = "some/prefix" + url = "s3://{}/{}".format(bucket_name, prefix) + + def test_init(self): + config = {"url": self.url} + remote = RemoteS3(None, config) + + self.assertEqual(remote.path_info, self.url) + + def test_grants(self): + config = { + "url": self.url, + "grant_read": "id=read-permission-id,id=other-read-permission-id", + "grant_read_acp": "id=read-acp-permission-id", + "grant_write_acp": "id=write-acp-permission-id", + "grant_full_control": "id=full-control-permission-id", + } + remote = RemoteS3(None, config) + + self.assertEqual( + remote.extra_args["GrantRead"], + "id=read-permission-id,id=other-read-permission-id", + ) + self.assertEqual( + remote.extra_args["GrantReadACP"], "id=read-acp-permission-id" + ) + self.assertEqual( + remote.extra_args["GrantWriteACP"], "id=write-acp-permission-id" + ) + self.assertEqual( + remote.extra_args["GrantFullControl"], + "id=full-control-permission-id", + ) From 25807f2690f52f9642310a51e3cc605617180d1f Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 8 Dec 2019 03:19:24 +0200 Subject: [PATCH 111/118] dvc: make flufl.lock opt-in and use zc.lockfile As it turned out (see issue numbers down below), we can't really take hardlinks for granted, so `flufl.lock` is not a panacea for all filesystems. Considering that the vast majority of filesystems that our users use support `zc.lockfile`(flock-based) and it has benefits like more reliable mechanism, auto-delete when process exits, more sturdy implementation, etc, it makes more sense to bring it back and use by default again. For filesystems that don't support `flock()`, users will be able to manually enable `flufl.lock` use through the config option. It would be ideal if we could auto-detect that flock is not supported, but in the real world, it turned out to be non-trivial, as it might hang forever in a kernel context, which makes the implementation way too complex for our purposes. So what we're doing instead is showing a message before locking with `zc.lockfile` that, under normal circumstances will disappear once the lock is taken or failed, otherwise it will point users to the related documentation where they can learn about how to opt-in for `flufl.lock`. Fixes #2831 Fixes #2897 Related #2860 --- dvc/command/base.py | 15 +++--- dvc/command/daemon.py | 7 ++- dvc/config.py | 2 + dvc/lock.py | 111 +++++++++++++++++++++++++----------------- dvc/repo/__init__.py | 7 ++- dvc/updater.py | 12 +++-- dvc/utils/__init__.py | 8 +++ setup.py | 8 +-- 8 files changed, 102 insertions(+), 68 deletions(-) diff --git a/dvc/command/base.py b/dvc/command/base.py index 56a9aed23..d8ccb2c8d 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -2,8 +2,6 @@ from __future__ import unicode_literals import logging -import colorama - logger = logging.getLogger(__name__) @@ -24,15 +22,13 @@ def fix_subparsers(subparsers): def append_doc_link(help_message, path): + from dvc.utils import format_link + if not path: return help_message doc_base = "https://man.dvc.org/" - return "{message}\nDocumentation: <{blue}{base}{path}{nc}>".format( - message=help_message, - base=doc_base, - path=path, - blue=colorama.Fore.CYAN, - nc=colorama.Fore.RESET, + return "{message}\nDocumentation: {link}".format( + message=help_message, link=format_link(doc_base + path) ) @@ -44,7 +40,8 @@ class CmdBase(object): self.repo = Repo() self.config = self.repo.config self.args = args - updater = Updater(self.repo.dvc_dir) + hardlink_lock = self.config.config["core"].get("hardlink_lock", False) + updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock) updater.check() @property diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index da9210584..26362287d 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -12,11 +12,16 @@ class CmdDaemonUpdater(CmdDaemonBase): def run(self): import os from dvc.repo import Repo + from dvc.config import Config from dvc.updater import Updater root_dir = Repo.find_root() dvc_dir = os.path.join(root_dir, Repo.DVC_DIR) - updater = Updater(dvc_dir) + config = Config(dvc_dir, verify=False) + hardlink_lock = config.config.get("core", {}).get( + "hardlink_lock", False + ) + updater = Updater(dvc_dir, hardlink_lock=hardlink_lock) updater.fetch(detach=False) return 0 diff --git a/dvc/config.py b/dvc/config.py index 25b091c67..c6ebc6399 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -125,6 +125,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes SECTION_CORE_INTERACTIVE = "interactive" SECTION_CORE_ANALYTICS = "analytics" SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs" + SECTION_CORE_HARDLINK_LOCK = "hardlink_lock" SECTION_CACHE = "cache" SECTION_CACHE_DIR = "dir" @@ -160,6 +161,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional(SECTION_CORE_INTERACTIVE, default=False): Bool, Optional(SECTION_CORE_ANALYTICS, default=True): Bool, SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)), + Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool, } # aws specific options diff --git a/dvc/lock.py b/dvc/lock.py index 147118c0f..b7c7560f7 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -6,12 +6,13 @@ import os import time from datetime import timedelta +import zc.lockfile from funcy.py3 import lkeep from dvc.exceptions import DvcException -from dvc.utils import makedirs -from dvc.utils.compat import is_py3 - +from dvc.utils import makedirs, format_link +from dvc.utils.compat import is_py3, is_py2 +from dvc.progress import Tqdm DEFAULT_TIMEOUT = 5 @@ -26,10 +27,60 @@ class LockError(DvcException): """Thrown when unable to acquire the lock for dvc repo.""" +class Lock(object): + """Class for dvc repo lock. + + Uses zc.lockfile as backend. + """ + + def __init__(self, lockfile, friendly=False, **kwargs): + self._friendly = friendly + self.lockfile = lockfile + self._lock = None + + @property + def files(self): + return [self.lockfile] + + def _do_lock(self): + try: + with Tqdm( + bar_format="{desc}", + disable=not self._friendly, + desc=( + "If DVC froze, see `hardlink_lock` in {}".format( + format_link("man.dvc.org/config#core") + ) + ), + ): + self._lock = zc.lockfile.LockFile(self.lockfile) + except zc.lockfile.LockError: + raise LockError(FAILED_TO_LOCK_MESSAGE) + + def lock(self): + try: + self._do_lock() + return + except LockError: + time.sleep(DEFAULT_TIMEOUT) + + self._do_lock() + + def unlock(self): + self._lock.close() + self._lock = None + + def __enter__(self): + self.lock() + + def __exit__(self, typ, value, tbck): + self.unlock() + + if is_py3: import flufl.lock - class Lock(flufl.lock.Lock): + class HardlinkLock(flufl.lock.Lock): """Class for dvc repo lock. Args: @@ -38,7 +89,7 @@ if is_py3: tmp_dir (str): a directory to store claim files. """ - def __init__(self, lockfile, tmp_dir=None): + def __init__(self, lockfile, tmp_dir=None, **kwargs): import socket self._tmp_dir = tmp_dir @@ -101,44 +152,14 @@ if is_py3: pass -else: - import zc.lockfile +def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False): + if hardlink_lock and is_py2: + raise DvcException( + "Hardlink locks are not supported on Python <3.5. " + "See `hardlink_lock` in {}".format( + format_link("man.dvc.org/config#core") + ) + ) - class Lock(object): - """Class for dvc repo lock. - - Uses zc.lockfile as backend. - """ - - def __init__(self, lockfile, tmp_dir=None): - self.lockfile = lockfile - self._lock = None - - @property - def files(self): - return [self.lockfile] - - def _do_lock(self): - try: - self._lock = zc.lockfile.LockFile(self.lockfile) - except zc.lockfile.LockError: - raise LockError(FAILED_TO_LOCK_MESSAGE) - - def lock(self): - try: - self._do_lock() - return - except LockError: - time.sleep(DEFAULT_TIMEOUT) - - self._do_lock() - - def unlock(self): - self._lock.close() - self._lock = None - - def __enter__(self): - self.lock() - - def __exit__(self, typ, value, tbck): - self.unlock() + cls = HardlinkLock if hardlink_lock else Lock + return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index ac013313b..86f03ac5e 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -69,7 +69,7 @@ class Repo(object): def __init__(self, root_dir=None): from dvc.state import State - from dvc.lock import Lock + from dvc.lock import make_lock from dvc.scm import SCM from dvc.cache import Cache from dvc.data_cloud import DataCloud @@ -88,9 +88,12 @@ class Repo(object): self.tree = WorkingTree(self.root_dir) - self.lock = Lock( + hardlink_lock = self.config.config["core"].get("hardlink_lock", False) + self.lock = make_lock( os.path.join(self.dvc_dir, "lock"), tmp_dir=os.path.join(self.dvc_dir, "tmp"), + hardlink_lock=hardlink_lock, + friendly=True, ) # NOTE: storing state and link_state in the repository itself to avoid # any possible state corruption in 'shared cache dir' scenario. diff --git a/dvc/updater.py b/dvc/updater.py index 77145ef9d..661221926 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -9,8 +9,7 @@ import colorama from packaging import version from dvc import __version__ -from dvc.lock import Lock -from dvc.lock import LockError +from dvc.lock import make_lock, LockError from dvc.utils import boxify from dvc.utils import env2bool from dvc.utils.pkg import PKG @@ -24,11 +23,14 @@ class Updater(object): # pragma: no cover TIMEOUT = 24 * 60 * 60 # every day TIMEOUT_GET = 10 - def __init__(self, dvc_dir): + def __init__(self, dvc_dir, friendly=False, hardlink_lock=False): self.dvc_dir = dvc_dir self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE) - self.lock = Lock( - self.updater_file + ".lock", tmp_dir=os.path.join(dvc_dir, "tmp") + self.lock = make_lock( + self.updater_file + ".lock", + tmp_dir=os.path.join(dvc_dir, "tmp"), + friendly=friendly, + hardlink_lock=hardlink_lock, ) self.current = version.parse(__version__).base_version diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 68ba41877..a1d15aa0b 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -420,3 +420,11 @@ def resolve_output(inp, out): if os.path.isdir(out): return os.path.join(out, name) return out + + +def format_link(link): + import colorama + + return "<{blue}{link}{nc}>".format( + blue=colorama.Fore.CYAN, link=link, nc=colorama.Fore.RESET + ) diff --git a/setup.py b/setup.py index d53fac5f6..29a4a0936 100644 --- a/setup.py +++ b/setup.py @@ -79,6 +79,7 @@ install_requires = [ "shortuuid>=0.5.0", "tqdm>=4.40.0,<5", "packaging>=19.0", + "zc.lockfile>=1.2.1", "win-unicode-console>=0.5; sys_platform == 'win32'", "pywin32>=225; sys_platform == 'win32'", ] @@ -163,12 +164,7 @@ setup( "ssh_gssapi": ssh_gssapi, "hdfs": hdfs, # NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1 - ":python_version=='2.7'": [ - "futures", - "pathlib2", - "contextlib2", - "zc.lockfile>=1.2.1", - ], + ":python_version=='2.7'": ["futures", "pathlib2", "contextlib2"], ":python_version>='3.0'": ["flufl.lock>=3.2"], "tests": tests_requirements, }, From 30f45512836eb7244af9949986069f47baff0604 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 13 Dec 2019 03:20:43 +0200 Subject: [PATCH 112/118] dvc: bump to 0.76.0 --- dvc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/version.py b/dvc/version.py index ea6c7b8a0..888ad45e6 100644 --- a/dvc/version.py +++ b/dvc/version.py @@ -7,7 +7,7 @@ import os import subprocess -_BASE_VERSION = "0.75.1" +_BASE_VERSION = "0.76.0" def _generate_version(base_version): From 173388b2f84227e019d1e189e85d62f6976f371e Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 12 Dec 2019 05:01:05 +0200 Subject: [PATCH 113/118] run: provide a hint if we get output duplication error Related to #2559 --- dvc/exceptions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 92c84ed0e..1beb8d6e1 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -47,7 +47,8 @@ class OutputDuplicationError(DvcException): ) msg = ( "file/directory '{}' is specified as an output in more than one " - "stage: {}" + "stage: {}\n" + "This is not allowed. Consider using a different output name." ).format(output, "\n ".join(stages)) super(OutputDuplicationError, self).__init__(msg) From e384685715e701fb43616b31158cfeee4fbaafd5 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 28 Nov 2019 18:03:19 +0200 Subject: [PATCH 114/118] dvc: release locks when running a command Currently only one instance of dvc could run in a particular repo because of the `.dvc/lock`. That gets furstrating when you are running some long-running command and blocked from doing anything else. This PR solved that issue for `dvc run` by releasing the repo lock while running the command. To protect against conflicts, we now use rwlock, which keeps the list of readers and writers for particular paths. The process looks like this: 1) aquire read locks for deps; 2) aquire write locks for outs; 3) release repo lock; 4) run cmd; 5) aquire repo lock; 6) release write locks for outs; 7) release read locks for outs; We also take rwlocks for operations such as `import`, `add` and `checkout` to ensure that we won't hit the same files that are being used by another `dvc run` in the background. We don't aquire rwlocks for the stage file itself, as we don't really care about anyone overwriting a stage file, as there is a very little difference here between parallel runs and sequential runs that overwrite the same stage file. Next steps might be to release repo lock on `import`(when dowloading stuff) and when computing checksums or copying files. Fixes #755 --- dvc/lock.py | 9 +- dvc/path_info.py | 13 ++- dvc/repo/__init__.py | 14 +-- dvc/rwlock.py | 191 ++++++++++++++++++++++++++++++++++++++ dvc/stage.py | 59 +++++++++++- dvc/utils/compat.py | 2 + tests/func/test_run.py | 2 +- tests/unit/test_rwlock.py | 98 +++++++++++++++++++ tests/unit/test_stage.py | 10 +- 9 files changed, 373 insertions(+), 25 deletions(-) create mode 100644 dvc/rwlock.py create mode 100644 tests/unit/test_rwlock.py diff --git a/dvc/lock.py b/dvc/lock.py index b7c7560f7..a8fc08c01 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -7,10 +7,9 @@ import time from datetime import timedelta import zc.lockfile -from funcy.py3 import lkeep from dvc.exceptions import DvcException -from dvc.utils import makedirs, format_link +from dvc.utils import format_link from dvc.utils.compat import is_py3, is_py2 from dvc.progress import Tqdm @@ -93,8 +92,6 @@ if is_py3: import socket self._tmp_dir = tmp_dir - if self._tmp_dir is not None: - makedirs(self._tmp_dir, exist_ok=True) # NOTE: this is basically Lock.__init__ copy-paste, except that # instead of using `socket.getfqdn()` we use `socket.gethostname()` @@ -120,10 +117,6 @@ if is_py3: def lockfile(self): return self._lockfile - @property - def files(self): - return lkeep([self._lockfile, self._tmp_dir]) - def lock(self): try: super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT)) diff --git a/dvc/path_info.py b/dvc/path_info.py index 860241778..773ae7dcf 100644 --- a/dvc/path_info.py +++ b/dvc/path_info.py @@ -23,7 +23,16 @@ if is_py2: fs_encoding = "utf-8" -class PathInfo(pathlib.PurePath): +class _BasePath(object): + def overlaps(self, other): + if isinstance(other, basestring): + other = self.__class__(other) + elif self.__class__ != other.__class__: + return False + return self == other or self.isin(other) or other.isin(self) + + +class PathInfo(pathlib.PurePath, _BasePath): # Use __slots__ in PathInfo objects following PurePath implementation. # This makes objects smaller and speeds up attribute access. # We don't add any fields so it's empty. @@ -138,7 +147,7 @@ class _URLPathParents(object): return "<{}.parents>".format(self.src) -class URLInfo(object): +class URLInfo(_BasePath): DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22, "hdfs": 0} def __init__(self, url): diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 86f03ac5e..8a0afc2d3 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -76,6 +76,7 @@ class Repo(object): from dvc.repo.metrics import Metrics from dvc.scm.tree import WorkingTree from dvc.repo.tag import Tag + from dvc.utils import makedirs root_dir = self.find_root(root_dir) @@ -88,6 +89,9 @@ class Repo(object): self.tree = WorkingTree(self.root_dir) + self.tmp_dir = os.path.join(self.dvc_dir, "tmp") + makedirs(self.tmp_dir, exist_ok=True) + hardlink_lock = self.config.config["core"].get("hardlink_lock", False) self.lock = make_lock( os.path.join(self.dvc_dir, "lock"), @@ -95,6 +99,7 @@ class Repo(object): hardlink_lock=hardlink_lock, friendly=True, ) + # NOTE: storing state and link_state in the repository itself to avoid # any possible state corruption in 'shared cache dir' scenario. self.state = State(self, self.config.config) @@ -165,9 +170,8 @@ class Repo(object): flist = ( [self.config.config_local_file, updater.updater_file] + + [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir] + self.state.files - + self.lock.files - + updater.lock.files ) if path_isin(self.cache.local.cache_dir, self.root_dir): @@ -352,11 +356,7 @@ class Repo(object): continue for out in outs: - if ( - out == dep.path_info - or dep.path_info.isin(out) - or out.isin(dep.path_info) - ): + if out.overlaps(dep.path_info): dep_stage = outs[out].stage dep_node = relpath(dep_stage.path, self.root_dir) G.add_node(dep_node, stage=dep_stage) diff --git a/dvc/rwlock.py b/dvc/rwlock.py new file mode 100644 index 000000000..29b7bce88 --- /dev/null +++ b/dvc/rwlock.py @@ -0,0 +1,191 @@ +from __future__ import unicode_literals + +import os +import json + +from collections import defaultdict +from contextlib import contextmanager + +from voluptuous import Schema, Invalid +from funcy.py3 import lmap, lfilter, lmapcat + +from .exceptions import DvcException +from .lock import LockError +from .utils.compat import ( + convert_to_unicode, + FileNotFoundError, + JSONDecodeError, + str, +) +from .utils.fs import relpath + +INFO_SCHEMA = {"pid": int, "cmd": str} + +SCHEMA = Schema({"write": {str: INFO_SCHEMA}, "read": {str: [INFO_SCHEMA]}}) + + +class RWLockFileCorruptedError(DvcException): + def __init__(self, path, cause): + super(RWLockFileCorruptedError, self).__init__( + "Unable to read RWLock-file '{}'. JSON structure is " + "corrupted".format(relpath(path)), + cause=cause, + ) + + +class RWLockFileFormatError(DvcException): + def __init__(self, path, cause): + super(RWLockFileFormatError, self).__init__( + "RWLock-file '{}' format error.".format(relpath(path)), cause=cause + ) + + +@contextmanager +def _edit_rwlock(lock_dir): + path = os.path.join(lock_dir, "rwlock") + try: + with open(path, "r") as fobj: + lock = json.load(fobj) + lock = SCHEMA(convert_to_unicode(lock)) + except FileNotFoundError: + lock = SCHEMA({}) + except JSONDecodeError as exc: + raise RWLockFileCorruptedError(path, cause=exc) + except Invalid as exc: + raise RWLockFileFormatError(path, cause=exc) + lock = defaultdict(dict, lock) + lock["read"] = defaultdict(list, lock["read"]) + lock["write"] = defaultdict(dict, lock["write"]) + yield lock + with open(path, "w+") as fobj: + json.dump(lock, fobj) + + +def _infos_to_str(infos): + return "\n".join( + " (PID {}): {}".format(info["pid"], info["cmd"]) for info in infos + ) + + +def _check_no_writers(lock, info, path_infos): + for path_info in path_infos: + blocking_urls = lfilter(path_info.overlaps, lock["write"]) + if not blocking_urls: + continue + + writers = lmap(lock["write"].get, blocking_urls) + writers = lfilter(lambda i: info != i, writers) + if not writers: + continue + + raise LockError( + "'{}' is busy, it is being written to by:\n{}".format( + str(path_info), _infos_to_str(writers) + ) + ) + + +def _check_no_readers(lock, info, path_infos): + for path_info in path_infos: + blocking_urls = lfilter(path_info.overlaps, lock["read"]) + if not blocking_urls: + continue + + readers = lmapcat(lock["read"].get, blocking_urls) + readers = lfilter(lambda i: info != i, readers) + if not readers: + continue + + raise LockError( + "'{}' is busy, it is being read by:\n{}".format( + str(path_info), _infos_to_str(readers) + ) + ) + + +def _acquire_read(lock, info, path_infos): + changes = [] + + for path_info in path_infos: + url = path_info.url + + readers = lock["read"][url] + if info in readers: + continue + + changes.append(url) + readers.append(info) + + return changes + + +def _acquire_write(lock, info, path_infos): + changes = [] + + for path_info in path_infos: + url = path_info.url + + if lock["write"][url] == info: + continue + + changes.append(url) + lock["write"][url] = info + + return changes + + +def _release_write(lock, info, changes): + for url in changes: + assert "write" in lock + assert url in lock["write"] + assert lock["write"][url] == info + del lock["write"][url] + if not lock["write"]: + del lock["write"] + + +def _release_read(lock, info, changes): + for url in changes: + assert "read" in lock + assert url in lock["read"] + assert info in lock["read"][url] + lock["read"][url].remove(info) + if not lock["read"][url]: + del lock["read"][url] + if not lock["read"]: + del lock["read"] + + +@contextmanager +def rwlock(tmp_dir, cmd, read, write): + """Create non-thread-safe RWLock for PathInfos. + + Args: + tmp_dir (str): existing directory where to create the rwlock file. + cmd (str): command that will be working on these PathInfos. + read ([PathInfo]): PathInfos that are going to be read. + write ([PathInfo]): PathInfos that are going to be written. + + Raises: + LockError: raised if PathInfo we want to read is being written to by + another command or if PathInfo we want to write is being written + to or read from by another command. + RWLockFileCorruptedError: raised if rwlock file is not a valid JSON. + RWLockFileFormatError: raised if rwlock file is a valid JSON, but + has internal format that doesn't pass our schema validation. + """ + info = {"pid": os.getpid(), "cmd": cmd} + + with _edit_rwlock(tmp_dir) as lock: + _check_no_writers(lock, info, read + write) + _check_no_readers(lock, info, write) + + rchanges = _acquire_read(lock, info, read) + wchanges = _acquire_write(lock, info, write) + + try: + yield + finally: + with _edit_rwlock(tmp_dir) as lock: + _release_write(lock, info, wchanges) + _release_read(lock, info, rchanges) diff --git a/dvc/stage.py b/dvc/stage.py index c3c5daec9..ce0a9668a 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -7,7 +7,10 @@ import re import signal import subprocess import threading + +from functools import wraps from itertools import chain +from funcy import decorator from voluptuous import Any, Schema, MultipleInvalid @@ -129,6 +132,52 @@ class MissingDataSource(DvcException): super(MissingDataSource, self).__init__(msg) +@decorator +def rwlocked(call, read=None, write=None): + import sys + from dvc.rwlock import rwlock + from dvc.dependency.repo import DependencyREPO + + if read is None: + read = [] + + if write is None: + write = [] + + stage = call._args[0] + + def _chain(names): + return [ + item.path_info + for attr in names + for item in getattr(stage, attr) + # There is no need to lock DependencyREPO deps, as there is no + # corresponding OutputREPO, so we can't even write it. + if not isinstance(item, DependencyREPO) + ] + + cmd = " ".join(sys.argv) + + with rwlock(stage.repo.tmp_dir, cmd, _chain(read), _chain(write)): + return call() + + +def unlocked_repo(f): + @wraps(f) + def wrapper(stage, *args, **kwargs): + stage.repo.state.dump() + stage.repo.lock.unlock() + stage.repo._reset() + try: + ret = f(stage, *args, **kwargs) + finally: + stage.repo.lock.lock() + stage.repo.state.load() + return ret + + return wrapper + + class Stage(object): STAGE_FILE = "Dvcfile" STAGE_FILE_SUFFIX = ".dvc" @@ -286,6 +335,7 @@ class Stage(object): return True return False + @rwlocked(read=["deps", "outs"]) def changed(self): # Short-circuit order: stage md5 is fast, deps are expected to change ret = ( @@ -299,6 +349,7 @@ class Stage(object): return ret + @rwlocked(write=["outs"]) def remove_outs(self, ignore_remove=False, force=False): """Used mainly for `dvc remove --outs` and :func:`Stage.reproduce`.""" for out in self.outs: @@ -316,6 +367,7 @@ class Stage(object): for out in self.outs: out.unprotect() + @rwlocked(write=["outs"]) def remove(self, force=False, remove_outs=True): if remove_outs: self.remove_outs(ignore_remove=True, force=force) @@ -323,6 +375,7 @@ class Stage(object): self.unprotect_outs() os.unlink(self.path) + @rwlocked(read=["deps"], write=["outs"]) def reproduce(self, interactive=False, **kwargs): if not kwargs.get("force", False) and not self.changed(): @@ -755,6 +808,7 @@ class Stage(object): ) self.save() + @rwlocked(write=["outs"]) def commit(self): for out in self.outs: out.commit() @@ -801,6 +855,7 @@ class Stage(object): if occurrence > 1: raise ArgumentDuplicationError(str(path)) + @unlocked_repo def _run(self): self._check_missing_deps() @@ -850,6 +905,7 @@ class Stage(object): if (p is None) or (p.returncode != 0): raise StageCmdFailedError(self) + @rwlocked(read=["deps"], write=["outs"]) def run(self, dry=False, no_commit=False, force=False): if (self.cmd or self.is_import) and not self.locked and not dry: self.remove_outs(ignore_remove=False, force=False) @@ -874,7 +930,6 @@ class Stage(object): self.outs[0].checkout() else: self.deps[0].download(self.outs[0]) - elif self.is_data_source: msg = "Verifying data sources in '{}'".format(self.relpath) logger.info(msg) @@ -904,6 +959,7 @@ class Stage(object): if paths: raise MissingDataSource(paths) + @rwlocked(write=["outs"]) def checkout(self, force=False, progress_callback=None): failed_checkouts = [] for out in self.outs: @@ -923,6 +979,7 @@ class Stage(object): return ret + @rwlocked(read=["deps", "outs"]) def status(self): ret = [] diff --git a/dvc/utils/compat.py b/dvc/utils/compat.py index 372c8d2c6..a4819fccc 100644 --- a/dvc/utils/compat.py +++ b/dvc/utils/compat.py @@ -123,6 +123,7 @@ if is_py2: makedirs = _makedirs range = xrange # noqa: F821 FileNotFoundError = IOError + JSONDecodeError = ValueError import StringIO import io @@ -164,6 +165,7 @@ elif is_py3: import configparser as ConfigParser # noqa: F401 from collections.abc import Mapping # noqa: F401 from contextlib import ExitStack # noqa: F401 + from json.decoder import JSONDecodeError # noqa: F401 builtin_str = str # noqa: F821 str = str # noqa: F821 diff --git a/tests/func/test_run.py b/tests/func/test_run.py index 7d9a51f29..61db6c98e 100644 --- a/tests/func/test_run.py +++ b/tests/func/test_run.py @@ -893,7 +893,7 @@ class TestShouldNotCheckoutUponCorruptedLocalHardlinkCache(TestDvc): ) patch_run = mock.patch.object(stage, "_run", wraps=stage._run) - with self.dvc.state: + with self.dvc.lock, self.dvc.state: with patch_checkout as mock_checkout: with patch_run as mock_run: stage.run() diff --git a/tests/unit/test_rwlock.py b/tests/unit/test_rwlock.py new file mode 100644 index 000000000..9e554960e --- /dev/null +++ b/tests/unit/test_rwlock.py @@ -0,0 +1,98 @@ +from __future__ import unicode_literals + +import os + +import pytest + +from dvc.rwlock import ( + rwlock, + _edit_rwlock, + RWLockFileFormatError, + RWLockFileCorruptedError, +) +from dvc.lock import LockError +from dvc.utils.compat import fspath +from dvc.path_info import PathInfo + + +def test_rwlock(tmp_path): + path = fspath(tmp_path) + foo = PathInfo("foo") + + with rwlock(path, "cmd1", [foo], []): + with pytest.raises(LockError): + with rwlock(path, "cmd2", [], [foo]): + pass + + with rwlock(path, "cmd1", [], [foo]): + with pytest.raises(LockError): + with rwlock(path, "cmd2", [foo], []): + pass + + with rwlock(path, "cmd1", [], [foo]): + with pytest.raises(LockError): + with rwlock(path, "cmd2", [], [foo]): + pass + + +def test_rwlock_reentrant(tmp_path): + path = fspath(tmp_path) + foo = PathInfo("foo") + + with rwlock(path, "cmd1", [], [foo]): + with rwlock(path, "cmd1", [], [foo]): + pass + with _edit_rwlock(path) as lock: + assert lock == { + "read": {}, + "write": {"foo": {"cmd": "cmd1", "pid": os.getpid()}}, + } + + with rwlock(path, "cmd", [foo], []): + with rwlock(path, "cmd", [foo], []): + pass + with _edit_rwlock(path) as lock: + assert lock == { + "read": {"foo": [{"cmd": "cmd", "pid": os.getpid()}]}, + "write": {}, + } + + +def test_rwlock_subdirs(tmp_path): + path = fspath(tmp_path) + foo = PathInfo("foo") + subfoo = PathInfo("foo/subfoo") + + with rwlock(path, "cmd1", [foo], []): + with pytest.raises(LockError): + with rwlock(path, "cmd2", [], [subfoo]): + pass + + with rwlock(path, "cmd1", [], [subfoo]): + with pytest.raises(LockError): + with rwlock(path, "cmd2", [foo], []): + pass + + with rwlock(path, "cmd1", [], [subfoo]): + with pytest.raises(LockError): + with rwlock(path, "cmd2", [], [foo]): + pass + + with rwlock(path, "cmd1", [subfoo], []): + with rwlock(path, "cmd2", [foo], []): + pass + + +def test_broken_rwlock(tmp_path): + dir_path = fspath(tmp_path) + path = tmp_path / "rwlock" + + path.write_text('{"broken": "format"}', encoding="utf-8") + with pytest.raises(RWLockFileFormatError): + with _edit_rwlock(dir_path): + pass + + path.write_text("{broken json", encoding="utf-8") + with pytest.raises(RWLockFileCorruptedError): + with _edit_rwlock(dir_path): + pass diff --git a/tests/unit/test_stage.py b/tests/unit/test_stage.py index 652c4b529..354fb6007 100644 --- a/tests/unit/test_stage.py +++ b/tests/unit/test_stage.py @@ -93,16 +93,14 @@ def test_stage_update(mocker): not isinstance(threading.current_thread(), threading._MainThread), reason="Not running in the main thread.", ) -def test_stage_run_ignore_sigint(mocker): - stage = Stage(None, "path") - +def test_stage_run_ignore_sigint(dvc_repo, mocker): proc = mocker.Mock() communicate = mocker.Mock() proc.configure_mock(returncode=0, communicate=communicate) popen = mocker.patch.object(subprocess, "Popen", return_value=proc) signal_mock = mocker.patch("signal.signal") - stage._run() + dvc_repo.run(cmd="path") assert popen.called_once() assert communicate.called_once_with() @@ -110,8 +108,8 @@ def test_stage_run_ignore_sigint(mocker): assert signal.getsignal(signal.SIGINT) == signal.default_int_handler -def test_always_changed(): - stage = Stage(None, "path", always_changed=True) +def test_always_changed(dvc_repo): + stage = Stage(dvc_repo, "path", always_changed=True) stage.save() assert stage.changed() assert stage.status()["path"] == ["always changed"] From cd461aec4fc1618c3c824067f7434258e8c3e923 Mon Sep 17 00:00:00 2001 From: Veselin Borisov Date: Fri, 13 Dec 2019 02:02:45 +0200 Subject: [PATCH 115/118] fixup! remote: Extend S3 remote support for custom ACL --- dvc/remote/s3.py | 54 +++++++----------------- tests/unit/remote/test_s3.py | 80 +++++++++++++++++++++--------------- 2 files changed, 63 insertions(+), 71 deletions(-) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 2594a8a63..6366fc426 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -7,7 +7,7 @@ import threading from funcy import cached_property, wrap_prop -from dvc.config import Config +from dvc.config import Config, ConfigError from dvc.exceptions import DvcException from dvc.exceptions import ETagMismatchError from dvc.path_info import CloudURLInfo @@ -318,44 +318,20 @@ class RemoteS3(RemoteBASE): 'SSEKMSKeyId', 'WebsiteRedirectLocation' ] """ - overwriting_acl_grantees = False - self.grant_full_control = config.get( - Config.SECTION_AWS_GRANT_FULL_CONTROL, "" - ) - if self.grant_full_control: - self.extra_args["GrantFullControl"] = self.grant_full_control - overwriting_acl_grantees = True + grants = { + Config.SECTION_AWS_GRANT_FULL_CONTROL: "GrantFullControl", + Config.SECTION_AWS_GRANT_READ: "GrantRead", + Config.SECTION_AWS_GRANT_READ_ACP: "GrantReadACP", + Config.SECTION_AWS_GRANT_WRITE_ACP: "GrantWriteACP", + } - self.grant_read = config.get(Config.SECTION_AWS_GRANT_READ, "") - if self.grant_read: - self.extra_args["GrantRead"] = self.grant_read - overwriting_acl_grantees = True + for grant_option, extra_args_key in grants.items(): + if config.get(grant_option, ""): + if self.acl: + raise ConfigError( + "`acl` and `grant_*` AWS S3 config options " + "are mutually exclusive" + ) - self.grant_read_acp = config.get(Config.SECTION_AWS_GRANT_READ_ACP, "") - if self.grant_read_acp: - self.extra_args["GrantReadACP"] = self.grant_read_acp - overwriting_acl_grantees = True - - self.grant_write_acp = config.get( - Config.SECTION_AWS_GRANT_WRITE_ACP, "" - ) - if self.grant_write_acp: - self.extra_args["GrantWriteACP"] = self.grant_write_acp - overwriting_acl_grantees = True - - if overwriting_acl_grantees: - overwriting_fields = [ - Config.SECTION_AWS_GRANT_FULL_CONTROL, - Config.SECTION_AWS_GRANT_READ, - Config.SECTION_AWS_GRANT_READ_ACP, - Config.SECTION_AWS_GRANT_WRITE_ACP, - ] - warning_message = [ - "Using one of the [{0}] will".format( - ", ".join(overwriting_fields) - ), - "overwrite default ACL Grantees list. Default ACL Grantees", - "list include Owner with FULL_CONTROL permissions.", - ] - logger.warning(" ".join(warning_message)) + self.extra_args[extra_args_key] = config.get(grant_option) diff --git a/tests/unit/remote/test_s3.py b/tests/unit/remote/test_s3.py index 53e96b1b6..57fcfcec7 100644 --- a/tests/unit/remote/test_s3.py +++ b/tests/unit/remote/test_s3.py @@ -1,40 +1,56 @@ -from unittest import TestCase +import pytest +from dvc.config import ConfigError from dvc.remote.s3 import RemoteS3 -class TestRemoteS3(TestCase): - bucket_name = "bucket-name" - prefix = "some/prefix" - url = "s3://{}/{}".format(bucket_name, prefix) +bucket_name = "bucket-name" +prefix = "some/prefix" +url = "s3://{}/{}".format(bucket_name, prefix) - def test_init(self): - config = {"url": self.url} - remote = RemoteS3(None, config) - self.assertEqual(remote.path_info, self.url) +@pytest.fixture(autouse=True) +def grants(): + return { + "grant_read": "id=read-permission-id,id=other-read-permission-id", + "grant_read_acp": "id=read-acp-permission-id", + "grant_write_acp": "id=write-acp-permission-id", + "grant_full_control": "id=full-control-permission-id", + } - def test_grants(self): - config = { - "url": self.url, - "grant_read": "id=read-permission-id,id=other-read-permission-id", - "grant_read_acp": "id=read-acp-permission-id", - "grant_write_acp": "id=write-acp-permission-id", - "grant_full_control": "id=full-control-permission-id", - } - remote = RemoteS3(None, config) - self.assertEqual( - remote.extra_args["GrantRead"], - "id=read-permission-id,id=other-read-permission-id", - ) - self.assertEqual( - remote.extra_args["GrantReadACP"], "id=read-acp-permission-id" - ) - self.assertEqual( - remote.extra_args["GrantWriteACP"], "id=write-acp-permission-id" - ) - self.assertEqual( - remote.extra_args["GrantFullControl"], - "id=full-control-permission-id", - ) +def test_init(): + config = {"url": url} + remote = RemoteS3(None, config) + + assert remote.path_info == url + + +def test_grants(): + config = { + "url": url, + "grant_read": "id=read-permission-id,id=other-read-permission-id", + "grant_read_acp": "id=read-acp-permission-id", + "grant_write_acp": "id=write-acp-permission-id", + "grant_full_control": "id=full-control-permission-id", + } + remote = RemoteS3(None, config) + + assert ( + remote.extra_args["GrantRead"] + == "id=read-permission-id,id=other-read-permission-id" + ) + assert remote.extra_args["GrantReadACP"] == "id=read-acp-permission-id" + assert remote.extra_args["GrantWriteACP"] == "id=write-acp-permission-id" + assert ( + remote.extra_args["GrantFullControl"] + == "id=full-control-permission-id" + ) + + +def test_grants_mutually_exclusive_acl_error(grants): + for grant_option, grant_value in grants.items(): + config = {"url": url, "acl": "public-read", grant_option: grant_value} + + with pytest.raises(ConfigError): + RemoteS3(None, config) From 3ed07cc4487141403c24a683f70e43226affaf33 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 13 Dec 2019 17:42:08 +0200 Subject: [PATCH 116/118] daemon: fix typo Thanks @pared for catching this one https://github.com/iterative/dvc/pull/2918#discussion_r357574263 . Will add a test on top, need to re-release ASAP --- dvc/command/daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index 26362287d..b391e6cee 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -17,7 +17,7 @@ class CmdDaemonUpdater(CmdDaemonBase): root_dir = Repo.find_root() dvc_dir = os.path.join(root_dir, Repo.DVC_DIR) - config = Config(dvc_dir, verify=False) + config = Config(dvc_dir, validate=False) hardlink_lock = config.config.get("core", {}).get( "hardlink_lock", False ) From d823f3b6c3ed06a337204e1ce63018f628fcca06 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 13 Dec 2019 17:43:26 +0200 Subject: [PATCH 117/118] dvc: bump to 0.77.0 --- dvc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/version.py b/dvc/version.py index 888ad45e6..915eba13b 100644 --- a/dvc/version.py +++ b/dvc/version.py @@ -7,7 +7,7 @@ import os import subprocess -_BASE_VERSION = "0.76.0" +_BASE_VERSION = "0.77.0" def _generate_version(base_version): From 75ecc9960d7a61d1a74c194839bb7092ba692a72 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 13 Dec 2019 20:28:55 +0200 Subject: [PATCH 118/118] dvc: bump to 0.77.1 --- dvc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/version.py b/dvc/version.py index 915eba13b..867a3fdb0 100644 --- a/dvc/version.py +++ b/dvc/version.py @@ -7,7 +7,7 @@ import os import subprocess -_BASE_VERSION = "0.77.0" +_BASE_VERSION = "0.77.1" def _generate_version(base_version):