From 775e1e837469f3ca3f9b6272346972453669e803 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 7 Oct 2020 23:51:14 +0300 Subject: [PATCH] repo: don't create .dvc in read-only repos (#4674) * lock: use ABC * lock: add LockNoop * plots: remove unused code * plots: create templates only on init * dvc: don't create .dvc in read-only repos * repo: explicitly init LockNoop --- dvc/cache/__init__.py | 3 ++ dvc/command/base.py | 7 +-- dvc/lock.py | 72 ++++++++++++++++++++++++---- dvc/repo/__init__.py | 82 ++++++++++++++----------------- dvc/repo/init.py | 2 + dvc/repo/plots/template.py | 85 +++++++++++++++++---------------- tests/func/metrics/test_show.py | 6 +++ tests/func/plots/test_plots.py | 3 ++ 8 files changed, 162 insertions(+), 98 deletions(-) diff --git a/dvc/cache/__init__.py b/dvc/cache/__init__.py index c4a742a5a..888bb2ddd 100644 --- a/dvc/cache/__init__.py +++ b/dvc/cache/__init__.py @@ -62,6 +62,9 @@ class Cache: if local: settings = {"name": local} + elif "dir" not in config: + self.local = None + return else: from ..config import LOCAL_COMMON diff --git a/dvc/command/base.py b/dvc/command/base.py index ae41e9be5..e55471b4d 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -40,9 +40,10 @@ class CmdBase(ABC): self.repo = Repo(uninitialized=self.UNINITIALIZED) self.config = self.repo.config self.args = args - hardlink_lock = self.config["core"].get("hardlink_lock", False) - updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock) - updater.check() + if self.repo.tmp_dir: + hardlink_lock = self.config["core"].get("hardlink_lock", False) + updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock) + updater.check() @property def default_targets(self): diff --git a/dvc/lock.py b/dvc/lock.py index 5ae5d5d2f..59e27679d 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -2,6 +2,7 @@ import hashlib import os +from abc import ABC, abstractmethod from datetime import timedelta import flufl.lock @@ -25,20 +26,74 @@ class LockError(DvcException): """Thrown when unable to acquire the lock for DVC repo.""" -class Lock: +class LockBase(ABC): + @abstractmethod + def __init__(self, lockfile): + self._lockfile = lockfile + + @property + def lockfile(self): + return self._lockfile + + @abstractmethod + def lock(self): + pass + + @abstractmethod + def unlock(self): + pass + + @property + @abstractmethod + def is_locked(self): + pass + + @abstractmethod + def __enter__(self): + pass + + @abstractmethod + def __exit__(self, typ, value, tbck): + pass + + +class LockNoop(LockBase): + def __init__( + self, *args, **kwargs + ): # pylint: disable=super-init-not-called + self._lock = False + + def lock(self): + self._lock = True + + def unlock(self): + self._lock = False + + @property + def is_locked(self): + return self._lock + + def __enter__(self): + self.lock() + + def __exit__(self, typ, value, tbck): + self.unlock() + + +class Lock(LockBase): """Class for DVC repo lock. Uses zc.lockfile as backend. """ def __init__(self, lockfile, friendly=False, **kwargs): + super().__init__(lockfile) self._friendly = friendly - self.lockfile = lockfile self._lock = None @property def files(self): - return [self.lockfile] + return [self._lockfile] def _do_lock(self): try: @@ -51,7 +106,7 @@ class Lock: ) ), ): - self._lock = zc.lockfile.LockFile(self.lockfile) + self._lock = zc.lockfile.LockFile(self._lockfile) except zc.lockfile.LockError: raise LockError(FAILED_TO_LOCK_MESSAGE) @@ -76,7 +131,7 @@ class Lock: self.unlock() -class HardlinkLock(flufl.lock.Lock): +class HardlinkLock(flufl.lock.Lock, LockBase): """Class for DVC repo lock. Args: @@ -90,6 +145,8 @@ class HardlinkLock(flufl.lock.Lock): ): # pylint: disable=super-init-not-called import socket + super().__init__(lockfile) + self._tmp_dir = tmp_dir # NOTE: this is basically Lock.__init__ copy-paste, except that @@ -105,17 +162,12 @@ class HardlinkLock(flufl.lock.Lock): # [1] https://github.com/iterative/dvc/issues/2582 self._hostname = socket.gethostname() - self._lockfile = lockfile self._lifetime = timedelta(days=365) # Lock for good by default self._separator = flufl.lock.SEP self._set_claimfile() self._owned = True self._retry_errnos = [] - @property - def lockfile(self): - return self._lockfile - def lock(self): # pylint: disable=arguments-differ try: super().lock(timedelta(seconds=DEFAULT_TIMEOUT)) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 68eaef81c..fb528ce62 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -90,7 +90,7 @@ class Repo: ): from dvc.cache import Cache from dvc.data_cloud import DataCloud - from dvc.lock import make_lock + from dvc.lock import LockNoop, make_lock from dvc.repo.experiments import Experiments from dvc.repo.metrics import Metrics from dvc.repo.params import Params @@ -101,71 +101,63 @@ class Repo: from dvc.tree.local import LocalTree from dvc.utils.fs import makedirs - if scm: - tree = scm.get_tree(rev) + try: + tree = scm.get_tree(rev) if rev else None self.root_dir = self.find_root(root_dir, tree) - self.scm = scm - self.tree = scm.get_tree( - rev, use_dvcignore=True, dvcignore_root=self.root_dir - ) - self.state = StateNoop() + self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) + self.tmp_dir = os.path.join(self.dvc_dir, "tmp") + makedirs(self.tmp_dir, exist_ok=True) + except NotDvcRepoError: + if not uninitialized: + raise + self.root_dir = SCM(root_dir or os.curdir).root_dir + self.dvc_dir = None + self.tmp_dir = None + + tree_kwargs = dict(use_dvcignore=True, dvcignore_root=self.root_dir) + if scm: + self.tree = scm.get_tree(rev, **tree_kwargs) else: - try: - root_dir = self.find_root(root_dir) - except NotDvcRepoError: - if not uninitialized: - raise - root_dir = SCM(root_dir or os.curdir).root_dir - self.root_dir = os.path.abspath(os.path.realpath(root_dir)) - self.tree = LocalTree( - self, - {"url": self.root_dir}, - use_dvcignore=True, - dvcignore_root=self.root_dir, - ) + self.tree = LocalTree(self, {"url": self.root_dir}, **tree_kwargs) - self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) self.config = Config(self.dvc_dir, tree=self.tree) + no_scm = self.config["core"].get("no_scm", False) + self.scm = scm if scm else SCM(self.root_dir, no_scm=no_scm) - if not scm: - no_scm = self.config["core"].get("no_scm", False) - self.scm = SCM(self.root_dir, no_scm=no_scm) - - self.tmp_dir = os.path.join(self.dvc_dir, "tmp") - makedirs(self.tmp_dir, exist_ok=True) - - hardlink_lock = self.config["core"].get("hardlink_lock", False) - self.lock = make_lock( - os.path.join(self.tmp_dir, "lock"), - tmp_dir=self.tmp_dir, - hardlink_lock=hardlink_lock, - friendly=True, - ) # used by RepoTree to determine if it should traverse subrepos self.subrepos = subrepos self.cache = Cache(self) self.cloud = DataCloud(self) - if not scm: + if scm or not self.dvc_dir: + self.lock = LockNoop() + self.state = StateNoop() + else: + self.lock = make_lock( + os.path.join(self.tmp_dir, "lock"), + tmp_dir=self.tmp_dir, + hardlink_lock=self.config["core"].get("hardlink_lock", False), + friendly=True, + ) + # NOTE: storing state and link_state in the repository itself to # avoid any possible state corruption in 'shared cache dir' # scenario. self.state = State(self.cache.local) + self.stage_cache = StageCache(self) - self.stage_cache = StageCache(self) + try: + self.experiments = Experiments(self) + except NotImplementedError: + self.experiments = None + + self._ignore() self.metrics = Metrics(self) self.plots = Plots(self) self.params = Params(self) - try: - self.experiments = Experiments(self) - except NotImplementedError: - self.experiments = None - - self._ignore() - @property def tree(self): return self._tree diff --git a/dvc/repo/init.py b/dvc/repo/init.py index 8360bf3c8..ecce3e4a8 100644 --- a/dvc/repo/init.py +++ b/dvc/repo/init.py @@ -100,6 +100,8 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): proj = Repo(root_dir) + proj.plots.templates.init() + scm.add( [config.files["repo"], dvcignore, proj.plots.templates.templates_dir] ) diff --git a/dvc/repo/plots/template.py b/dvc/repo/plots/template.py index a4e95df21..a0a1ac0a0 100644 --- a/dvc/repo/plots/template.py +++ b/dvc/repo/plots/template.py @@ -33,7 +33,18 @@ class Template: DEFAULT_NAME = None def __init__(self, content=None, name=None): - self.content = self.DEFAULT_CONTENT if content is None else content + if content: + self.content = content + else: + self.content = ( + json.dumps( + self.DEFAULT_CONTENT, + indent=self.INDENT, + separators=self.SEPARATORS, + ) + + "\n" + ) + self.name = name or self.DEFAULT_NAME assert self.content and self.name self.filename = self.name + self.EXTENSION @@ -216,60 +227,54 @@ class PlotTemplates: def templates_dir(self): return os.path.join(self.dvc_dir, self.TEMPLATES_DIR) - @cached_property - def default_template(self): - default_plot_path = os.path.join(self.templates_dir, "default.json") - if not os.path.exists(default_plot_path): - raise TemplateNotFoundError(os.path.relpath(default_plot_path)) - return default_plot_path - def get_template(self, path): if os.path.exists(path): return path - t_path = os.path.join(self.templates_dir, path) - if os.path.exists(t_path): - return t_path + if self.dvc_dir and os.path.exists(self.dvc_dir): + t_path = os.path.join(self.templates_dir, path) + if os.path.exists(t_path): + return t_path - all_templates = [ - os.path.join(root, file) - for root, _, files in os.walk(self.templates_dir) - for file in files - ] - matches = [ - template - for template in all_templates - if os.path.splitext(template)[0] == t_path - ] - if matches: - assert len(matches) == 1 - return matches[0] + all_templates = [ + os.path.join(root, file) + for root, _, files in os.walk(self.templates_dir) + for file in files + ] + matches = [ + template + for template in all_templates + if os.path.splitext(template)[0] == t_path + ] + if matches: + assert len(matches) == 1 + return matches[0] raise TemplateNotFoundError(path) def __init__(self, dvc_dir): self.dvc_dir = dvc_dir - if not os.path.exists(self.templates_dir): - makedirs(self.templates_dir, exist_ok=True) - for t in self.TEMPLATES: - self.dump(t()) + def init(self): + makedirs(self.templates_dir, exist_ok=True) + for t in self.TEMPLATES: + self._dump(t()) - def dump(self, template): + def _dump(self, template): path = os.path.join(self.templates_dir, template.filename) with open(path, "w") as fd: - json.dump( - template.content, - fd, - indent=template.INDENT, - separators=template.SEPARATORS, - ) - fd.write("\n") + fd.write(template.content) def load(self, name): - path = self.get_template(name) + try: + path = self.get_template(name) - with open(path) as fd: - content = fd.read() + with open(path) as fd: + content = fd.read() - return Template(content, name=name) + return Template(content, name=name) + except TemplateNotFoundError: + for template in self.TEMPLATES: + if template.DEFAULT_NAME == name: + return template() + raise diff --git a/tests/func/metrics/test_show.py b/tests/func/metrics/test_show.py index 96d3f9d01..818be806d 100644 --- a/tests/func/metrics/test_show.py +++ b/tests/func/metrics/test_show.py @@ -125,6 +125,9 @@ def test_show_non_metric(tmp_dir, scm, use_dvc): "": {"metrics.yaml": {"foo": 1.1}} } + if not use_dvc: + assert not (tmp_dir / ".dvc").exists() + @pytest.mark.parametrize("use_dvc", [True, False]) def test_show_non_metric_branch(tmp_dir, scm, use_dvc): @@ -142,6 +145,9 @@ def test_show_non_metric_branch(tmp_dir, scm, use_dvc): "branch": {"metrics.yaml": {"foo": 2.2}}, } + if not use_dvc: + assert not (tmp_dir / ".dvc").exists() + def test_non_metric_and_recurisve_show(tmp_dir, dvc, run_copy_metrics): tmp_dir.gen( diff --git a/tests/func/plots/test_plots.py b/tests/func/plots/test_plots.py index 758886fda..cdb478629 100644 --- a/tests/func/plots/test_plots.py +++ b/tests/func/plots/test_plots.py @@ -638,6 +638,9 @@ def test_show_non_plot(tmp_dir, scm, use_dvc): assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD assert plot_content["encoding"]["y"]["field"] == "val" + if not use_dvc: + assert not (tmp_dir / ".dvc").exists() + def test_show_non_plot_and_plot_with_params( tmp_dir, scm, dvc, run_copy_metrics