repo: don't create .dvc in read-only repos (#4674)

* lock: use ABC

* lock: add LockNoop

* plots: remove unused code

* plots: create templates only on init

* dvc: don't create .dvc in read-only repos

* repo: explicitly init LockNoop
This commit is contained in:
Ruslan Kuprieiev
2020-10-07 23:51:14 +03:00
committed by GitHub
parent 548c2dcc1f
commit 775e1e8374
8 changed files with 162 additions and 98 deletions
+3
View File
@@ -62,6 +62,9 @@ class Cache:
if local:
settings = {"name": local}
elif "dir" not in config:
self.local = None
return
else:
from ..config import LOCAL_COMMON
+4 -3
View File
@@ -40,9 +40,10 @@ class CmdBase(ABC):
self.repo = Repo(uninitialized=self.UNINITIALIZED)
self.config = self.repo.config
self.args = args
hardlink_lock = self.config["core"].get("hardlink_lock", False)
updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
updater.check()
if self.repo.tmp_dir:
hardlink_lock = self.config["core"].get("hardlink_lock", False)
updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
updater.check()
@property
def default_targets(self):
+62 -10
View File
@@ -2,6 +2,7 @@
import hashlib
import os
from abc import ABC, abstractmethod
from datetime import timedelta
import flufl.lock
@@ -25,20 +26,74 @@ class LockError(DvcException):
"""Thrown when unable to acquire the lock for DVC repo."""
class Lock:
class LockBase(ABC):
@abstractmethod
def __init__(self, lockfile):
self._lockfile = lockfile
@property
def lockfile(self):
return self._lockfile
@abstractmethod
def lock(self):
pass
@abstractmethod
def unlock(self):
pass
@property
@abstractmethod
def is_locked(self):
pass
@abstractmethod
def __enter__(self):
pass
@abstractmethod
def __exit__(self, typ, value, tbck):
pass
class LockNoop(LockBase):
def __init__(
self, *args, **kwargs
): # pylint: disable=super-init-not-called
self._lock = False
def lock(self):
self._lock = True
def unlock(self):
self._lock = False
@property
def is_locked(self):
return self._lock
def __enter__(self):
self.lock()
def __exit__(self, typ, value, tbck):
self.unlock()
class Lock(LockBase):
"""Class for DVC repo lock.
Uses zc.lockfile as backend.
"""
def __init__(self, lockfile, friendly=False, **kwargs):
super().__init__(lockfile)
self._friendly = friendly
self.lockfile = lockfile
self._lock = None
@property
def files(self):
return [self.lockfile]
return [self._lockfile]
def _do_lock(self):
try:
@@ -51,7 +106,7 @@ class Lock:
)
),
):
self._lock = zc.lockfile.LockFile(self.lockfile)
self._lock = zc.lockfile.LockFile(self._lockfile)
except zc.lockfile.LockError:
raise LockError(FAILED_TO_LOCK_MESSAGE)
@@ -76,7 +131,7 @@ class Lock:
self.unlock()
class HardlinkLock(flufl.lock.Lock):
class HardlinkLock(flufl.lock.Lock, LockBase):
"""Class for DVC repo lock.
Args:
@@ -90,6 +145,8 @@ class HardlinkLock(flufl.lock.Lock):
): # pylint: disable=super-init-not-called
import socket
super().__init__(lockfile)
self._tmp_dir = tmp_dir
# NOTE: this is basically Lock.__init__ copy-paste, except that
@@ -105,17 +162,12 @@ class HardlinkLock(flufl.lock.Lock):
# [1] https://github.com/iterative/dvc/issues/2582
self._hostname = socket.gethostname()
self._lockfile = lockfile
self._lifetime = timedelta(days=365) # Lock for good by default
self._separator = flufl.lock.SEP
self._set_claimfile()
self._owned = True
self._retry_errnos = []
@property
def lockfile(self):
return self._lockfile
def lock(self): # pylint: disable=arguments-differ
try:
super().lock(timedelta(seconds=DEFAULT_TIMEOUT))
+37 -45
View File
@@ -90,7 +90,7 @@ class Repo:
):
from dvc.cache import Cache
from dvc.data_cloud import DataCloud
from dvc.lock import make_lock
from dvc.lock import LockNoop, make_lock
from dvc.repo.experiments import Experiments
from dvc.repo.metrics import Metrics
from dvc.repo.params import Params
@@ -101,71 +101,63 @@ class Repo:
from dvc.tree.local import LocalTree
from dvc.utils.fs import makedirs
if scm:
tree = scm.get_tree(rev)
try:
tree = scm.get_tree(rev) if rev else None
self.root_dir = self.find_root(root_dir, tree)
self.scm = scm
self.tree = scm.get_tree(
rev, use_dvcignore=True, dvcignore_root=self.root_dir
)
self.state = StateNoop()
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
makedirs(self.tmp_dir, exist_ok=True)
except NotDvcRepoError:
if not uninitialized:
raise
self.root_dir = SCM(root_dir or os.curdir).root_dir
self.dvc_dir = None
self.tmp_dir = None
tree_kwargs = dict(use_dvcignore=True, dvcignore_root=self.root_dir)
if scm:
self.tree = scm.get_tree(rev, **tree_kwargs)
else:
try:
root_dir = self.find_root(root_dir)
except NotDvcRepoError:
if not uninitialized:
raise
root_dir = SCM(root_dir or os.curdir).root_dir
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
self.tree = LocalTree(
self,
{"url": self.root_dir},
use_dvcignore=True,
dvcignore_root=self.root_dir,
)
self.tree = LocalTree(self, {"url": self.root_dir}, **tree_kwargs)
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
self.config = Config(self.dvc_dir, tree=self.tree)
no_scm = self.config["core"].get("no_scm", False)
self.scm = scm if scm else SCM(self.root_dir, no_scm=no_scm)
if not scm:
no_scm = self.config["core"].get("no_scm", False)
self.scm = SCM(self.root_dir, no_scm=no_scm)
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
makedirs(self.tmp_dir, exist_ok=True)
hardlink_lock = self.config["core"].get("hardlink_lock", False)
self.lock = make_lock(
os.path.join(self.tmp_dir, "lock"),
tmp_dir=self.tmp_dir,
hardlink_lock=hardlink_lock,
friendly=True,
)
# used by RepoTree to determine if it should traverse subrepos
self.subrepos = subrepos
self.cache = Cache(self)
self.cloud = DataCloud(self)
if not scm:
if scm or not self.dvc_dir:
self.lock = LockNoop()
self.state = StateNoop()
else:
self.lock = make_lock(
os.path.join(self.tmp_dir, "lock"),
tmp_dir=self.tmp_dir,
hardlink_lock=self.config["core"].get("hardlink_lock", False),
friendly=True,
)
# NOTE: storing state and link_state in the repository itself to
# avoid any possible state corruption in 'shared cache dir'
# scenario.
self.state = State(self.cache.local)
self.stage_cache = StageCache(self)
self.stage_cache = StageCache(self)
try:
self.experiments = Experiments(self)
except NotImplementedError:
self.experiments = None
self._ignore()
self.metrics = Metrics(self)
self.plots = Plots(self)
self.params = Params(self)
try:
self.experiments = Experiments(self)
except NotImplementedError:
self.experiments = None
self._ignore()
@property
def tree(self):
return self._tree
+2
View File
@@ -100,6 +100,8 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
proj = Repo(root_dir)
proj.plots.templates.init()
scm.add(
[config.files["repo"], dvcignore, proj.plots.templates.templates_dir]
)
+45 -40
View File
@@ -33,7 +33,18 @@ class Template:
DEFAULT_NAME = None
def __init__(self, content=None, name=None):
self.content = self.DEFAULT_CONTENT if content is None else content
if content:
self.content = content
else:
self.content = (
json.dumps(
self.DEFAULT_CONTENT,
indent=self.INDENT,
separators=self.SEPARATORS,
)
+ "\n"
)
self.name = name or self.DEFAULT_NAME
assert self.content and self.name
self.filename = self.name + self.EXTENSION
@@ -216,60 +227,54 @@ class PlotTemplates:
def templates_dir(self):
return os.path.join(self.dvc_dir, self.TEMPLATES_DIR)
@cached_property
def default_template(self):
default_plot_path = os.path.join(self.templates_dir, "default.json")
if not os.path.exists(default_plot_path):
raise TemplateNotFoundError(os.path.relpath(default_plot_path))
return default_plot_path
def get_template(self, path):
if os.path.exists(path):
return path
t_path = os.path.join(self.templates_dir, path)
if os.path.exists(t_path):
return t_path
if self.dvc_dir and os.path.exists(self.dvc_dir):
t_path = os.path.join(self.templates_dir, path)
if os.path.exists(t_path):
return t_path
all_templates = [
os.path.join(root, file)
for root, _, files in os.walk(self.templates_dir)
for file in files
]
matches = [
template
for template in all_templates
if os.path.splitext(template)[0] == t_path
]
if matches:
assert len(matches) == 1
return matches[0]
all_templates = [
os.path.join(root, file)
for root, _, files in os.walk(self.templates_dir)
for file in files
]
matches = [
template
for template in all_templates
if os.path.splitext(template)[0] == t_path
]
if matches:
assert len(matches) == 1
return matches[0]
raise TemplateNotFoundError(path)
def __init__(self, dvc_dir):
self.dvc_dir = dvc_dir
if not os.path.exists(self.templates_dir):
makedirs(self.templates_dir, exist_ok=True)
for t in self.TEMPLATES:
self.dump(t())
def init(self):
makedirs(self.templates_dir, exist_ok=True)
for t in self.TEMPLATES:
self._dump(t())
def dump(self, template):
def _dump(self, template):
path = os.path.join(self.templates_dir, template.filename)
with open(path, "w") as fd:
json.dump(
template.content,
fd,
indent=template.INDENT,
separators=template.SEPARATORS,
)
fd.write("\n")
fd.write(template.content)
def load(self, name):
path = self.get_template(name)
try:
path = self.get_template(name)
with open(path) as fd:
content = fd.read()
with open(path) as fd:
content = fd.read()
return Template(content, name=name)
return Template(content, name=name)
except TemplateNotFoundError:
for template in self.TEMPLATES:
if template.DEFAULT_NAME == name:
return template()
raise
+6
View File
@@ -125,6 +125,9 @@ def test_show_non_metric(tmp_dir, scm, use_dvc):
"": {"metrics.yaml": {"foo": 1.1}}
}
if not use_dvc:
assert not (tmp_dir / ".dvc").exists()
@pytest.mark.parametrize("use_dvc", [True, False])
def test_show_non_metric_branch(tmp_dir, scm, use_dvc):
@@ -142,6 +145,9 @@ def test_show_non_metric_branch(tmp_dir, scm, use_dvc):
"branch": {"metrics.yaml": {"foo": 2.2}},
}
if not use_dvc:
assert not (tmp_dir / ".dvc").exists()
def test_non_metric_and_recurisve_show(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen(
+3
View File
@@ -638,6 +638,9 @@ def test_show_non_plot(tmp_dir, scm, use_dvc):
assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
assert plot_content["encoding"]["y"]["field"] == "val"
if not use_dvc:
assert not (tmp_dir / ".dvc").exists()
def test_show_non_plot_and_plot_with_params(
tmp_dir, scm, dvc, run_copy_metrics