mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
repo: don't create .dvc in read-only repos (#4674)
* lock: use ABC * lock: add LockNoop * plots: remove unused code * plots: create templates only on init * dvc: don't create .dvc in read-only repos * repo: explicitly init LockNoop
This commit is contained in:
Vendored
+3
@@ -62,6 +62,9 @@ class Cache:
|
||||
|
||||
if local:
|
||||
settings = {"name": local}
|
||||
elif "dir" not in config:
|
||||
self.local = None
|
||||
return
|
||||
else:
|
||||
from ..config import LOCAL_COMMON
|
||||
|
||||
|
||||
+4
-3
@@ -40,9 +40,10 @@ class CmdBase(ABC):
|
||||
self.repo = Repo(uninitialized=self.UNINITIALIZED)
|
||||
self.config = self.repo.config
|
||||
self.args = args
|
||||
hardlink_lock = self.config["core"].get("hardlink_lock", False)
|
||||
updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
|
||||
updater.check()
|
||||
if self.repo.tmp_dir:
|
||||
hardlink_lock = self.config["core"].get("hardlink_lock", False)
|
||||
updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
|
||||
updater.check()
|
||||
|
||||
@property
|
||||
def default_targets(self):
|
||||
|
||||
+62
-10
@@ -2,6 +2,7 @@
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import timedelta
|
||||
|
||||
import flufl.lock
|
||||
@@ -25,20 +26,74 @@ class LockError(DvcException):
|
||||
"""Thrown when unable to acquire the lock for DVC repo."""
|
||||
|
||||
|
||||
class Lock:
|
||||
class LockBase(ABC):
|
||||
@abstractmethod
|
||||
def __init__(self, lockfile):
|
||||
self._lockfile = lockfile
|
||||
|
||||
@property
|
||||
def lockfile(self):
|
||||
return self._lockfile
|
||||
|
||||
@abstractmethod
|
||||
def lock(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def unlock(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def is_locked(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def __enter__(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def __exit__(self, typ, value, tbck):
|
||||
pass
|
||||
|
||||
|
||||
class LockNoop(LockBase):
|
||||
def __init__(
|
||||
self, *args, **kwargs
|
||||
): # pylint: disable=super-init-not-called
|
||||
self._lock = False
|
||||
|
||||
def lock(self):
|
||||
self._lock = True
|
||||
|
||||
def unlock(self):
|
||||
self._lock = False
|
||||
|
||||
@property
|
||||
def is_locked(self):
|
||||
return self._lock
|
||||
|
||||
def __enter__(self):
|
||||
self.lock()
|
||||
|
||||
def __exit__(self, typ, value, tbck):
|
||||
self.unlock()
|
||||
|
||||
|
||||
class Lock(LockBase):
|
||||
"""Class for DVC repo lock.
|
||||
|
||||
Uses zc.lockfile as backend.
|
||||
"""
|
||||
|
||||
def __init__(self, lockfile, friendly=False, **kwargs):
|
||||
super().__init__(lockfile)
|
||||
self._friendly = friendly
|
||||
self.lockfile = lockfile
|
||||
self._lock = None
|
||||
|
||||
@property
|
||||
def files(self):
|
||||
return [self.lockfile]
|
||||
return [self._lockfile]
|
||||
|
||||
def _do_lock(self):
|
||||
try:
|
||||
@@ -51,7 +106,7 @@ class Lock:
|
||||
)
|
||||
),
|
||||
):
|
||||
self._lock = zc.lockfile.LockFile(self.lockfile)
|
||||
self._lock = zc.lockfile.LockFile(self._lockfile)
|
||||
except zc.lockfile.LockError:
|
||||
raise LockError(FAILED_TO_LOCK_MESSAGE)
|
||||
|
||||
@@ -76,7 +131,7 @@ class Lock:
|
||||
self.unlock()
|
||||
|
||||
|
||||
class HardlinkLock(flufl.lock.Lock):
|
||||
class HardlinkLock(flufl.lock.Lock, LockBase):
|
||||
"""Class for DVC repo lock.
|
||||
|
||||
Args:
|
||||
@@ -90,6 +145,8 @@ class HardlinkLock(flufl.lock.Lock):
|
||||
): # pylint: disable=super-init-not-called
|
||||
import socket
|
||||
|
||||
super().__init__(lockfile)
|
||||
|
||||
self._tmp_dir = tmp_dir
|
||||
|
||||
# NOTE: this is basically Lock.__init__ copy-paste, except that
|
||||
@@ -105,17 +162,12 @@ class HardlinkLock(flufl.lock.Lock):
|
||||
# [1] https://github.com/iterative/dvc/issues/2582
|
||||
self._hostname = socket.gethostname()
|
||||
|
||||
self._lockfile = lockfile
|
||||
self._lifetime = timedelta(days=365) # Lock for good by default
|
||||
self._separator = flufl.lock.SEP
|
||||
self._set_claimfile()
|
||||
self._owned = True
|
||||
self._retry_errnos = []
|
||||
|
||||
@property
|
||||
def lockfile(self):
|
||||
return self._lockfile
|
||||
|
||||
def lock(self): # pylint: disable=arguments-differ
|
||||
try:
|
||||
super().lock(timedelta(seconds=DEFAULT_TIMEOUT))
|
||||
|
||||
+37
-45
@@ -90,7 +90,7 @@ class Repo:
|
||||
):
|
||||
from dvc.cache import Cache
|
||||
from dvc.data_cloud import DataCloud
|
||||
from dvc.lock import make_lock
|
||||
from dvc.lock import LockNoop, make_lock
|
||||
from dvc.repo.experiments import Experiments
|
||||
from dvc.repo.metrics import Metrics
|
||||
from dvc.repo.params import Params
|
||||
@@ -101,71 +101,63 @@ class Repo:
|
||||
from dvc.tree.local import LocalTree
|
||||
from dvc.utils.fs import makedirs
|
||||
|
||||
if scm:
|
||||
tree = scm.get_tree(rev)
|
||||
try:
|
||||
tree = scm.get_tree(rev) if rev else None
|
||||
self.root_dir = self.find_root(root_dir, tree)
|
||||
self.scm = scm
|
||||
self.tree = scm.get_tree(
|
||||
rev, use_dvcignore=True, dvcignore_root=self.root_dir
|
||||
)
|
||||
self.state = StateNoop()
|
||||
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
|
||||
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
|
||||
makedirs(self.tmp_dir, exist_ok=True)
|
||||
except NotDvcRepoError:
|
||||
if not uninitialized:
|
||||
raise
|
||||
self.root_dir = SCM(root_dir or os.curdir).root_dir
|
||||
self.dvc_dir = None
|
||||
self.tmp_dir = None
|
||||
|
||||
tree_kwargs = dict(use_dvcignore=True, dvcignore_root=self.root_dir)
|
||||
if scm:
|
||||
self.tree = scm.get_tree(rev, **tree_kwargs)
|
||||
else:
|
||||
try:
|
||||
root_dir = self.find_root(root_dir)
|
||||
except NotDvcRepoError:
|
||||
if not uninitialized:
|
||||
raise
|
||||
root_dir = SCM(root_dir or os.curdir).root_dir
|
||||
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
|
||||
self.tree = LocalTree(
|
||||
self,
|
||||
{"url": self.root_dir},
|
||||
use_dvcignore=True,
|
||||
dvcignore_root=self.root_dir,
|
||||
)
|
||||
self.tree = LocalTree(self, {"url": self.root_dir}, **tree_kwargs)
|
||||
|
||||
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
|
||||
self.config = Config(self.dvc_dir, tree=self.tree)
|
||||
no_scm = self.config["core"].get("no_scm", False)
|
||||
self.scm = scm if scm else SCM(self.root_dir, no_scm=no_scm)
|
||||
|
||||
if not scm:
|
||||
no_scm = self.config["core"].get("no_scm", False)
|
||||
self.scm = SCM(self.root_dir, no_scm=no_scm)
|
||||
|
||||
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
|
||||
makedirs(self.tmp_dir, exist_ok=True)
|
||||
|
||||
hardlink_lock = self.config["core"].get("hardlink_lock", False)
|
||||
self.lock = make_lock(
|
||||
os.path.join(self.tmp_dir, "lock"),
|
||||
tmp_dir=self.tmp_dir,
|
||||
hardlink_lock=hardlink_lock,
|
||||
friendly=True,
|
||||
)
|
||||
# used by RepoTree to determine if it should traverse subrepos
|
||||
self.subrepos = subrepos
|
||||
|
||||
self.cache = Cache(self)
|
||||
self.cloud = DataCloud(self)
|
||||
|
||||
if not scm:
|
||||
if scm or not self.dvc_dir:
|
||||
self.lock = LockNoop()
|
||||
self.state = StateNoop()
|
||||
else:
|
||||
self.lock = make_lock(
|
||||
os.path.join(self.tmp_dir, "lock"),
|
||||
tmp_dir=self.tmp_dir,
|
||||
hardlink_lock=self.config["core"].get("hardlink_lock", False),
|
||||
friendly=True,
|
||||
)
|
||||
|
||||
# NOTE: storing state and link_state in the repository itself to
|
||||
# avoid any possible state corruption in 'shared cache dir'
|
||||
# scenario.
|
||||
self.state = State(self.cache.local)
|
||||
self.stage_cache = StageCache(self)
|
||||
|
||||
self.stage_cache = StageCache(self)
|
||||
try:
|
||||
self.experiments = Experiments(self)
|
||||
except NotImplementedError:
|
||||
self.experiments = None
|
||||
|
||||
self._ignore()
|
||||
|
||||
self.metrics = Metrics(self)
|
||||
self.plots = Plots(self)
|
||||
self.params = Params(self)
|
||||
|
||||
try:
|
||||
self.experiments = Experiments(self)
|
||||
except NotImplementedError:
|
||||
self.experiments = None
|
||||
|
||||
self._ignore()
|
||||
|
||||
@property
|
||||
def tree(self):
|
||||
return self._tree
|
||||
|
||||
@@ -100,6 +100,8 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
|
||||
|
||||
proj = Repo(root_dir)
|
||||
|
||||
proj.plots.templates.init()
|
||||
|
||||
scm.add(
|
||||
[config.files["repo"], dvcignore, proj.plots.templates.templates_dir]
|
||||
)
|
||||
|
||||
+45
-40
@@ -33,7 +33,18 @@ class Template:
|
||||
DEFAULT_NAME = None
|
||||
|
||||
def __init__(self, content=None, name=None):
|
||||
self.content = self.DEFAULT_CONTENT if content is None else content
|
||||
if content:
|
||||
self.content = content
|
||||
else:
|
||||
self.content = (
|
||||
json.dumps(
|
||||
self.DEFAULT_CONTENT,
|
||||
indent=self.INDENT,
|
||||
separators=self.SEPARATORS,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
self.name = name or self.DEFAULT_NAME
|
||||
assert self.content and self.name
|
||||
self.filename = self.name + self.EXTENSION
|
||||
@@ -216,60 +227,54 @@ class PlotTemplates:
|
||||
def templates_dir(self):
|
||||
return os.path.join(self.dvc_dir, self.TEMPLATES_DIR)
|
||||
|
||||
@cached_property
|
||||
def default_template(self):
|
||||
default_plot_path = os.path.join(self.templates_dir, "default.json")
|
||||
if not os.path.exists(default_plot_path):
|
||||
raise TemplateNotFoundError(os.path.relpath(default_plot_path))
|
||||
return default_plot_path
|
||||
|
||||
def get_template(self, path):
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
t_path = os.path.join(self.templates_dir, path)
|
||||
if os.path.exists(t_path):
|
||||
return t_path
|
||||
if self.dvc_dir and os.path.exists(self.dvc_dir):
|
||||
t_path = os.path.join(self.templates_dir, path)
|
||||
if os.path.exists(t_path):
|
||||
return t_path
|
||||
|
||||
all_templates = [
|
||||
os.path.join(root, file)
|
||||
for root, _, files in os.walk(self.templates_dir)
|
||||
for file in files
|
||||
]
|
||||
matches = [
|
||||
template
|
||||
for template in all_templates
|
||||
if os.path.splitext(template)[0] == t_path
|
||||
]
|
||||
if matches:
|
||||
assert len(matches) == 1
|
||||
return matches[0]
|
||||
all_templates = [
|
||||
os.path.join(root, file)
|
||||
for root, _, files in os.walk(self.templates_dir)
|
||||
for file in files
|
||||
]
|
||||
matches = [
|
||||
template
|
||||
for template in all_templates
|
||||
if os.path.splitext(template)[0] == t_path
|
||||
]
|
||||
if matches:
|
||||
assert len(matches) == 1
|
||||
return matches[0]
|
||||
|
||||
raise TemplateNotFoundError(path)
|
||||
|
||||
def __init__(self, dvc_dir):
|
||||
self.dvc_dir = dvc_dir
|
||||
|
||||
if not os.path.exists(self.templates_dir):
|
||||
makedirs(self.templates_dir, exist_ok=True)
|
||||
for t in self.TEMPLATES:
|
||||
self.dump(t())
|
||||
def init(self):
|
||||
makedirs(self.templates_dir, exist_ok=True)
|
||||
for t in self.TEMPLATES:
|
||||
self._dump(t())
|
||||
|
||||
def dump(self, template):
|
||||
def _dump(self, template):
|
||||
path = os.path.join(self.templates_dir, template.filename)
|
||||
with open(path, "w") as fd:
|
||||
json.dump(
|
||||
template.content,
|
||||
fd,
|
||||
indent=template.INDENT,
|
||||
separators=template.SEPARATORS,
|
||||
)
|
||||
fd.write("\n")
|
||||
fd.write(template.content)
|
||||
|
||||
def load(self, name):
|
||||
path = self.get_template(name)
|
||||
try:
|
||||
path = self.get_template(name)
|
||||
|
||||
with open(path) as fd:
|
||||
content = fd.read()
|
||||
with open(path) as fd:
|
||||
content = fd.read()
|
||||
|
||||
return Template(content, name=name)
|
||||
return Template(content, name=name)
|
||||
except TemplateNotFoundError:
|
||||
for template in self.TEMPLATES:
|
||||
if template.DEFAULT_NAME == name:
|
||||
return template()
|
||||
raise
|
||||
|
||||
@@ -125,6 +125,9 @@ def test_show_non_metric(tmp_dir, scm, use_dvc):
|
||||
"": {"metrics.yaml": {"foo": 1.1}}
|
||||
}
|
||||
|
||||
if not use_dvc:
|
||||
assert not (tmp_dir / ".dvc").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_dvc", [True, False])
|
||||
def test_show_non_metric_branch(tmp_dir, scm, use_dvc):
|
||||
@@ -142,6 +145,9 @@ def test_show_non_metric_branch(tmp_dir, scm, use_dvc):
|
||||
"branch": {"metrics.yaml": {"foo": 2.2}},
|
||||
}
|
||||
|
||||
if not use_dvc:
|
||||
assert not (tmp_dir / ".dvc").exists()
|
||||
|
||||
|
||||
def test_non_metric_and_recurisve_show(tmp_dir, dvc, run_copy_metrics):
|
||||
tmp_dir.gen(
|
||||
|
||||
@@ -638,6 +638,9 @@ def test_show_non_plot(tmp_dir, scm, use_dvc):
|
||||
assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
|
||||
assert plot_content["encoding"]["y"]["field"] == "val"
|
||||
|
||||
if not use_dvc:
|
||||
assert not (tmp_dir / ".dvc").exists()
|
||||
|
||||
|
||||
def test_show_non_plot_and_plot_with_params(
|
||||
tmp_dir, scm, dvc, run_copy_metrics
|
||||
|
||||
Reference in New Issue
Block a user