mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
63972c3d12 | ||
|
|
7027c631c5 | ||
|
|
e8f6cd7f5b | ||
|
|
10f2bb93fa | ||
|
|
2fecc66cb9 | ||
|
|
b47998463f | ||
|
|
d53b740f35 | ||
|
|
9198fa7459 | ||
|
|
6954efc6e6 | ||
|
|
d77af8196e | ||
|
|
c5920b00ab | ||
|
|
ce8036e535 | ||
|
|
b7ad2dc841 | ||
|
|
5ab948f8b6 | ||
|
|
f9fbe97509 | ||
|
|
2fba2e030a | ||
|
|
3e951162b8 | ||
|
|
d74358eecd | ||
|
|
58c9ff0de2 | ||
|
|
f0e991790e | ||
|
|
b7cd785eba | ||
|
|
21a3efe2c7 | ||
|
|
865d355554 | ||
|
|
84712c4acc | ||
|
|
0f16e15453 | ||
|
|
61f30ed5ba | ||
|
|
cde15e3d3e | ||
|
|
7672e324d2 | ||
|
|
130b76c7a5 | ||
|
|
040458a726 | ||
|
|
75e795c491 | ||
|
|
4dac2092c9 | ||
|
|
19eaced304 | ||
|
|
184028f176 | ||
|
|
5f63d538d2 | ||
|
|
324ff4be59 | ||
|
|
3f846a1794 | ||
|
|
b52c925d5f | ||
|
|
4a4f07968c | ||
|
|
09136f2378 | ||
|
|
aaca1401ea | ||
|
|
2aaf065ea3 | ||
|
|
d4ec871a76 | ||
|
|
f44ddcf64e | ||
|
|
b88b3b81c6 | ||
|
|
81d04f887a | ||
|
|
afbb3a0add | ||
|
|
b8aa8ad951 | ||
|
|
9897f7c470 | ||
|
|
cf7f125c14 | ||
|
|
cec4b4e003 | ||
|
|
ff4fcc937b | ||
|
|
1676a7a1cd | ||
|
|
974cf097dd | ||
|
|
597ab98d0e | ||
|
|
833e82a7ea | ||
|
|
3916d4fa07 | ||
|
|
d81b56978f | ||
|
|
1e7efffe42 | ||
|
|
cc16b28a26 | ||
|
|
451c7aac04 | ||
|
|
7b4466c982 | ||
|
|
7ca3a1119c | ||
|
|
2c026066db | ||
|
|
7f17c6aa03 | ||
|
|
9f015fac1d | ||
|
|
666e203bbb | ||
|
|
ce946d1160 | ||
|
|
76b31ac282 | ||
|
|
6093a1fe97 | ||
|
|
d62a54c7b0 | ||
|
|
c922da4f3f | ||
|
|
2a16b021ad |
@@ -46,9 +46,6 @@ jobs:
|
||||
env:
|
||||
- PYTHON_VERSION=3.8.0
|
||||
- PATH=/c/Python38:/c/Python38/Scripts:$PATH
|
||||
- name: "3.5 on Linux"
|
||||
language: python
|
||||
python: 3.5
|
||||
- name: "3.6 on Linux"
|
||||
language: python
|
||||
python: 3.6
|
||||
|
||||
+2
-1
@@ -101,7 +101,8 @@ Snap (Snapcraft/Linux)
|
||||
snap install dvc --classic
|
||||
|
||||
This corresponds to the latest tagged release.
|
||||
Add ``--edge`` for the latest ``master`` version.
|
||||
Add ``--beta`` for the latest tagged release candidate,
|
||||
or ``--edge`` for the latest ``master`` version.
|
||||
|
||||
Choco (Chocolatey/Windows)
|
||||
--------------------------
|
||||
|
||||
+3
-5
@@ -72,7 +72,7 @@ def send(path):
|
||||
url = "https://analytics.dvc.org"
|
||||
headers = {"content-type": "application/json"}
|
||||
|
||||
with open(path, "r") as fobj:
|
||||
with open(path) as fobj:
|
||||
report = json.load(fobj)
|
||||
|
||||
report.update(_runtime_info())
|
||||
@@ -158,7 +158,7 @@ def _find_or_create_user_id():
|
||||
try:
|
||||
with Lock(lockfile):
|
||||
try:
|
||||
with open(fname, "r") as fobj:
|
||||
with open(fname) as fobj:
|
||||
user_id = json.load(fobj)["user_id"]
|
||||
|
||||
except (FileNotFoundError, ValueError, KeyError):
|
||||
@@ -170,6 +170,4 @@ def _find_or_create_user_id():
|
||||
return user_id
|
||||
|
||||
except LockError:
|
||||
logger.debug(
|
||||
"Failed to acquire '{lockfile}'".format(lockfile=lockfile)
|
||||
)
|
||||
logger.debug(f"Failed to acquire '{lockfile}'")
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ class UrlNotDvcRepoError(DvcException):
|
||||
"""Thrown if the given URL is not a DVC repository."""
|
||||
|
||||
def __init__(self, url):
|
||||
super().__init__("'{}' is not a DVC repository.".format(url))
|
||||
super().__init__(f"'{url}' is not a DVC repository.")
|
||||
|
||||
|
||||
def get_url(path, repo=None, rev=None, remote=None):
|
||||
|
||||
+2
-2
@@ -40,7 +40,7 @@ def _make_remote_property(name):
|
||||
return cached_property(getter)
|
||||
|
||||
|
||||
class Cache(object):
|
||||
class Cache:
|
||||
"""Class that manages cache locations of a DVC repo.
|
||||
|
||||
Args:
|
||||
@@ -101,7 +101,7 @@ class NamedCacheItem:
|
||||
self.children[checksum].update(child_item)
|
||||
|
||||
|
||||
class NamedCache(object):
|
||||
class NamedCache:
|
||||
def __init__(self):
|
||||
self._items = defaultdict(lambda: defaultdict(NamedCacheItem))
|
||||
self.external = defaultdict(set)
|
||||
|
||||
+4
-8
@@ -27,7 +27,7 @@ from .command import (
|
||||
move,
|
||||
params,
|
||||
pipeline,
|
||||
plot,
|
||||
plots,
|
||||
remote,
|
||||
remove,
|
||||
repro,
|
||||
@@ -74,7 +74,7 @@ COMMANDS = [
|
||||
version,
|
||||
update,
|
||||
git_hook,
|
||||
plot,
|
||||
plots,
|
||||
]
|
||||
|
||||
|
||||
@@ -116,14 +116,10 @@ def get_parent_parser():
|
||||
|
||||
log_level_group = parent_parser.add_mutually_exclusive_group()
|
||||
log_level_group.add_argument(
|
||||
"-q", "--quiet", action="store_true", default=False, help="Be quiet."
|
||||
"-q", "--quiet", action="count", default=0, help="Be quiet."
|
||||
)
|
||||
log_level_group.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Be verbose.",
|
||||
"-v", "--verbose", action="count", default=0, help="Be verbose."
|
||||
)
|
||||
|
||||
return parent_parser
|
||||
|
||||
+2
-3
@@ -26,7 +26,7 @@ def append_doc_link(help_message, path):
|
||||
)
|
||||
|
||||
|
||||
class CmdBase(object):
|
||||
class CmdBase:
|
||||
def __init__(self, args):
|
||||
from dvc.repo import Repo
|
||||
from dvc.updater import Updater
|
||||
@@ -43,8 +43,7 @@ class CmdBase(object):
|
||||
"""Default targets for `dvc repro`."""
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
|
||||
msg = "assuming default target '{}'.".format(PIPELINE_FILE)
|
||||
logger.warning(msg)
|
||||
logger.debug(f"assuming default target '{PIPELINE_FILE}'.")
|
||||
return [PIPELINE_FILE]
|
||||
|
||||
# Abstract methods that have to be implemented by any inheritance class
|
||||
|
||||
@@ -11,6 +11,21 @@ from dvc.exceptions import DvcException
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _show_md(diff):
|
||||
from dvc.utils.diff import table
|
||||
|
||||
rows = []
|
||||
for status in ["added", "deleted", "modified"]:
|
||||
entries = diff.get(status, [])
|
||||
if not entries:
|
||||
continue
|
||||
paths = sorted([entry["path"] for entry in entries])
|
||||
for path in paths:
|
||||
rows.append([status, path])
|
||||
|
||||
return table(["Status", "Path"], rows, True)
|
||||
|
||||
|
||||
class CmdDiff(CmdBase):
|
||||
@staticmethod
|
||||
def _format(diff):
|
||||
@@ -103,6 +118,8 @@ class CmdDiff(CmdBase):
|
||||
|
||||
if self.args.show_json:
|
||||
logger.info(json.dumps(diff))
|
||||
elif self.args.show_md:
|
||||
logger.info(_show_md(diff))
|
||||
elif diff:
|
||||
logger.info(self._format(diff))
|
||||
|
||||
@@ -147,4 +164,10 @@ def add_parser(subparsers, parent_parser):
|
||||
action="store_true",
|
||||
default=False,
|
||||
)
|
||||
diff_parser.add_argument(
|
||||
"--show-md",
|
||||
help="Show tabulated output in the Markdown format (GFM).",
|
||||
action="store_true",
|
||||
default=False,
|
||||
)
|
||||
diff_parser.set_defaults(func=CmdDiff)
|
||||
|
||||
@@ -16,7 +16,7 @@ class CmdGetUrl(CmdBaseNoRepo):
|
||||
Repo.get_url(self.args.url, out=self.args.out)
|
||||
return 0
|
||||
except DvcException:
|
||||
logger.exception("failed to get '{}'".format(self.args.url))
|
||||
logger.exception(f"failed to get '{self.args.url}'")
|
||||
return 1
|
||||
|
||||
|
||||
|
||||
@@ -31,14 +31,14 @@ class CmdList(CmdBaseNoRepo):
|
||||
self.args.path,
|
||||
rev=self.args.rev,
|
||||
recursive=self.args.recursive,
|
||||
outs_only=self.args.outs_only,
|
||||
dvc_only=self.args.dvc_only,
|
||||
)
|
||||
if entries:
|
||||
entries = _prettify(entries, sys.stdout.isatty())
|
||||
logger.info("\n".join(entries))
|
||||
return 0
|
||||
except DvcException:
|
||||
logger.exception("failed to list '{}'".format(self.args.url))
|
||||
logger.exception(f"failed to list '{self.args.url}'")
|
||||
return 1
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Recursively list files.",
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"--outs-only", action="store_true", help="Show only DVC outputs."
|
||||
"--dvc-only", action="store_true", help="Show only DVC outputs."
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"--rev",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
|
||||
|
||||
class LsColors(object):
|
||||
class LsColors:
|
||||
default = "rs=0:di=01;34:ex=01;32"
|
||||
|
||||
def __init__(self, lscolors=None):
|
||||
@@ -45,4 +45,4 @@ class LsColors(object):
|
||||
if not val:
|
||||
return text
|
||||
rs = self._codes.get("rs", 0)
|
||||
return "\033[{}m{}\033[{}m".format(val, text, rs)
|
||||
return f"\033[{val}m{text}\033[{rs}m"
|
||||
|
||||
+43
-16
@@ -19,16 +19,16 @@ def show_metrics(
|
||||
|
||||
for branch, val in metrics.items():
|
||||
if all_branches or all_tags or all_commits:
|
||||
logger.info("{branch}:".format(branch=branch))
|
||||
logger.info(f"{branch}:")
|
||||
|
||||
for fname, metric in val.items():
|
||||
if not isinstance(metric, dict):
|
||||
logger.info("\t{}: {}".format(fname, str(metric)))
|
||||
continue
|
||||
|
||||
logger.info("\t{}:".format(fname))
|
||||
logger.info(f"\t{fname}:")
|
||||
for key, value in flatten(format_dict(metric), ".").items():
|
||||
logger.info("\t\t{}: {}".format(key, value))
|
||||
logger.info(f"\t\t{key}: {value}")
|
||||
|
||||
if missing:
|
||||
raise BadMetricError(missing)
|
||||
@@ -68,7 +68,7 @@ class CmdMetricsAdd(CmdBase):
|
||||
try:
|
||||
self.repo.metrics.add(self.args.path)
|
||||
except DvcException:
|
||||
msg = "failed to add metric file '{}'".format(self.args.path)
|
||||
msg = f"failed to add metric file '{self.args.path}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
|
||||
@@ -80,14 +80,14 @@ class CmdMetricsRemove(CmdBase):
|
||||
try:
|
||||
self.repo.metrics.remove(self.args.path)
|
||||
except DvcException:
|
||||
msg = "failed to remove metric file '{}'".format(self.args.path)
|
||||
msg = f"failed to remove metric file '{self.args.path}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _show_diff(diff):
|
||||
def _show_diff(diff, markdown=False, no_path=False, old=False):
|
||||
from collections import OrderedDict
|
||||
|
||||
from dvc.utils.diff import table
|
||||
@@ -96,16 +96,23 @@ def _show_diff(diff):
|
||||
for fname, mdiff in diff.items():
|
||||
sorted_mdiff = OrderedDict(sorted(mdiff.items()))
|
||||
for metric, change in sorted_mdiff.items():
|
||||
rows.append(
|
||||
[
|
||||
fname,
|
||||
metric,
|
||||
change["new"],
|
||||
change.get("diff", "diff not supported"),
|
||||
]
|
||||
)
|
||||
row = [] if no_path else [fname]
|
||||
row.append(metric)
|
||||
if old:
|
||||
row.append(change.get("old"))
|
||||
row.append(change["new"])
|
||||
row.append(change.get("diff", "diff not supported"))
|
||||
rows.append(row)
|
||||
|
||||
return table(["Path", "Metric", "Value", "Change"], rows)
|
||||
header = [] if no_path else ["Path"]
|
||||
header.append("Metric")
|
||||
if old:
|
||||
header.extend(["Old", "New"])
|
||||
else:
|
||||
header.append("Value")
|
||||
header.append("Change")
|
||||
|
||||
return table(header, rows, markdown)
|
||||
|
||||
|
||||
class CmdMetricsDiff(CmdBase):
|
||||
@@ -124,7 +131,9 @@ class CmdMetricsDiff(CmdBase):
|
||||
|
||||
logger.info(json.dumps(diff))
|
||||
else:
|
||||
table = _show_diff(diff)
|
||||
table = _show_diff(
|
||||
diff, self.args.show_md, self.args.no_path, self.args.old
|
||||
)
|
||||
if table:
|
||||
logger.info(table)
|
||||
|
||||
@@ -263,6 +272,24 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--show-md",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show tabulated output in the Markdown format (GFM).",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--no-path",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Don't show metric path.",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--old",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show old metric value.",
|
||||
)
|
||||
metrics_diff_parser.set_defaults(func=CmdMetricsDiff)
|
||||
|
||||
METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
|
||||
|
||||
+25
-4
@@ -8,16 +8,25 @@ from dvc.exceptions import DvcException
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _show_diff(diff):
|
||||
def _show_diff(diff, markdown=False, no_path=False):
|
||||
from dvc.utils.diff import table
|
||||
|
||||
rows = []
|
||||
for fname, pdiff in diff.items():
|
||||
sorted_pdiff = OrderedDict(sorted(pdiff.items()))
|
||||
for param, change in sorted_pdiff.items():
|
||||
rows.append([fname, param, change["old"], change["new"]])
|
||||
row = [] if no_path else [fname]
|
||||
row.append(param)
|
||||
row.append(change["old"])
|
||||
row.append(change["new"])
|
||||
rows.append(row)
|
||||
|
||||
return table(["Path", "Param", "Old", "New"], rows)
|
||||
header = [] if no_path else ["Path"]
|
||||
header.append("Param")
|
||||
header.append("Old")
|
||||
header.append("New")
|
||||
|
||||
return table(header, rows, markdown)
|
||||
|
||||
|
||||
class CmdParamsDiff(CmdBase):
|
||||
@@ -34,7 +43,7 @@ class CmdParamsDiff(CmdBase):
|
||||
|
||||
logger.info(json.dumps(diff))
|
||||
else:
|
||||
table = _show_diff(diff)
|
||||
table = _show_diff(diff, self.args.show_md, self.args.no_path)
|
||||
if table:
|
||||
logger.info(table)
|
||||
|
||||
@@ -94,4 +103,16 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
params_diff_parser.add_argument(
|
||||
"--show-md",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show tabulated output in the Markdown format (GFM).",
|
||||
)
|
||||
params_diff_parser.add_argument(
|
||||
"--no-path",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Don't show params path.",
|
||||
)
|
||||
params_diff_parser.set_defaults(func=CmdParamsDiff)
|
||||
|
||||
+33
-14
@@ -10,11 +10,10 @@ logger = logging.getLogger(__name__)
|
||||
class CmdPipelineShow(CmdBase):
|
||||
def _show(self, target, commands, outs, locked):
|
||||
import networkx
|
||||
from dvc import dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
stage = dvcfile.Dvcfile(self.repo, path).stages[name]
|
||||
stage = self.repo.get_stage(path, name)
|
||||
G = self.repo.graph
|
||||
stages = networkx.dfs_postorder_nodes(G, stage)
|
||||
if locked:
|
||||
@@ -31,14 +30,38 @@ class CmdPipelineShow(CmdBase):
|
||||
else:
|
||||
logger.info(stage.addressing)
|
||||
|
||||
@staticmethod
|
||||
def _build_output_graph(G, target_stage):
|
||||
import networkx
|
||||
from itertools import product
|
||||
|
||||
nodes = {str(out) for out in target_stage.outs}
|
||||
edges = []
|
||||
|
||||
for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
|
||||
from_stage_deps = {dep.path_info.parts for dep in from_stage.deps}
|
||||
to_outs = {
|
||||
to_out
|
||||
for to_out in to_stage.outs
|
||||
if to_out.path_info.parts in from_stage_deps
|
||||
}
|
||||
from_outs = {
|
||||
from_out
|
||||
for from_out in from_stage.outs
|
||||
if str(from_out) in nodes
|
||||
}
|
||||
nodes |= {str(to_out) for to_out in to_outs}
|
||||
for from_out, to_out in product(from_outs, to_outs):
|
||||
edges.append((str(from_out), str(to_out)))
|
||||
return nodes, edges
|
||||
|
||||
def _build_graph(self, target, commands=False, outs=False):
|
||||
import networkx
|
||||
from dvc import dvcfile
|
||||
from dvc.repo.graph import get_pipeline
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
|
||||
target_stage = self.repo.get_stage(path, name)
|
||||
G = get_pipeline(self.repo.pipelines, target_stage)
|
||||
|
||||
nodes = set()
|
||||
@@ -47,10 +70,7 @@ class CmdPipelineShow(CmdBase):
|
||||
if stage.cmd is None:
|
||||
continue
|
||||
nodes.add(stage.cmd)
|
||||
elif outs:
|
||||
for out in stage.outs:
|
||||
nodes.add(str(out))
|
||||
else:
|
||||
elif not outs:
|
||||
nodes.add(stage.addressing)
|
||||
|
||||
edges = []
|
||||
@@ -59,13 +79,12 @@ class CmdPipelineShow(CmdBase):
|
||||
if to_stage.cmd is None:
|
||||
continue
|
||||
edges.append((from_stage.cmd, to_stage.cmd))
|
||||
elif outs:
|
||||
for from_out in from_stage.outs:
|
||||
for to_out in to_stage.outs:
|
||||
edges.append((str(from_out), str(to_out)))
|
||||
else:
|
||||
elif not outs:
|
||||
edges.append((from_stage.addressing, to_stage.addressing))
|
||||
|
||||
if outs:
|
||||
nodes, edges = self._build_output_graph(G, target_stage)
|
||||
|
||||
return list(nodes), edges, networkx.is_tree(G)
|
||||
|
||||
def _show_ascii(self, target, commands, outs):
|
||||
@@ -143,7 +162,7 @@ class CmdPipelineShow(CmdBase):
|
||||
self.args.locked,
|
||||
)
|
||||
except DvcException:
|
||||
msg = "failed to show pipeline for '{}'".format(target)
|
||||
msg = f"failed to show pipeline for '{target}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
@@ -1,240 +0,0 @@
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.repo.plot.data import WORKSPACE_REVISION_NAME
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdPLot(CmdBase):
|
||||
def _revisions(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def _result_file(self):
|
||||
if self.args.file:
|
||||
return self.args.file
|
||||
|
||||
extension = self._result_extension()
|
||||
base = self._result_basename()
|
||||
|
||||
result_file = base + extension
|
||||
return result_file
|
||||
|
||||
def _result_basename(self):
|
||||
if self.args.datafile:
|
||||
return self.args.datafile
|
||||
return "plot"
|
||||
|
||||
def _result_extension(self):
|
||||
if not self.args.no_html:
|
||||
return ".html"
|
||||
elif self.args.template:
|
||||
return os.path.splitext(self.args.template)[-1]
|
||||
return ".json"
|
||||
|
||||
def run(self):
|
||||
fields = None
|
||||
jsonpath = None
|
||||
if self.args.select:
|
||||
if self.args.select.startswith("$"):
|
||||
jsonpath = self.args.select
|
||||
else:
|
||||
fields = set(self.args.select.split(","))
|
||||
try:
|
||||
plot_string = self.repo.plot(
|
||||
datafile=self.args.datafile,
|
||||
template=self.args.template,
|
||||
revisions=self._revisions(),
|
||||
fields=fields,
|
||||
x_field=self.args.x,
|
||||
y_field=self.args.y,
|
||||
path=jsonpath,
|
||||
embed=not self.args.no_html,
|
||||
csv_header=not self.args.no_csv_header,
|
||||
title=self.args.title,
|
||||
x_title=self.args.xlab,
|
||||
y_title=self.args.ylab,
|
||||
)
|
||||
|
||||
if self.args.stdout:
|
||||
logger.info(plot_string)
|
||||
else:
|
||||
result_path = self._result_file()
|
||||
with open(result_path, "w") as fobj:
|
||||
fobj.write(plot_string)
|
||||
|
||||
logger.info(
|
||||
"file://{}".format(
|
||||
os.path.join(self.repo.root_dir, result_path)
|
||||
)
|
||||
)
|
||||
|
||||
except DvcException:
|
||||
logger.exception("")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class CmdPlotShow(CmdPLot):
|
||||
def _revisions(self):
|
||||
return None
|
||||
|
||||
|
||||
class CmdPlotDiff(CmdPLot):
|
||||
def _revisions(self):
|
||||
revisions = self.args.revisions or []
|
||||
if len(revisions) <= 1:
|
||||
if len(revisions) == 0 and self.repo.scm.is_dirty():
|
||||
revisions.append("HEAD")
|
||||
revisions.append(WORKSPACE_REVISION_NAME)
|
||||
return revisions
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
PLOT_HELP = (
|
||||
"Generating plots for metrics stored in structured files "
|
||||
"(JSON, CSV, TSV)."
|
||||
)
|
||||
|
||||
plot_parser = subparsers.add_parser(
|
||||
"plot",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PLOT_HELP, "plot"),
|
||||
help=PLOT_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plot_subparsers = plot_parser.add_subparsers(
|
||||
dest="cmd",
|
||||
help="Use `dvc plot CMD --help` to display command-specific help.",
|
||||
)
|
||||
|
||||
fix_subparsers(plot_subparsers)
|
||||
|
||||
SHOW_HELP = "Generate a plot image file from a metrics file."
|
||||
plot_show_parser = plot_subparsers.add_parser(
|
||||
"show",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(SHOW_HELP, "plot/show"),
|
||||
help=SHOW_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plot_show_parser.add_argument(
|
||||
"-t",
|
||||
"--template",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help="File to be injected with data.",
|
||||
)
|
||||
plot_show_parser.add_argument(
|
||||
"-f", "--file", default=None, help="Name of the generated file."
|
||||
)
|
||||
plot_show_parser.add_argument(
|
||||
"-s",
|
||||
"--select",
|
||||
default=None,
|
||||
help="Choose which field(s) or JSONPath to include in the plot.",
|
||||
)
|
||||
plot_show_parser.add_argument(
|
||||
"-x", default=None, help="Field name for x axis."
|
||||
)
|
||||
plot_show_parser.add_argument(
|
||||
"-y", default=None, help="Field name for y axis."
|
||||
)
|
||||
plot_show_parser.add_argument(
|
||||
"--stdout",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print plot specification to stdout.",
|
||||
)
|
||||
plot_show_parser.add_argument(
|
||||
"--no-csv-header",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Required when CSV or TSV datafile does not have a header.",
|
||||
)
|
||||
plot_show_parser.add_argument(
|
||||
"--no-html",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Do not wrap Vega plot JSON with HTML.",
|
||||
)
|
||||
plot_show_parser.add_argument("--title", default=None, help="Plot title.")
|
||||
plot_show_parser.add_argument("--xlab", default=None, help="X axis title.")
|
||||
plot_show_parser.add_argument("--ylab", default=None, help="Y axis title.")
|
||||
plot_show_parser.add_argument(
|
||||
"datafile", nargs="?", default=None, help="Metrics file to visualize.",
|
||||
)
|
||||
plot_show_parser.set_defaults(func=CmdPlotShow)
|
||||
|
||||
PLOT_DIFF_HELP = (
|
||||
"Plot metrics differences between commits in the DVC "
|
||||
"repository, or between the last commit and the workspace."
|
||||
)
|
||||
plot_diff_parser = plot_subparsers.add_parser(
|
||||
"diff",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PLOT_DIFF_HELP, "plot/diff"),
|
||||
help=PLOT_DIFF_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plot_diff_parser.add_argument(
|
||||
"-t",
|
||||
"--template",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help="File to be injected with data.",
|
||||
)
|
||||
plot_diff_parser.add_argument(
|
||||
"-d",
|
||||
"--datafile",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help="Metrics file to visualize.",
|
||||
)
|
||||
plot_diff_parser.add_argument(
|
||||
"-f", "--file", default=None, help="Name of the generated file."
|
||||
)
|
||||
plot_diff_parser.add_argument(
|
||||
"-s",
|
||||
"--select",
|
||||
default=None,
|
||||
help="Choose which field(s) or JSONPath to include in the plot.",
|
||||
)
|
||||
plot_diff_parser.add_argument(
|
||||
"-x", default=None, help="Field name for x axis."
|
||||
)
|
||||
plot_diff_parser.add_argument(
|
||||
"-y", default=None, help="Field name for y axis."
|
||||
)
|
||||
plot_diff_parser.add_argument(
|
||||
"--stdout",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print plot specification to stdout.",
|
||||
)
|
||||
plot_diff_parser.add_argument(
|
||||
"--no-csv-header",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Provided CSV ot TSV datafile does not have a header.",
|
||||
)
|
||||
plot_diff_parser.add_argument(
|
||||
"--no-html",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Do not wrap Vega plot JSON with HTML.",
|
||||
)
|
||||
plot_diff_parser.add_argument("--title", default=None, help="Plot title.")
|
||||
plot_diff_parser.add_argument("--xlab", default=None, help="X axis title.")
|
||||
plot_diff_parser.add_argument("--ylab", default=None, help="Y axis title.")
|
||||
plot_diff_parser.add_argument(
|
||||
"revisions",
|
||||
nargs="*",
|
||||
default=None,
|
||||
help="Git revisions to plot from",
|
||||
)
|
||||
plot_diff_parser.set_defaults(func=CmdPlotDiff)
|
||||
@@ -0,0 +1,219 @@
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils import format_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PAGE_HTML = """<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>DVC Plot</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
|
||||
</head>
|
||||
<body>
|
||||
{divs}
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
DIV_HTML = """<div id = "{id}"></div>
|
||||
<script type = "text/javascript">
|
||||
var spec = {vega_json};
|
||||
vegaEmbed('#{id}', spec);
|
||||
</script>"""
|
||||
|
||||
|
||||
class CmdPlots(CmdBase):
|
||||
def _func(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
plots = self._func(
|
||||
targets=self.args.targets,
|
||||
template=self.args.template,
|
||||
x_field=self.args.x,
|
||||
y_field=self.args.y,
|
||||
csv_header=not self.args.no_csv_header,
|
||||
title=self.args.title,
|
||||
x_title=self.args.xlab,
|
||||
y_title=self.args.ylab,
|
||||
)
|
||||
|
||||
if self.args.show_json:
|
||||
import json
|
||||
|
||||
logger.info(json.dumps(plots))
|
||||
return 0
|
||||
|
||||
divs = [
|
||||
DIV_HTML.format(id=f"plot{i}", vega_json=plot)
|
||||
for i, plot in enumerate(plots.values())
|
||||
]
|
||||
html = PAGE_HTML.format(divs="\n".join(divs))
|
||||
path = self.args.out or "plots.html"
|
||||
|
||||
with open(path, "w") as fobj:
|
||||
fobj.write(html)
|
||||
|
||||
logger.info(
|
||||
"file://{}".format(os.path.join(self.repo.root_dir, path))
|
||||
)
|
||||
|
||||
except DvcException:
|
||||
logger.exception("")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class CmdPlotsShow(CmdPlots):
|
||||
def _func(self, *args, **kwargs):
|
||||
return self.repo.plots.show(*args, **kwargs)
|
||||
|
||||
|
||||
class CmdPlotsDiff(CmdPlots):
|
||||
def _func(self, *args, **kwargs):
|
||||
return self.repo.plots.diff(*args, revs=self.args.revisions, **kwargs)
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
PLOTS_HELP = (
|
||||
"Generating plots for metrics stored in structured files "
|
||||
"(JSON, CSV, TSV)."
|
||||
)
|
||||
|
||||
plots_parser = subparsers.add_parser(
|
||||
"plots",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PLOTS_HELP, "plots"),
|
||||
help=PLOTS_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_subparsers = plots_parser.add_subparsers(
|
||||
dest="cmd",
|
||||
help="Use `dvc plots CMD --help` to display command-specific help.",
|
||||
)
|
||||
|
||||
fix_subparsers(plots_subparsers)
|
||||
|
||||
SHOW_HELP = "Generate a plots image file from a metrics file."
|
||||
plots_show_parser = plots_subparsers.add_parser(
|
||||
"show",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(SHOW_HELP, "plots/show"),
|
||||
help=SHOW_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-t",
|
||||
"--template",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help=(
|
||||
"Special JSON or HTML schema file to inject with the data. "
|
||||
"See {}".format(
|
||||
format_link("https://man.dvc.org/plots#plot-templates")
|
||||
)
|
||||
),
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-o", "--out", default=None, help="Destination path to save plots to.",
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-x", default=None, help="Field name for x axis."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-y", default=None, help="Field name for y axis."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"--no-csv-header",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Required when CSV or TSV datafile does not have a header.",
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
plots_show_parser.add_argument("--title", default=None, help="Plot title.")
|
||||
plots_show_parser.add_argument(
|
||||
"--xlab", default=None, help="X axis title."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"--ylab", default=None, help="Y axis title."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="Metrics files to visualize. Shows all plots by default.",
|
||||
)
|
||||
plots_show_parser.set_defaults(func=CmdPlotsShow)
|
||||
|
||||
PLOTS_DIFF_HELP = (
|
||||
"Plot differences in metrics between commits in the DVC "
|
||||
"repository, or between the last commit and the workspace."
|
||||
)
|
||||
plots_diff_parser = plots_subparsers.add_parser(
|
||||
"diff",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PLOTS_DIFF_HELP, "plots/diff"),
|
||||
help=PLOTS_DIFF_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-t",
|
||||
"--template",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help=(
|
||||
"Special JSON or HTML schema file to inject with the data. "
|
||||
"See {}".format(
|
||||
format_link("https://man.dvc.org/plots#plot-templates")
|
||||
)
|
||||
),
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--targets",
|
||||
nargs="*",
|
||||
help="Metrics file to visualize. Shows all plots by default.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-o", "--out", default=None, help="Destination path to save plots to.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-x", default=None, help="Field name for x axis."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-y", default=None, help="Field name for y axis."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--no-csv-header",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Provided CSV ot TSV datafile does not have a header.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
plots_diff_parser.add_argument("--title", default=None, help="Plot title.")
|
||||
plots_diff_parser.add_argument(
|
||||
"--xlab", default=None, help="X axis title."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--ylab", default=None, help="Y axis title."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"revisions", nargs="*", default=None, help="Git commits to plot from",
|
||||
)
|
||||
plots_diff_parser.set_defaults(func=CmdPlotsDiff)
|
||||
+54
-7
@@ -18,17 +18,13 @@ class CmdRemote(CmdConfig):
|
||||
|
||||
def _check_exists(self, conf):
|
||||
if self.args.name not in conf["remote"]:
|
||||
raise ConfigError(
|
||||
"remote '{}' doesn't exists.".format(self.args.name)
|
||||
)
|
||||
raise ConfigError(f"remote '{self.args.name}' doesn't exists.")
|
||||
|
||||
|
||||
class CmdRemoteAdd(CmdRemote):
|
||||
def run(self):
|
||||
if self.args.default:
|
||||
logger.info(
|
||||
"Setting '{}' as a default remote.".format(self.args.name)
|
||||
)
|
||||
logger.info(f"Setting '{self.args.name}' as a default remote.")
|
||||
|
||||
with self.config.edit(self.args.level) as conf:
|
||||
if self.args.name in conf["remote"] and not self.args.force:
|
||||
@@ -85,7 +81,18 @@ class CmdRemoteDefault(CmdRemote):
|
||||
if self.args.unset:
|
||||
conf["core"].pop("remote", None)
|
||||
else:
|
||||
conf["core"]["remote"] = self.args.name
|
||||
merged_conf = self.config.load_config_to_level(
|
||||
self.args.level
|
||||
)
|
||||
if (
|
||||
self.args.name in conf["remote"]
|
||||
or self.args.name in merged_conf["remote"]
|
||||
):
|
||||
conf["core"]["remote"] = self.args.name
|
||||
else:
|
||||
raise ConfigError(
|
||||
"default remote must be present in remote list."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -97,6 +104,35 @@ class CmdRemoteList(CmdRemote):
|
||||
return 0
|
||||
|
||||
|
||||
class CmdRemoteRename(CmdRemote):
|
||||
def _rename_default(self, conf):
|
||||
if conf["core"].get("remote") == self.args.name:
|
||||
conf["core"]["remote"] = self.args.new
|
||||
|
||||
def run(self):
|
||||
all_config = self.config.load_config_to_level(None)
|
||||
if self.args.new in all_config.get("remote", {}):
|
||||
raise ConfigError(
|
||||
"Rename failed. Remote name '{}' already exists.".format(
|
||||
{self.args.new}
|
||||
)
|
||||
)
|
||||
|
||||
with self.config.edit(self.args.level) as conf:
|
||||
self._check_exists(conf)
|
||||
conf["remote"][self.args.new] = conf["remote"][self.args.name]
|
||||
del conf["remote"][self.args.name]
|
||||
self._rename_default(conf)
|
||||
|
||||
for level in reversed(self.config.LEVELS):
|
||||
if level == self.args.level:
|
||||
break
|
||||
with self.config.edit(level) as level_conf:
|
||||
self._rename_default(level_conf)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
from dvc.command.config import parent_config_parser
|
||||
|
||||
@@ -213,3 +249,14 @@ def add_parser(subparsers, parent_parser):
|
||||
"name", help="Name of the remote to remove."
|
||||
)
|
||||
remote_remove_parser.set_defaults(func=CmdRemoteRemove)
|
||||
REMOTE_RENAME_HELP = "Rename a DVC remote"
|
||||
remote_rename_parser = remote_subparsers.add_parser(
|
||||
"rename",
|
||||
parents=[parent_config_parser, parent_parser],
|
||||
description=append_doc_link(REMOTE_RENAME_HELP, "remote/rename"),
|
||||
help=REMOTE_RENAME_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
remote_rename_parser.add_argument("name", help="Remote to be renamed")
|
||||
remote_rename_parser.add_argument("new", help="New name of the remote")
|
||||
remote_rename_parser.set_defaults(func=CmdRemoteRename)
|
||||
|
||||
@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdRemove(CmdBase):
|
||||
def _is_outs_only(self, target):
|
||||
def _is_dvc_only(self, target):
|
||||
if not self.args.purge:
|
||||
return True
|
||||
|
||||
@@ -31,10 +31,10 @@ class CmdRemove(CmdBase):
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
outs_only = self._is_outs_only(target)
|
||||
self.repo.remove(target, outs_only=outs_only)
|
||||
dvc_only = self._is_dvc_only(target)
|
||||
self.repo.remove(target, dvc_only=dvc_only)
|
||||
except DvcException:
|
||||
logger.exception("failed to remove '{}'".format(target))
|
||||
logger.exception(f"failed to remove '{target}'")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
+14
-3
@@ -5,6 +5,7 @@ import os
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.command.metrics import show_metrics
|
||||
from dvc.command.status import CmdDataStatus
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -33,10 +34,11 @@ class CmdRepro(CmdBase):
|
||||
interactive=self.args.interactive,
|
||||
pipeline=self.args.pipeline,
|
||||
all_pipelines=self.args.all_pipelines,
|
||||
ignore_build_cache=self.args.ignore_build_cache,
|
||||
run_cache=not self.args.no_run_cache,
|
||||
no_commit=self.args.no_commit,
|
||||
downstream=self.args.downstream,
|
||||
recursive=self.args.recursive,
|
||||
force_downstream=self.args.force_downstream,
|
||||
)
|
||||
|
||||
if len(stages) == 0:
|
||||
@@ -68,7 +70,7 @@ def add_parser(subparsers, parent_parser):
|
||||
repro_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="DVC-file to reproduce. 'Dvcfile' by default.",
|
||||
help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"-f",
|
||||
@@ -136,7 +138,16 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Reproduce all stages in the specified directory.",
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"--ignore-build-cache",
|
||||
"--no-run-cache",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Execute stage commands even if they have already been run with "
|
||||
"the same command/dependencies/outputs/etc before."
|
||||
),
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"--force-downstream",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Reproduce all descendants of a changed stage even if their "
|
||||
|
||||
+27
-7
@@ -16,6 +16,8 @@ class CmdRun(CmdBase):
|
||||
self.args.outs_no_cache,
|
||||
self.args.metrics,
|
||||
self.args.metrics_no_cache,
|
||||
self.args.plots,
|
||||
self.args.plots_no_cache,
|
||||
self.args.outs_persist,
|
||||
self.args.outs_persist_no_cache,
|
||||
self.args.params,
|
||||
@@ -24,8 +26,8 @@ class CmdRun(CmdBase):
|
||||
): # pragma: no cover
|
||||
logger.error(
|
||||
"too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
|
||||
"`-m`, `-M`, `-p`, `--outs-persist`, "
|
||||
"`--outs-persist-no-cache`, `command`."
|
||||
"`-m`, `-M`, `-p`, `--plots`, `--plots-no-cache`, "
|
||||
"`--outs-persist`, `--outs-persist-no-cache`, `command`."
|
||||
)
|
||||
return 1
|
||||
|
||||
@@ -36,13 +38,15 @@ class CmdRun(CmdBase):
|
||||
outs_no_cache=self.args.outs_no_cache,
|
||||
metrics=self.args.metrics,
|
||||
metrics_no_cache=self.args.metrics_no_cache,
|
||||
plots=self.args.plots,
|
||||
plots_no_cache=self.args.plots_no_cache,
|
||||
deps=self.args.deps,
|
||||
params=self.args.params,
|
||||
fname=self.args.file,
|
||||
wdir=self.args.wdir,
|
||||
no_exec=self.args.no_exec,
|
||||
overwrite=self.args.overwrite_dvcfile,
|
||||
ignore_build_cache=self.args.ignore_build_cache,
|
||||
run_cache=not self.args.no_run_cache,
|
||||
no_commit=self.args.no_commit,
|
||||
outs_persist=self.args.outs_persist,
|
||||
outs_persist_no_cache=self.args.outs_persist_no_cache,
|
||||
@@ -75,7 +79,7 @@ class CmdRun(CmdBase):
|
||||
if " " not in argument or '"' in argument:
|
||||
return argument
|
||||
|
||||
return '"{}"'.format(argument)
|
||||
return f'"{argument}"'
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
@@ -140,6 +144,20 @@ def add_parser(subparsers, parent_parser):
|
||||
"(do not put into DVC cache).",
|
||||
metavar="<path>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--plots",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare output plot file.",
|
||||
metavar="<path>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--plots-no-cache",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare output plot file (do not put into DVC cache).",
|
||||
metavar="<path>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"-f",
|
||||
"--file",
|
||||
@@ -165,11 +183,13 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Overwrite existing DVC-file without asking for confirmation.",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--ignore-build-cache",
|
||||
"--no-run-cache",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Run this stage even if it has been already ran with the same "
|
||||
"command/dependencies/outputs/etc before.",
|
||||
help=(
|
||||
"Execute the command even if this stage has already been run "
|
||||
"with the same command/dependencies/outputs/etc before."
|
||||
),
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--no-commit",
|
||||
|
||||
@@ -19,7 +19,7 @@ class CmdDataStatus(CmdDataBase):
|
||||
ind = indent * self.STATUS_INDENT
|
||||
|
||||
if isinstance(status, str):
|
||||
logger.info("{}{}".format(ind, status))
|
||||
logger.info(f"{ind}{status}")
|
||||
return
|
||||
|
||||
if isinstance(status, list):
|
||||
@@ -33,7 +33,7 @@ class CmdDataStatus(CmdDataBase):
|
||||
if isinstance(value, str):
|
||||
logger.info("{}{}{}".format(ind, self._normalize(value), key))
|
||||
elif value:
|
||||
logger.info("{}{}:".format(ind, key))
|
||||
logger.info(f"{ind}{key}:")
|
||||
self._show(value, indent + 1)
|
||||
|
||||
def run(self):
|
||||
|
||||
@@ -13,7 +13,7 @@ class CmdUnprotect(CmdBase):
|
||||
try:
|
||||
self.repo.unprotect(target)
|
||||
except DvcException:
|
||||
msg = "failed to unprotect '{}'".format(target)
|
||||
msg = f"failed to unprotect '{target}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
+16
-6
@@ -10,12 +10,15 @@ logger = logging.getLogger(__name__)
|
||||
class CmdUpdate(CmdBase):
|
||||
def run(self):
|
||||
ret = 0
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
self.repo.update(target, self.args.rev)
|
||||
except DvcException:
|
||||
logger.exception("failed to update '{}'.".format(target))
|
||||
ret = 1
|
||||
try:
|
||||
self.repo.update(
|
||||
targets=self.args.targets,
|
||||
rev=self.args.rev,
|
||||
recursive=self.args.recursive,
|
||||
)
|
||||
except DvcException:
|
||||
logger.exception("failed update data")
|
||||
ret = 1
|
||||
return ret
|
||||
|
||||
|
||||
@@ -37,4 +40,11 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Git revision (e.g. SHA, branch, tag)",
|
||||
metavar="<commit>",
|
||||
)
|
||||
update_parser.add_argument(
|
||||
"-R",
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Update all stages in the specified directory.",
|
||||
)
|
||||
update_parser.set_defaults(func=CmdUpdate)
|
||||
|
||||
@@ -28,12 +28,12 @@ class CmdVersion(CmdBaseNoRepo):
|
||||
from dvc.repo import Repo
|
||||
|
||||
info = [
|
||||
"DVC version: {}".format(__version__),
|
||||
"Python version: {}".format(platform.python_version()),
|
||||
"Platform: {}".format(platform.platform()),
|
||||
"Binary: {}".format(is_binary()),
|
||||
"Package: {}".format(PKG),
|
||||
"Supported remotes: {}".format(self.get_supported_remotes()),
|
||||
f"DVC version: {__version__}",
|
||||
f"Python version: {platform.python_version()}",
|
||||
f"Platform: {platform.platform()}",
|
||||
f"Binary: {is_binary()}",
|
||||
f"Package: {PKG}",
|
||||
f"Supported remotes: {self.get_supported_remotes()}",
|
||||
]
|
||||
|
||||
try:
|
||||
@@ -51,7 +51,7 @@ class CmdVersion(CmdBaseNoRepo):
|
||||
if psutil:
|
||||
fs_type = self.get_fs_type(repo.cache.local.cache_dir)
|
||||
info.append(
|
||||
"Filesystem type (cache directory): {}".format(fs_type)
|
||||
f"Filesystem type (cache directory): {fs_type}"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
@@ -72,7 +72,7 @@ class CmdVersion(CmdBaseNoRepo):
|
||||
|
||||
if psutil:
|
||||
fs_root = self.get_fs_type(os.path.abspath(root_directory))
|
||||
info.append("Filesystem type (workspace): {}".format(fs_root))
|
||||
info.append(f"Filesystem type (workspace): {fs_root}")
|
||||
|
||||
logger.info("\n".join(info))
|
||||
return 0
|
||||
@@ -115,7 +115,7 @@ class CmdVersion(CmdBaseNoRepo):
|
||||
os.unlink(dst)
|
||||
except DvcException:
|
||||
status = "not supported"
|
||||
cache.append("{name} - {status}".format(name=name, status=status))
|
||||
cache.append(f"{name} - {status}")
|
||||
os.remove(src)
|
||||
|
||||
return ", ".join(cache)
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
"""Python version compatibility code"""
|
||||
|
||||
# Backport os.fspath() from Python 3.6
|
||||
try:
|
||||
from os import fspath # noqa: F821
|
||||
|
||||
fspath_py35 = lambda s: s # noqa: E731
|
||||
except ImportError:
|
||||
|
||||
def fspath(path):
|
||||
"""Return the path representation of a path-like object.
|
||||
|
||||
If str or bytes is passed in, it is returned unchanged. Otherwise the
|
||||
os.PathLike interface is used to get the path representation. If the
|
||||
path representation is not str or bytes, TypeError is raised. If the
|
||||
provided path is not str, bytes, or os.PathLike, TypeError is raised.
|
||||
"""
|
||||
if isinstance(path, (str, bytes)):
|
||||
return path
|
||||
|
||||
# Work from the object's type to match method resolution of other magic
|
||||
# methods.
|
||||
path_type = type(path)
|
||||
try:
|
||||
path_repr = path_type.__fspath__(path)
|
||||
except AttributeError:
|
||||
if hasattr(path_type, "__fspath__"):
|
||||
raise
|
||||
else:
|
||||
raise TypeError(
|
||||
"expected str, bytes or os.PathLike object, "
|
||||
"not " + path_type.__name__
|
||||
)
|
||||
if isinstance(path_repr, (str, bytes)):
|
||||
return path_repr
|
||||
else:
|
||||
raise TypeError(
|
||||
"expected {}.__fspath__() to return str or bytes, "
|
||||
"not {}".format(path_type.__name__, type(path_repr).__name__)
|
||||
)
|
||||
|
||||
fspath_py35 = fspath
|
||||
+8
-7
@@ -31,7 +31,7 @@ class ConfigError(DvcException):
|
||||
"""DVC config exception."""
|
||||
|
||||
def __init__(self, msg):
|
||||
super().__init__("config file error: {}".format(msg))
|
||||
super().__init__(f"config file error: {msg}")
|
||||
|
||||
|
||||
class NoRemoteError(ConfigError):
|
||||
@@ -89,7 +89,7 @@ def ByUrl(mapping):
|
||||
if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
|
||||
return schemas[""](data)
|
||||
if parsed.scheme not in schemas:
|
||||
raise Invalid("Unsupported URL type {}://".format(parsed.scheme))
|
||||
raise Invalid(f"Unsupported URL type {parsed.scheme}://")
|
||||
|
||||
return schemas[parsed.scheme](data)
|
||||
|
||||
@@ -151,6 +151,7 @@ SCHEMA = {
|
||||
Optional("listobjects", default=False): Bool,
|
||||
Optional("use_ssl", default=True): Bool,
|
||||
"sse": str,
|
||||
"sse_kms_key_id": str,
|
||||
"acl": str,
|
||||
"grant_read": str,
|
||||
"grant_read_acp": str,
|
||||
@@ -289,7 +290,7 @@ class Config(dict):
|
||||
Raises:
|
||||
ConfigError: thrown if config has an invalid format.
|
||||
"""
|
||||
conf = self._load_config_to_level()
|
||||
conf = self.load_config_to_level()
|
||||
|
||||
if validate:
|
||||
conf = self.validate(conf)
|
||||
@@ -349,7 +350,7 @@ class Config(dict):
|
||||
}
|
||||
return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
|
||||
|
||||
def _load_config_to_level(self, level=None):
|
||||
def load_config_to_level(self, level=None):
|
||||
merged_conf = {}
|
||||
for merge_level in self.LEVELS:
|
||||
if merge_level == level:
|
||||
@@ -368,7 +369,7 @@ class Config(dict):
|
||||
|
||||
conf = self._save_paths(conf, self.files[level])
|
||||
|
||||
merged_conf = self._load_config_to_level(level)
|
||||
merged_conf = self.load_config_to_level(level)
|
||||
_merge(merged_conf, conf)
|
||||
self.validate(merged_conf)
|
||||
|
||||
@@ -389,7 +390,7 @@ def _load_config(filename):
|
||||
|
||||
|
||||
def _save_config(filename, conf_dict):
|
||||
logger.debug("Writing '{}'.".format(filename))
|
||||
logger.debug(f"Writing '{filename}'.")
|
||||
os.makedirs(os.path.dirname(filename), exist_ok=True)
|
||||
|
||||
config = configobj.ConfigObj(_pack_remotes(conf_dict))
|
||||
@@ -416,7 +417,7 @@ def _pack_remotes(conf):
|
||||
|
||||
# Transform remote.name -> 'remote "name"'
|
||||
for name, val in conf["remote"].items():
|
||||
result['remote "{}"'.format(name)] = val
|
||||
result[f'remote "{name}"'] = val
|
||||
result.pop("remote", None)
|
||||
|
||||
return result
|
||||
|
||||
+2
-2
@@ -66,7 +66,7 @@ def _spawn_posix(cmd, env):
|
||||
|
||||
|
||||
def _spawn(cmd, env):
|
||||
logger.debug("Trying to spawn '{}'".format(cmd))
|
||||
logger.debug(f"Trying to spawn '{cmd}'")
|
||||
|
||||
if os.name == "nt":
|
||||
_spawn_windows(cmd, env)
|
||||
@@ -75,7 +75,7 @@ def _spawn(cmd, env):
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug("Spawned '{}'".format(cmd))
|
||||
logger.debug(f"Spawned '{cmd}'")
|
||||
|
||||
|
||||
def daemon(args):
|
||||
|
||||
+7
-7
@@ -37,7 +37,7 @@ def find_pager():
|
||||
if env_pager:
|
||||
return make_pager(env_pager)
|
||||
|
||||
if os.system("({}) 2>{}".format(DEFAULT_PAGER, os.devnull)) == 0:
|
||||
if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
|
||||
return make_pager(DEFAULT_PAGER_FORMATTED)
|
||||
|
||||
logger.warning(
|
||||
@@ -49,7 +49,7 @@ def find_pager():
|
||||
return pydoc.plainpager
|
||||
|
||||
|
||||
class VertexViewer(object):
|
||||
class VertexViewer:
|
||||
"""Class to define vertex box boundaries that will be accounted for during
|
||||
graph building by grandalf.
|
||||
|
||||
@@ -75,7 +75,7 @@ class VertexViewer(object):
|
||||
return self._w
|
||||
|
||||
|
||||
class AsciiCanvas(object):
|
||||
class AsciiCanvas:
|
||||
"""Class for drawing in ASCII.
|
||||
|
||||
Args:
|
||||
@@ -145,21 +145,21 @@ class AsciiCanvas(object):
|
||||
if dx == 0:
|
||||
y = y0
|
||||
else:
|
||||
y = y0 + int(round((x - x0) * dy / float((dx))))
|
||||
y = y0 + int(round((x - x0) * dy / float(dx)))
|
||||
self.point(x, y, char)
|
||||
elif y0 < y1:
|
||||
for y in range(y0, y1 + 1):
|
||||
if dy == 0:
|
||||
x = x0
|
||||
else:
|
||||
x = x0 + int(round((y - y0) * dx / float((dy))))
|
||||
x = x0 + int(round((y - y0) * dx / float(dy)))
|
||||
self.point(x, y, char)
|
||||
else:
|
||||
for y in range(y1, y0 + 1):
|
||||
if dy == 0:
|
||||
x = x0
|
||||
else:
|
||||
x = x1 + int(round((y - y1) * dx / float((dy))))
|
||||
x = x1 + int(round((y - y1) * dx / float(dy)))
|
||||
self.point(x, y, char)
|
||||
|
||||
def text(self, x, y, text):
|
||||
@@ -213,7 +213,7 @@ def _build_sugiyama_layout(vertexes, edges):
|
||||
# Y
|
||||
#
|
||||
|
||||
vertexes = {v: Vertex(" {} ".format(v)) for v in vertexes}
|
||||
vertexes = {v: Vertex(f" {v} ") for v in vertexes}
|
||||
# NOTE: reverting edges to correctly orientate the graph
|
||||
edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
|
||||
vertexes = vertexes.values()
|
||||
|
||||
+3
-19
@@ -8,7 +8,7 @@ from dvc.remote import Remote
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DataCloud(object):
|
||||
class DataCloud:
|
||||
"""Class that manages dvc remotes.
|
||||
|
||||
Args:
|
||||
@@ -48,12 +48,7 @@ class DataCloud(object):
|
||||
return Remote(self.repo, name=name)
|
||||
|
||||
def push(
|
||||
self,
|
||||
cache,
|
||||
jobs=None,
|
||||
remote=None,
|
||||
show_checksums=False,
|
||||
run_cache=False,
|
||||
self, cache, jobs=None, remote=None, show_checksums=False,
|
||||
):
|
||||
"""Push data items in a cloud-agnostic way.
|
||||
|
||||
@@ -67,20 +62,12 @@ class DataCloud(object):
|
||||
"""
|
||||
remote = self.get_remote(remote, "push")
|
||||
|
||||
if run_cache:
|
||||
self.repo.stage_cache.push(remote)
|
||||
|
||||
return self.repo.cache.local.push(
|
||||
cache, jobs=jobs, remote=remote, show_checksums=show_checksums,
|
||||
)
|
||||
|
||||
def pull(
|
||||
self,
|
||||
cache,
|
||||
jobs=None,
|
||||
remote=None,
|
||||
show_checksums=False,
|
||||
run_cache=False,
|
||||
self, cache, jobs=None, remote=None, show_checksums=False,
|
||||
):
|
||||
"""Pull data items in a cloud-agnostic way.
|
||||
|
||||
@@ -94,9 +81,6 @@ class DataCloud(object):
|
||||
"""
|
||||
remote = self.get_remote(remote, "pull")
|
||||
|
||||
if run_cache:
|
||||
self.repo.stage_cache.pull(remote)
|
||||
|
||||
downloaded_items_num = self.repo.cache.local.pull(
|
||||
cache, jobs=jobs, remote=remote, show_checksums=show_checksums
|
||||
)
|
||||
|
||||
@@ -2,6 +2,7 @@ from collections import defaultdict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import dvc.output as output
|
||||
from dvc.dependency.azure import AzureDependency
|
||||
from dvc.dependency.gs import GSDependency
|
||||
from dvc.dependency.hdfs import HDFSDependency
|
||||
from dvc.dependency.http import HTTPDependency
|
||||
@@ -17,6 +18,7 @@ from dvc.scheme import Schemes
|
||||
from .repo import RepoDependency
|
||||
|
||||
DEPS = [
|
||||
AzureDependency,
|
||||
GSDependency,
|
||||
HDFSDependency,
|
||||
HTTPDependency,
|
||||
@@ -30,6 +32,7 @@ DEP_MAP = {
|
||||
Schemes.LOCAL: LocalDependency,
|
||||
Schemes.SSH: SSHDependency,
|
||||
Schemes.S3: S3Dependency,
|
||||
Schemes.AZURE: AzureDependency,
|
||||
Schemes.GS: GSDependency,
|
||||
Schemes.HDFS: HDFSDependency,
|
||||
Schemes.HTTP: HTTPDependency,
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.azure import AzureRemote
|
||||
|
||||
|
||||
class AzureDependency(BaseDependency, BaseOutput):
|
||||
REMOTE = AzureRemote
|
||||
@@ -3,24 +3,22 @@ from dvc.exceptions import DvcException
|
||||
|
||||
class DependencyDoesNotExistError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = "dependency '{}' does not exist".format(path)
|
||||
msg = f"dependency '{path}' does not exist"
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class DependencyIsNotFileOrDirError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = "dependency '{}' is not a file or directory".format(path)
|
||||
msg = f"dependency '{path}' is not a file or directory"
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class DependencyIsStageFileError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__(
|
||||
"Stage file '{}' cannot be a dependency.".format(path)
|
||||
)
|
||||
super().__init__(f"Stage file '{path}' cannot be a dependency.")
|
||||
|
||||
|
||||
class BaseDependency(object):
|
||||
class BaseDependency:
|
||||
IS_DEPENDENCY = True
|
||||
|
||||
DoesNotExistError = DependencyDoesNotExistError
|
||||
|
||||
@@ -5,7 +5,6 @@ import dpath.util
|
||||
import yaml
|
||||
from voluptuous import Any
|
||||
|
||||
from dvc.compat import fspath_py35
|
||||
from dvc.dependency.local import LocalDependency
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -87,12 +86,12 @@ class ParamsDependency(LocalDependency):
|
||||
if not self.exists:
|
||||
return {}
|
||||
|
||||
with self.repo.tree.open(fspath_py35(self.path_info), "r") as fobj:
|
||||
with self.repo.tree.open(self.path_info, "r") as fobj:
|
||||
try:
|
||||
config = yaml.safe_load(fobj)
|
||||
except yaml.YAMLError as exc:
|
||||
raise BadParamFileError(
|
||||
"Unable to read parameters from '{}'".format(self)
|
||||
f"Unable to read parameters from '{self}'"
|
||||
) from exc
|
||||
|
||||
ret = {}
|
||||
|
||||
+18
-7
@@ -50,13 +50,24 @@ class RepoDependency(LocalDependency):
|
||||
return external_repo(d["url"], rev=rev)
|
||||
|
||||
def _get_checksum(self, locked=True):
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
with self._make_repo(locked=locked) as repo:
|
||||
try:
|
||||
return repo.find_out_by_relpath(self.def_path).info["md5"]
|
||||
except OutputNotFoundError:
|
||||
path = PathInfo(os.path.join(repo.root_dir, self.def_path))
|
||||
|
||||
# we want stream but not fetch, so DVC out directories are
|
||||
# walked, but dir contents is not fetched
|
||||
tree = RepoTree(repo, stream=True)
|
||||
|
||||
# We are polluting our repo cache with some dir listing here
|
||||
return self.repo.cache.local.get_checksum(path)
|
||||
if tree.isdir(path):
|
||||
return self.repo.cache.local.get_dir_checksum(
|
||||
path, tree=tree
|
||||
)
|
||||
return tree.get_file_checksum(path)
|
||||
|
||||
def status(self):
|
||||
current_checksum = self._get_checksum(locked=True)
|
||||
@@ -76,16 +87,16 @@ class RepoDependency(LocalDependency):
|
||||
def download(self, to):
|
||||
with self._make_repo() as repo:
|
||||
if self.def_repo.get(self.PARAM_REV_LOCK) is None:
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
|
||||
|
||||
if hasattr(repo, "cache"):
|
||||
repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
|
||||
|
||||
repo.pull_to(self.def_path, to.path_info)
|
||||
cache = self.repo.cache.local
|
||||
with repo.use_cache(cache):
|
||||
_, _, cache_infos = repo.fetch_external([self.def_path])
|
||||
cache.checkout(to.path_info, cache_infos[0])
|
||||
|
||||
def update(self, rev=None):
|
||||
if rev:
|
||||
self.def_repo[self.PARAM_REV] = rev
|
||||
|
||||
with self._make_repo(locked=False) as repo:
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
|
||||
|
||||
+27
-12
@@ -32,8 +32,7 @@ PIPELINE_LOCK = "dvc.lock"
|
||||
|
||||
|
||||
class LockfileCorruptedError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__("Lockfile '{}' is corrupted.".format(path))
|
||||
pass
|
||||
|
||||
|
||||
def is_valid_filename(path):
|
||||
@@ -80,7 +79,7 @@ class FileMixin:
|
||||
) == os.path.abspath(other.path)
|
||||
|
||||
def __str__(self):
|
||||
return "{}: {}".format(self.__class__.__name__, self.relpath)
|
||||
return f"{self.__class__.__name__}: {self.relpath}"
|
||||
|
||||
@property
|
||||
def relpath(self):
|
||||
@@ -103,7 +102,7 @@ class FileMixin:
|
||||
with self.repo.tree.open(self.path) as fd:
|
||||
stage_text = fd.read()
|
||||
d = parse_stage(stage_text, self.path)
|
||||
self.validate(d, self.path)
|
||||
self.validate(d, self.relpath)
|
||||
return d, stage_text
|
||||
|
||||
@classmethod
|
||||
@@ -112,7 +111,7 @@ class FileMixin:
|
||||
try:
|
||||
cls.SCHEMA(d)
|
||||
except MultipleInvalid as exc:
|
||||
raise StageFileFormatError(fname, exc)
|
||||
raise StageFileFormatError(f"'{fname}' format error: {exc}")
|
||||
|
||||
def remove_with_prompt(self, force=False):
|
||||
raise NotImplementedError
|
||||
@@ -176,25 +175,29 @@ class PipelineFile(FileMixin):
|
||||
def _lockfile(self):
|
||||
return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
|
||||
|
||||
def dump(self, stage, update_pipeline=False, **kwargs):
|
||||
def dump(self, stage, update_pipeline=False, no_lock=False):
|
||||
"""Dumps given stage appropriately in the dvcfile."""
|
||||
from dvc.stage import PipelineStage
|
||||
|
||||
assert isinstance(stage, PipelineStage)
|
||||
check_dvc_filename(self.path)
|
||||
self._dump_lockfile(stage)
|
||||
|
||||
if update_pipeline and not stage.is_data_source:
|
||||
self._dump_pipeline_file(stage)
|
||||
|
||||
if not no_lock:
|
||||
self._dump_lockfile(stage)
|
||||
|
||||
def _dump_lockfile(self, stage):
|
||||
self._lockfile.dump(stage)
|
||||
|
||||
def _dump_pipeline_file(self, stage):
|
||||
data = {}
|
||||
if self.exists():
|
||||
with open(self.path, "r") as fd:
|
||||
with open(self.path) as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
else:
|
||||
logger.info("Creating '%s'", self.relpath)
|
||||
open(self.path, "w+").close()
|
||||
|
||||
data["stages"] = data.get("stages", {})
|
||||
@@ -205,6 +208,9 @@ class PipelineFile(FileMixin):
|
||||
else:
|
||||
data["stages"].update(stage_data)
|
||||
|
||||
logger.info(
|
||||
"Adding stage '%s' to '%s'", stage.name, self.relpath,
|
||||
)
|
||||
dump_stage_file(self.path, data)
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
|
||||
@@ -238,23 +244,32 @@ class Lockfile(FileMixin):
|
||||
with self.repo.tree.open(self.path) as fd:
|
||||
data = parse_stage(fd.read(), self.path)
|
||||
try:
|
||||
self.validate(data, fname=self.path)
|
||||
self.validate(data, fname=self.relpath)
|
||||
except StageFileFormatError:
|
||||
raise LockfileCorruptedError(self.path)
|
||||
raise LockfileCorruptedError(
|
||||
f"Lockfile '{self.relpath}' is corrupted."
|
||||
)
|
||||
return data
|
||||
|
||||
def dump(self, stage, **kwargs):
|
||||
stage_data = serialize.to_lockfile(stage)
|
||||
if not self.exists():
|
||||
modified = True
|
||||
logger.info("Generating lock file '%s'", self.relpath)
|
||||
data = stage_data
|
||||
open(self.path, "w+").close()
|
||||
else:
|
||||
with self.repo.tree.open(self.path, "r") as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
modified = data.get(stage.name, {}) != stage_data.get(
|
||||
stage.name, {}
|
||||
)
|
||||
if modified:
|
||||
logger.info("Updating lock file '%s'", self.relpath)
|
||||
data.update(stage_data)
|
||||
|
||||
dump_stage_file(self.path, data)
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
if modified:
|
||||
self.repo.scm.track_file(self.relpath)
|
||||
|
||||
|
||||
class Dvcfile:
|
||||
|
||||
+26
-25
@@ -34,8 +34,7 @@ class OutputDuplicationError(DvcException):
|
||||
)
|
||||
else:
|
||||
msg = "output '{}' is already specified in stages:\n{}".format(
|
||||
output,
|
||||
"\n".join("\t- {}".format(s.addressing) for s in stages),
|
||||
output, "\n".join(f"\t- {s.addressing}" for s in stages),
|
||||
)
|
||||
super().__init__(msg)
|
||||
self.stages = stages
|
||||
@@ -102,7 +101,7 @@ class ArgumentDuplicationError(DvcException):
|
||||
|
||||
def __init__(self, path):
|
||||
assert isinstance(path, str)
|
||||
super().__init__("file '{}' is specified more than once.".format(path))
|
||||
super().__init__(f"file '{path}' is specified more than once.")
|
||||
|
||||
|
||||
class MoveNotDataSourceError(DvcException):
|
||||
@@ -158,7 +157,7 @@ class InitError(DvcException):
|
||||
class ReproductionError(DvcException):
|
||||
def __init__(self, dvc_file_name):
|
||||
self.path = dvc_file_name
|
||||
super().__init__("failed to reproduce '{}'".format(dvc_file_name))
|
||||
super().__init__(f"failed to reproduce '{dvc_file_name}'")
|
||||
|
||||
|
||||
class BadMetricError(DvcException):
|
||||
@@ -166,7 +165,7 @@ class BadMetricError(DvcException):
|
||||
super().__init__(
|
||||
"the following metrics do not exist, "
|
||||
"are not metric files or are malformed: {paths}".format(
|
||||
paths=", ".join("'{}'".format(path) for path in paths)
|
||||
paths=", ".join(f"'{path}'" for path in paths)
|
||||
)
|
||||
)
|
||||
|
||||
@@ -204,7 +203,7 @@ class OverlappingOutputPathsError(DvcException):
|
||||
|
||||
class CheckoutErrorSuggestGit(DvcException):
|
||||
def __init__(self, target):
|
||||
super().__init__("Did you mean `git checkout {}`?".format(target))
|
||||
super().__init__(f"Did you mean `git checkout {target}`?")
|
||||
|
||||
|
||||
class ETagMismatchError(DvcException):
|
||||
@@ -218,9 +217,7 @@ class ETagMismatchError(DvcException):
|
||||
class FileMissingError(DvcException):
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(
|
||||
"Can't find '{}' neither locally nor on remote".format(path)
|
||||
)
|
||||
super().__init__(f"Can't find '{path}' neither locally nor on remote")
|
||||
|
||||
|
||||
class DvcIgnoreInCollectedDirError(DvcException):
|
||||
@@ -245,18 +242,14 @@ class DownloadError(DvcException):
|
||||
def __init__(self, amount):
|
||||
self.amount = amount
|
||||
|
||||
super().__init__(
|
||||
"{amount} files failed to download".format(amount=amount)
|
||||
)
|
||||
super().__init__(f"{amount} files failed to download")
|
||||
|
||||
|
||||
class UploadError(DvcException):
|
||||
def __init__(self, amount):
|
||||
self.amount = amount
|
||||
|
||||
super().__init__(
|
||||
"{amount} files failed to upload".format(amount=amount)
|
||||
)
|
||||
super().__init__(f"{amount} files failed to upload")
|
||||
|
||||
|
||||
class CheckoutError(DvcException):
|
||||
@@ -278,7 +271,7 @@ class CollectCacheError(DvcException):
|
||||
class NoRemoteInExternalRepoError(DvcException):
|
||||
def __init__(self, url):
|
||||
super().__init__(
|
||||
"No DVC remote is specified in target repository '{}'.".format(url)
|
||||
f"No DVC remote is specified in target repository '{url}'."
|
||||
)
|
||||
|
||||
|
||||
@@ -293,7 +286,7 @@ class NoOutputInExternalRepoError(DvcException):
|
||||
|
||||
class HTTPError(DvcException):
|
||||
def __init__(self, code, reason):
|
||||
super().__init__("'{} {}'".format(code, reason))
|
||||
super().__init__(f"'{code} {reason}'")
|
||||
|
||||
|
||||
class PathMissingError(DvcException):
|
||||
@@ -301,19 +294,15 @@ class PathMissingError(DvcException):
|
||||
"The path '{}' does not exist in the target repository '{}'"
|
||||
" neither as a DVC output nor as a Git-tracked file."
|
||||
)
|
||||
default_msg_output_only = (
|
||||
default_msg_dvc_only = (
|
||||
"The path '{}' does not exist in the target repository '{}'"
|
||||
" as an DVC output."
|
||||
)
|
||||
|
||||
def __init__(self, path, repo, output_only=False):
|
||||
msg = (
|
||||
self.default_msg
|
||||
if not output_only
|
||||
else self.default_msg_output_only
|
||||
)
|
||||
def __init__(self, path, repo, dvc_only=False):
|
||||
msg = self.default_msg if not dvc_only else self.default_msg_dvc_only
|
||||
super().__init__(msg.format(path, repo))
|
||||
self.output_only = output_only
|
||||
self.dvc_only = dvc_only
|
||||
|
||||
|
||||
class RemoteCacheRequiredError(DvcException):
|
||||
@@ -333,3 +322,15 @@ class RemoteCacheRequiredError(DvcException):
|
||||
|
||||
class IsADirectoryError(DvcException):
|
||||
"""Raised when a file operation is requested on a directory."""
|
||||
|
||||
|
||||
class NoOutputOrStageError(DvcException):
|
||||
"""
|
||||
Raised when the target is neither an output nor a stage name in dvc.yaml
|
||||
"""
|
||||
|
||||
def __init__(self, target, file):
|
||||
super().__init__(
|
||||
f"'{target}' "
|
||||
f"does not exist as an output or a stage name in '{file}'"
|
||||
)
|
||||
|
||||
+115
-82
@@ -4,13 +4,12 @@ import tempfile
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from distutils.dir_util import copy_tree
|
||||
from typing import Iterable
|
||||
|
||||
from funcy import cached_property, retry, suppress, wrap_with
|
||||
from funcy import cached_property, retry, wrap_with
|
||||
|
||||
from dvc.compat import fspath
|
||||
from dvc.config import NoRemoteError, NotDvcRepoError
|
||||
from dvc.exceptions import (
|
||||
CheckoutError,
|
||||
FileMissingError,
|
||||
NoOutputInExternalRepoError,
|
||||
NoRemoteInExternalRepoError,
|
||||
@@ -19,9 +18,10 @@ from dvc.exceptions import (
|
||||
)
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.tree import RepoTree
|
||||
from dvc.scm.git import Git
|
||||
from dvc.utils import tmp_fname
|
||||
from dvc.utils.fs import fs_copy, move, remove
|
||||
from dvc.scm.tree import is_working_tree
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -30,10 +30,12 @@ logger = logging.getLogger(__name__)
|
||||
def external_repo(url, rev=None, for_write=False):
|
||||
logger.debug("Creating external repo %s@%s", url, rev)
|
||||
path = _cached_clone(url, rev, for_write=for_write)
|
||||
if not rev:
|
||||
rev = "HEAD"
|
||||
try:
|
||||
repo = ExternalRepo(path, url)
|
||||
repo = ExternalRepo(path, url, rev, for_write=for_write)
|
||||
except NotDvcRepoError:
|
||||
repo = ExternalGitRepo(path, url)
|
||||
repo = ExternalGitRepo(path, url, rev)
|
||||
|
||||
try:
|
||||
yield repo
|
||||
@@ -65,59 +67,99 @@ def clean_repos():
|
||||
_remove(path)
|
||||
|
||||
|
||||
class ExternalRepo(Repo):
|
||||
def __init__(self, root_dir, url):
|
||||
super().__init__(root_dir)
|
||||
class BaseExternalRepo:
|
||||
|
||||
_local_cache = None
|
||||
|
||||
@property
|
||||
def local_cache(self):
|
||||
if hasattr(self, "cache"):
|
||||
return self.cache.local
|
||||
return self._local_cache
|
||||
|
||||
@contextmanager
|
||||
def use_cache(self, cache):
|
||||
"""Use the specified cache in place of default tmpdir cache for
|
||||
download operations.
|
||||
"""
|
||||
if hasattr(self, "cache"):
|
||||
save_cache = self.cache.local
|
||||
self.cache.local = cache
|
||||
self._local_cache = cache
|
||||
|
||||
yield
|
||||
|
||||
if hasattr(self, "cache"):
|
||||
self.cache.local = save_cache
|
||||
self._local_cache = None
|
||||
|
||||
@cached_property
|
||||
def repo_tree(self):
|
||||
return RepoTree(self, fetch=True)
|
||||
|
||||
def get_rev(self):
|
||||
if is_working_tree(self.tree):
|
||||
return self.scm.get_rev()
|
||||
if hasattr(self.tree, "tree"):
|
||||
return self.tree.tree.rev
|
||||
return self.tree.rev
|
||||
|
||||
def fetch_external(self, paths: Iterable, **kwargs):
|
||||
"""Fetch specified external repo paths into cache.
|
||||
|
||||
Returns 3-tuple in the form
|
||||
(downloaded, failed, list(cache_infos))
|
||||
where cache_infos can be used as checkout targets for the
|
||||
fetched paths.
|
||||
"""
|
||||
download_results = []
|
||||
failed = 0
|
||||
|
||||
paths = [PathInfo(self.root_dir) / path for path in paths]
|
||||
|
||||
def download_update(result):
|
||||
download_results.append(result)
|
||||
|
||||
save_infos = []
|
||||
for path in paths:
|
||||
if not self.repo_tree.exists(path):
|
||||
raise PathMissingError(path, self.url)
|
||||
save_info = self.local_cache.save(
|
||||
path,
|
||||
None,
|
||||
tree=self.repo_tree,
|
||||
download_callback=download_update,
|
||||
)
|
||||
save_infos.append(save_info)
|
||||
|
||||
return sum(download_results), failed, save_infos
|
||||
|
||||
def get_external(self, path, dest):
|
||||
"""Convenience wrapper for fetch_external and checkout."""
|
||||
if self.local_cache:
|
||||
# fetch DVC and git files to tmpdir cache, then checkout
|
||||
_, _, save_infos = self.fetch_external([path])
|
||||
self.local_cache.checkout(PathInfo(dest), save_infos[0])
|
||||
else:
|
||||
# git-only erepo with no cache, just copy files directly
|
||||
# to dest
|
||||
path = PathInfo(self.root_dir) / path
|
||||
if not self.repo_tree.exists(path):
|
||||
raise PathMissingError(path, self.url)
|
||||
self.repo_tree.copytree(path, dest)
|
||||
|
||||
|
||||
class ExternalRepo(Repo, BaseExternalRepo):
|
||||
def __init__(self, root_dir, url, rev, for_write=False):
|
||||
if for_write:
|
||||
super().__init__(root_dir)
|
||||
else:
|
||||
root_dir = os.path.realpath(root_dir)
|
||||
super().__init__(root_dir, scm=Git(root_dir), rev=rev)
|
||||
self.url = url
|
||||
self._set_cache_dir()
|
||||
self._fix_upstream()
|
||||
|
||||
def pull_to(self, path, to_info):
|
||||
"""
|
||||
Pull the corresponding file or directory specified by `path` and
|
||||
checkout it into `to_info`.
|
||||
|
||||
It works with files tracked by Git and DVC, and also local files
|
||||
outside the repository.
|
||||
"""
|
||||
out = None
|
||||
path_info = PathInfo(self.root_dir) / path
|
||||
|
||||
with suppress(OutputNotFoundError):
|
||||
(out,) = self.find_outs_by_path(fspath(path_info), strict=False)
|
||||
|
||||
try:
|
||||
if out and out.use_cache:
|
||||
self._pull_cached(out, path_info, to_info)
|
||||
return
|
||||
|
||||
# Check if it is handled by Git (it can't have an absolute path)
|
||||
if os.path.isabs(path):
|
||||
raise FileNotFoundError
|
||||
|
||||
fs_copy(fspath(path_info), fspath(to_info))
|
||||
except FileNotFoundError:
|
||||
raise PathMissingError(path, self.url)
|
||||
|
||||
def _pull_cached(self, out, path_info, dest):
|
||||
with self.state:
|
||||
tmp = PathInfo(tmp_fname(dest))
|
||||
src = tmp / path_info.relative_to(out.path_info)
|
||||
|
||||
out.path_info = tmp
|
||||
|
||||
# Only pull unless all needed cache is present
|
||||
if out.changed_cache(filter_info=src):
|
||||
self.cloud.pull(out.get_used_cache(filter_info=src))
|
||||
|
||||
try:
|
||||
out.checkout(filter_info=src)
|
||||
except CheckoutError:
|
||||
raise FileNotFoundError
|
||||
|
||||
move(src, dest)
|
||||
remove(tmp)
|
||||
|
||||
@wrap_with(threading.Lock())
|
||||
def _set_cache_dir(self):
|
||||
try:
|
||||
@@ -126,6 +168,7 @@ class ExternalRepo(Repo):
|
||||
cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
|
||||
|
||||
self.cache.local.cache_dir = cache_dir
|
||||
self._local_cache = self.cache.local
|
||||
|
||||
def _fix_upstream(self):
|
||||
if not os.path.isdir(self.url):
|
||||
@@ -166,10 +209,11 @@ class ExternalRepo(Repo):
|
||||
self.config["core"]["remote"] = "auto-generated-upstream"
|
||||
|
||||
|
||||
class ExternalGitRepo:
|
||||
def __init__(self, root_dir, url):
|
||||
self.root_dir = root_dir
|
||||
class ExternalGitRepo(BaseExternalRepo):
|
||||
def __init__(self, root_dir, url, rev):
|
||||
self.root_dir = os.path.realpath(root_dir)
|
||||
self.url = url
|
||||
self.tree = self.scm.get_tree(rev)
|
||||
|
||||
@cached_property
|
||||
def scm(self):
|
||||
@@ -182,23 +226,15 @@ class ExternalGitRepo:
|
||||
def find_out_by_relpath(self, path):
|
||||
raise OutputNotFoundError(path, self)
|
||||
|
||||
def pull_to(self, path, to_info):
|
||||
try:
|
||||
# Git handled files can't have absolute path
|
||||
if os.path.isabs(path):
|
||||
raise FileNotFoundError
|
||||
|
||||
fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
|
||||
except FileNotFoundError:
|
||||
raise PathMissingError(path, self.url)
|
||||
|
||||
@contextmanager
|
||||
def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
|
||||
"""Opens a specified resource as a file object."""
|
||||
tree = RepoTree(self)
|
||||
try:
|
||||
abs_path = os.path.join(self.root_dir, path)
|
||||
with open(abs_path, mode, encoding=encoding) as fd:
|
||||
yield fd
|
||||
with tree.open(
|
||||
path, mode=mode, encoding=encoding, **kwargs
|
||||
) as fobj:
|
||||
yield fobj
|
||||
except FileNotFoundError:
|
||||
raise PathMissingError(path, self.url)
|
||||
|
||||
@@ -210,14 +246,12 @@ def _cached_clone(url, rev, for_write=False):
|
||||
revision checked out. If for_write is set prevents reusing this dir via
|
||||
cache.
|
||||
"""
|
||||
if not for_write and Git.is_sha(rev) and (url, rev) in CLONES:
|
||||
return CLONES[url, rev]
|
||||
|
||||
# even if we have already cloned this repo, we may need to
|
||||
# fetch/fast-forward to get specified rev
|
||||
clone_path = _clone_default_branch(url, rev)
|
||||
rev_sha = Git(clone_path).resolve_rev(rev or "HEAD")
|
||||
|
||||
if not for_write and (url, rev_sha) in CLONES:
|
||||
return CLONES[url, rev_sha]
|
||||
if not for_write and (url) in CLONES:
|
||||
return CLONES[url]
|
||||
|
||||
# Copy to a new dir to keep the clone clean
|
||||
repo_path = tempfile.mkdtemp("dvc-erepo")
|
||||
@@ -225,11 +259,10 @@ def _cached_clone(url, rev, for_write=False):
|
||||
copy_tree(clone_path, repo_path)
|
||||
|
||||
# Check out the specified revision
|
||||
if rev is not None:
|
||||
if for_write:
|
||||
_git_checkout(repo_path, rev)
|
||||
|
||||
if not for_write:
|
||||
CLONES[url, rev_sha] = repo_path
|
||||
else:
|
||||
CLONES[url] = repo_path
|
||||
return repo_path
|
||||
|
||||
|
||||
|
||||
+88
-18
@@ -5,13 +5,14 @@ from funcy import cached_property
|
||||
from pathspec import PathSpec
|
||||
from pathspec.patterns import GitWildMatchPattern
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.scm.tree import BaseTree
|
||||
from dvc.utils import relpath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DvcIgnore(object):
|
||||
class DvcIgnore:
|
||||
DVCIGNORE_FILE = ".dvcignore"
|
||||
|
||||
def __call__(self, root, dirs, files):
|
||||
@@ -35,12 +36,19 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
return dirs, files
|
||||
|
||||
def matches(self, dirname, basename):
|
||||
abs_path = os.path.join(dirname, basename)
|
||||
rel_path = relpath(abs_path, self.dirname)
|
||||
|
||||
if os.pardir + os.sep in rel_path:
|
||||
# NOTE: `relpath` is too slow, so we have to assume that both
|
||||
# `dirname` and `self.dirname` are relative or absolute together.
|
||||
prefix = self.dirname + os.sep
|
||||
if dirname == self.dirname:
|
||||
path = basename
|
||||
elif dirname.startswith(prefix):
|
||||
rel = dirname[len(prefix) :]
|
||||
# NOTE: `os.path.join` is ~x5.5 slower
|
||||
path = f"{rel}{os.sep}{basename}"
|
||||
else:
|
||||
return False
|
||||
return self.ignore_spec.match_file(rel_path)
|
||||
|
||||
return self.ignore_spec.match_file(path)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.ignore_file_path)
|
||||
@@ -76,14 +84,14 @@ class DvcIgnoreRepo(DvcIgnore):
|
||||
def is_dvc_repo(directory):
|
||||
from dvc.repo import Repo
|
||||
|
||||
return os.path.isdir(os.path.join(directory, Repo.DVC_DIR))
|
||||
return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
|
||||
|
||||
dirs = [d for d in dirs if not is_dvc_repo(d)]
|
||||
|
||||
return dirs, files
|
||||
|
||||
|
||||
class DvcIgnoreFilter(object):
|
||||
class DvcIgnoreFilter:
|
||||
def __init__(self, tree):
|
||||
self.tree = tree
|
||||
self.ignores = {
|
||||
@@ -119,27 +127,89 @@ class CleanTree(BaseTree):
|
||||
return self.tree.tree_root
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8"):
|
||||
return self.tree.open(path, mode, encoding)
|
||||
if self.isfile(path):
|
||||
return self.tree.open(path, mode, encoding)
|
||||
raise FileNotFoundError
|
||||
|
||||
def exists(self, path):
|
||||
return self.tree.exists(path)
|
||||
if self.tree.exists(path) and self._parents_exist(path):
|
||||
if self.tree.isdir(path):
|
||||
return self._valid_dirname(path)
|
||||
return self._valid_filename(path)
|
||||
return False
|
||||
|
||||
def isdir(self, path):
|
||||
return self.tree.isdir(path)
|
||||
return (
|
||||
self.tree.isdir(path)
|
||||
and self._parents_exist(path)
|
||||
and self._valid_dirname(path)
|
||||
)
|
||||
|
||||
def _valid_dirname(self, path):
|
||||
dirname, basename = os.path.split(os.path.normpath(path))
|
||||
dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
|
||||
if dirs:
|
||||
return True
|
||||
return False
|
||||
|
||||
def isfile(self, path):
|
||||
return self.tree.isfile(path)
|
||||
return (
|
||||
self.tree.isfile(path)
|
||||
and self._parents_exist(path)
|
||||
and self._valid_filename(path)
|
||||
)
|
||||
|
||||
def _valid_filename(self, path):
|
||||
dirname, basename = os.path.split(os.path.normpath(path))
|
||||
_, files = self.dvcignore(os.path.abspath(dirname), [], [basename])
|
||||
if files:
|
||||
return True
|
||||
return False
|
||||
|
||||
def isexec(self, path):
|
||||
return self.tree.isexec(path)
|
||||
return self.exists(path) and self.tree.isexec(path)
|
||||
|
||||
def _parents_exist(self, path):
|
||||
from dvc.repo import Repo
|
||||
|
||||
path = PathInfo(path)
|
||||
|
||||
# if parent is tree_root or inside a .dvc dir we can skip this check
|
||||
if path.parent == self.tree_root or Repo.DVC_DIR in path.parts:
|
||||
return True
|
||||
|
||||
# if path is outside of tree, assume this is a local remote/local cache
|
||||
# link/move operation where we do not need to filter ignores
|
||||
path = relpath(path, self.tree_root)
|
||||
if path.startswith("..") or (
|
||||
os.name == "nt"
|
||||
and not os.path.commonprefix(
|
||||
[os.path.abspath(path), self.tree_root]
|
||||
)
|
||||
):
|
||||
return True
|
||||
|
||||
# check if parent directories are in our ignores, starting from
|
||||
# tree_root
|
||||
for parent_dir in reversed(PathInfo(path).parents):
|
||||
dirname, basename = os.path.split(parent_dir)
|
||||
if basename == ".":
|
||||
# parent_dir == tree_root
|
||||
continue
|
||||
dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
|
||||
if not dirs:
|
||||
return False
|
||||
return True
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
for root, dirs, files in self.tree.walk(top, topdown):
|
||||
dirs[:], files[:] = self.dvcignore(root, dirs, files)
|
||||
dirs[:], files[:] = self.dvcignore(
|
||||
os.path.abspath(root), dirs, files
|
||||
)
|
||||
|
||||
yield root, dirs, files
|
||||
|
||||
def walk_files(self, top):
|
||||
for root, _, files in self.walk(top):
|
||||
for file in files:
|
||||
yield os.path.join(root, file)
|
||||
def stat(self, path):
|
||||
if self.exists(path):
|
||||
return self.tree.stat(path)
|
||||
raise FileNotFoundError
|
||||
|
||||
+6
-2
@@ -7,13 +7,17 @@
|
||||
TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
|
||||
|
||||
|
||||
def istextfile(fname, blocksize=512):
|
||||
def istextfile(fname, blocksize=512, tree=None):
|
||||
""" Uses heuristics to guess whether the given file is text or binary,
|
||||
by reading a single block of bytes from the file.
|
||||
If more than 30% of the chars in the block are non-text, or there
|
||||
are NUL ('\x00') bytes in the block, assume this is a binary file.
|
||||
"""
|
||||
with open(fname, "rb") as fobj:
|
||||
if tree:
|
||||
open_func = tree.open
|
||||
else:
|
||||
open_func = open
|
||||
with open_func(fname, "rb") as fobj:
|
||||
block = fobj.read(blocksize)
|
||||
|
||||
if not block:
|
||||
|
||||
+2
-2
@@ -25,7 +25,7 @@ class LockError(DvcException):
|
||||
"""Thrown when unable to acquire the lock for DVC repo."""
|
||||
|
||||
|
||||
class Lock(object):
|
||||
class Lock:
|
||||
"""Class for DVC repo lock.
|
||||
|
||||
Uses zc.lockfile as backend.
|
||||
@@ -47,7 +47,7 @@ class Lock(object):
|
||||
disable=not self._friendly,
|
||||
desc=(
|
||||
"If DVC froze, see `hardlink_lock` in {}".format(
|
||||
format_link("man.dvc.org/config#core")
|
||||
format_link("https://man.dvc.org/config#core")
|
||||
)
|
||||
),
|
||||
):
|
||||
|
||||
+50
-10
@@ -18,20 +18,45 @@ FOOTER = (
|
||||
)
|
||||
|
||||
|
||||
def addLoggingLevel(levelName, levelNum, methodName=None):
|
||||
"""
|
||||
Adds a new logging level to the `logging` module and the
|
||||
currently configured logging class.
|
||||
|
||||
Based on https://stackoverflow.com/questions/2183233
|
||||
"""
|
||||
if methodName is None:
|
||||
methodName = levelName.lower()
|
||||
|
||||
assert not hasattr(logging, levelName)
|
||||
assert not hasattr(logging, methodName)
|
||||
assert not hasattr(logging.getLoggerClass(), methodName)
|
||||
|
||||
def logForLevel(self, message, *args, **kwargs):
|
||||
if self.isEnabledFor(levelNum):
|
||||
self._log(levelNum, message, args, **kwargs)
|
||||
|
||||
def logToRoot(message, *args, **kwargs):
|
||||
logging.log(levelNum, message, *args, **kwargs)
|
||||
|
||||
logging.addLevelName(levelNum, levelName)
|
||||
setattr(logging, levelName, levelNum)
|
||||
setattr(logging.getLoggerClass(), methodName, logForLevel)
|
||||
setattr(logging, methodName, logToRoot)
|
||||
|
||||
|
||||
class LoggingException(Exception):
|
||||
def __init__(self, record):
|
||||
msg = "failed to log {}".format(str(record))
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class ExcludeErrorsFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.levelno < logging.WARNING
|
||||
def excludeFilter(level):
|
||||
class ExcludeLevelFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.levelno < level
|
||||
|
||||
|
||||
class ExcludeInfoFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.levelno < logging.INFO
|
||||
return ExcludeLevelFilter
|
||||
|
||||
|
||||
class ColorFormatter(logging.Formatter):
|
||||
@@ -47,6 +72,7 @@ class ColorFormatter(logging.Formatter):
|
||||
"""
|
||||
|
||||
color_code = {
|
||||
"TRACE": colorama.Fore.GREEN,
|
||||
"DEBUG": colorama.Fore.BLUE,
|
||||
"WARNING": colorama.Fore.YELLOW,
|
||||
"ERROR": colorama.Fore.RED,
|
||||
@@ -116,7 +142,11 @@ class LoggerHandler(logging.StreamHandler):
|
||||
|
||||
|
||||
def _is_verbose():
|
||||
return logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG
|
||||
return (
|
||||
logging.NOTSET
|
||||
< logging.getLogger("dvc").getEffectiveLevel()
|
||||
<= logging.DEBUG
|
||||
)
|
||||
|
||||
|
||||
def _iter_causes(exc):
|
||||
@@ -152,12 +182,14 @@ def disable_other_loggers():
|
||||
def setup(level=logging.INFO):
|
||||
colorama.init()
|
||||
|
||||
addLoggingLevel("TRACE", logging.DEBUG - 5)
|
||||
logging.config.dictConfig(
|
||||
{
|
||||
"version": 1,
|
||||
"filters": {
|
||||
"exclude_errors": {"()": ExcludeErrorsFilter},
|
||||
"exclude_info": {"()": ExcludeInfoFilter},
|
||||
"exclude_errors": {"()": excludeFilter(logging.WARNING)},
|
||||
"exclude_info": {"()": excludeFilter(logging.INFO)},
|
||||
"exclude_debug": {"()": excludeFilter(logging.DEBUG)},
|
||||
},
|
||||
"formatters": {"color": {"()": ColorFormatter}},
|
||||
"handlers": {
|
||||
@@ -175,6 +207,13 @@ def setup(level=logging.INFO):
|
||||
"stream": "ext://sys.stdout",
|
||||
"filters": ["exclude_info"],
|
||||
},
|
||||
"console_trace": {
|
||||
"class": "dvc.logger.LoggerHandler",
|
||||
"level": "TRACE",
|
||||
"formatter": "color",
|
||||
"stream": "ext://sys.stdout",
|
||||
"filters": ["exclude_debug"],
|
||||
},
|
||||
"console_errors": {
|
||||
"class": "dvc.logger.LoggerHandler",
|
||||
"level": "WARNING",
|
||||
@@ -188,6 +227,7 @@ def setup(level=logging.INFO):
|
||||
"handlers": [
|
||||
"console_info",
|
||||
"console_debug",
|
||||
"console_trace",
|
||||
"console_errors",
|
||||
],
|
||||
},
|
||||
|
||||
+11
-6
@@ -18,7 +18,6 @@ from dvc.utils import format_link
|
||||
|
||||
"".encode("idna")
|
||||
|
||||
|
||||
logger = logging.getLogger("dvc")
|
||||
|
||||
|
||||
@@ -38,11 +37,17 @@ def main(argv=None):
|
||||
try:
|
||||
args = parse_args(argv)
|
||||
|
||||
if args.quiet:
|
||||
logger.setLevel(logging.CRITICAL)
|
||||
|
||||
elif args.verbose:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
verbosity = args.verbose - args.quiet
|
||||
if verbosity:
|
||||
logger.setLevel(
|
||||
{
|
||||
-2: logging.CRITICAL,
|
||||
-1: logging.ERROR,
|
||||
1: logging.DEBUG,
|
||||
2: logging.TRACE,
|
||||
}[max(-2, min(verbosity, 2))]
|
||||
)
|
||||
logger.trace(args)
|
||||
|
||||
cmd = args.func(args)
|
||||
ret = cmd.run()
|
||||
|
||||
+27
-15
@@ -54,10 +54,11 @@ SCHEMA = CHECKSUMS_SCHEMA.copy()
|
||||
SCHEMA[Required(BaseOutput.PARAM_PATH)] = str
|
||||
SCHEMA[BaseOutput.PARAM_CACHE] = bool
|
||||
SCHEMA[BaseOutput.PARAM_METRIC] = BaseOutput.METRIC_SCHEMA
|
||||
SCHEMA[BaseOutput.PARAM_PLOT] = bool
|
||||
SCHEMA[BaseOutput.PARAM_PERSIST] = bool
|
||||
|
||||
|
||||
def _get(stage, p, info, cache, metric, persist=False):
|
||||
def _get(stage, p, info, cache, metric, plot=False, persist=False):
|
||||
parsed = urlparse(p)
|
||||
|
||||
if parsed.scheme == "remote":
|
||||
@@ -69,6 +70,7 @@ def _get(stage, p, info, cache, metric, persist=False):
|
||||
cache=cache,
|
||||
remote=remote,
|
||||
metric=metric,
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
)
|
||||
|
||||
@@ -81,6 +83,7 @@ def _get(stage, p, info, cache, metric, persist=False):
|
||||
cache=cache,
|
||||
remote=None,
|
||||
metric=metric,
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
)
|
||||
return LocalOutput(
|
||||
@@ -90,6 +93,7 @@ def _get(stage, p, info, cache, metric, persist=False):
|
||||
cache=cache,
|
||||
remote=None,
|
||||
metric=metric,
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
)
|
||||
|
||||
@@ -100,26 +104,34 @@ def loadd_from(stage, d_list):
|
||||
p = d.pop(BaseOutput.PARAM_PATH)
|
||||
cache = d.pop(BaseOutput.PARAM_CACHE, True)
|
||||
metric = d.pop(BaseOutput.PARAM_METRIC, False)
|
||||
plot = d.pop(BaseOutput.PARAM_PLOT, False)
|
||||
persist = d.pop(BaseOutput.PARAM_PERSIST, False)
|
||||
ret.append(
|
||||
_get(
|
||||
stage, p, info=d, cache=cache, metric=metric, persist=persist,
|
||||
)
|
||||
)
|
||||
return ret
|
||||
|
||||
|
||||
def loads_from(stage, s_list, use_cache=True, metric=False, persist=False):
|
||||
ret = []
|
||||
for s in s_list:
|
||||
ret.append(
|
||||
_get(
|
||||
stage,
|
||||
s,
|
||||
info={},
|
||||
cache=use_cache,
|
||||
p,
|
||||
info=d,
|
||||
cache=cache,
|
||||
metric=metric,
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
)
|
||||
)
|
||||
return ret
|
||||
|
||||
|
||||
def loads_from(
|
||||
stage, s_list, use_cache=True, metric=False, plot=False, persist=False
|
||||
):
|
||||
return [
|
||||
_get(
|
||||
stage,
|
||||
s,
|
||||
info={},
|
||||
cache=use_cache,
|
||||
metric=metric,
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
)
|
||||
for s in s_list
|
||||
]
|
||||
|
||||
+46
-32
@@ -19,28 +19,32 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class OutputDoesNotExistError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = "output '{}' does not exist".format(path)
|
||||
msg = f"output '{path}' does not exist"
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class OutputIsNotFileOrDirError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = "output '{}' is not a file or directory".format(path)
|
||||
msg = f"output '{path}' is not a file or directory"
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class OutputAlreadyTrackedError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = "output '{}' is already tracked by SCM (e.g. Git)".format(path)
|
||||
msg = f""" output '{path}' is already tracked by SCM (e.g. Git).
|
||||
You can remove it from Git, then add to DVC.
|
||||
To stop tracking from Git:
|
||||
git rm -r --cached '{path}'
|
||||
git commit -m "stop tracking {path}" """
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class OutputIsStageFileError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__("Stage file '{}' cannot be an output.".format(path))
|
||||
super().__init__(f"Stage file '{path}' cannot be an output.")
|
||||
|
||||
|
||||
class BaseOutput(object):
|
||||
class BaseOutput:
|
||||
IS_DEPENDENCY = False
|
||||
|
||||
REMOTE = BaseRemote
|
||||
@@ -50,6 +54,8 @@ class BaseOutput(object):
|
||||
PARAM_METRIC = "metric"
|
||||
PARAM_METRIC_TYPE = "type"
|
||||
PARAM_METRIC_XPATH = "xpath"
|
||||
PARAM_PLOT = "plot"
|
||||
PARAM_PLOT_TEMPLATE = "template"
|
||||
PARAM_PERSIST = "persist"
|
||||
|
||||
METRIC_SCHEMA = Any(
|
||||
@@ -75,6 +81,7 @@ class BaseOutput(object):
|
||||
remote=None,
|
||||
cache=True,
|
||||
metric=False,
|
||||
plot=False,
|
||||
persist=False,
|
||||
):
|
||||
self._validate_output_path(path)
|
||||
@@ -95,6 +102,7 @@ class BaseOutput(object):
|
||||
self.remote = remote or self.REMOTE(self.repo, {})
|
||||
self.use_cache = False if self.IS_DEPENDENCY else cache
|
||||
self.metric = False if self.IS_DEPENDENCY else metric
|
||||
self.plot = False if self.IS_DEPENDENCY else plot
|
||||
self.persist = persist
|
||||
|
||||
self.path_info = self._parse_path(remote, path)
|
||||
@@ -228,28 +236,24 @@ class BaseOutput(object):
|
||||
raise self.IsNotFileOrDirError(self)
|
||||
|
||||
if self.is_empty:
|
||||
logger.warning("'{}' is empty.".format(self))
|
||||
logger.warning(f"'{self}' is empty.")
|
||||
|
||||
self.ignore()
|
||||
|
||||
if not self.use_cache:
|
||||
self.info = self.save_info()
|
||||
if self.metric:
|
||||
if self.metric or self.plot:
|
||||
self.verify_metric()
|
||||
if not self.IS_DEPENDENCY:
|
||||
logger.info(
|
||||
"Output '{}' doesn't use cache. Skipping saving.".format(
|
||||
self
|
||||
)
|
||||
logger.debug(
|
||||
"Output '%s' doesn't use cache. Skipping saving.", self
|
||||
)
|
||||
return
|
||||
|
||||
assert not self.IS_DEPENDENCY
|
||||
|
||||
if not self.changed():
|
||||
logger.info(
|
||||
"Output '{}' didn't change. Skipping saving.".format(self)
|
||||
)
|
||||
logger.debug("Output '%s' didn't change. Skipping saving.", self)
|
||||
return
|
||||
|
||||
self.info = self.save_info()
|
||||
@@ -265,7 +269,8 @@ class BaseOutput(object):
|
||||
if self.IS_DEPENDENCY:
|
||||
return ret
|
||||
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
if not self.use_cache:
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
|
||||
if isinstance(self.metric, dict):
|
||||
if (
|
||||
@@ -274,15 +279,19 @@ class BaseOutput(object):
|
||||
):
|
||||
del self.metric[self.PARAM_METRIC_XPATH]
|
||||
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
ret[self.PARAM_PERSIST] = self.persist
|
||||
if self.metric:
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
|
||||
if self.plot:
|
||||
ret[self.PARAM_PLOT] = self.plot
|
||||
|
||||
if self.persist:
|
||||
ret[self.PARAM_PERSIST] = self.persist
|
||||
|
||||
return ret
|
||||
|
||||
def verify_metric(self):
|
||||
raise DvcException(
|
||||
"verify metric is not supported for {}".format(self.scheme)
|
||||
)
|
||||
raise DvcException(f"verify metric is not supported for {self.scheme}")
|
||||
|
||||
def download(self, to):
|
||||
self.remote.download(self.path_info, to.path_info)
|
||||
@@ -343,7 +352,18 @@ class BaseOutput(object):
|
||||
if self.exists:
|
||||
self.remote.unprotect(self.path_info)
|
||||
|
||||
def _collect_used_dir_cache(
|
||||
def get_dir_cache(self, **kwargs):
|
||||
if not self.is_dir_checksum:
|
||||
raise DvcException("cannot get dir cache for file checksum")
|
||||
if self.cache.changed_cache_file(self.checksum):
|
||||
self.repo.cloud.pull(
|
||||
NamedCache.make("local", self.checksum, str(self)),
|
||||
show_checksums=False,
|
||||
**kwargs,
|
||||
)
|
||||
return self.dir_cache
|
||||
|
||||
def collect_used_dir_cache(
|
||||
self, remote=None, force=False, jobs=None, filter_info=None
|
||||
):
|
||||
"""Get a list of `info`s related to the given directory.
|
||||
@@ -367,16 +387,10 @@ class BaseOutput(object):
|
||||
|
||||
cache = NamedCache()
|
||||
|
||||
if self.cache.changed_cache_file(self.checksum):
|
||||
try:
|
||||
self.repo.cloud.pull(
|
||||
NamedCache.make("local", self.checksum, str(self)),
|
||||
jobs=jobs,
|
||||
remote=remote,
|
||||
show_checksums=False,
|
||||
)
|
||||
except DvcException:
|
||||
logger.debug("failed to pull cache for '{}'".format(self))
|
||||
try:
|
||||
self.get_dir_cache(jobs=jobs, remote=remote)
|
||||
except DvcException:
|
||||
logger.debug(f"failed to pull cache for '{self}'")
|
||||
|
||||
if self.cache.changed_cache_file(self.checksum):
|
||||
msg = (
|
||||
@@ -453,7 +467,7 @@ class BaseOutput(object):
|
||||
return ret
|
||||
|
||||
ret.add_child_cache(
|
||||
self.checksum, self._collect_used_dir_cache(**kwargs),
|
||||
self.checksum, self.collect_used_dir_cache(**kwargs),
|
||||
)
|
||||
|
||||
return ret
|
||||
|
||||
+8
-8
@@ -2,7 +2,6 @@ import logging
|
||||
import os
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.compat import fspath_py35
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.istextfile import istextfile
|
||||
from dvc.output.base import BaseOutput
|
||||
@@ -38,7 +37,7 @@ class LocalOutput(BaseOutput):
|
||||
if not p.is_absolute():
|
||||
p = self.stage.wdir / p
|
||||
|
||||
abs_p = os.path.abspath(os.path.normpath(fspath_py35(p)))
|
||||
abs_p = os.path.abspath(os.path.normpath(p))
|
||||
return self.REMOTE.path_cls(abs_p)
|
||||
|
||||
def __str__(self):
|
||||
@@ -72,17 +71,18 @@ class LocalOutput(BaseOutput):
|
||||
return ret
|
||||
|
||||
def verify_metric(self):
|
||||
if not self.metric:
|
||||
if not self.metric or self.plot:
|
||||
return
|
||||
|
||||
path = fspath_py35(self.path_info)
|
||||
path = os.fspath(self.path_info)
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
|
||||
name = "metrics" if self.metric else "plot"
|
||||
if os.path.isdir(path):
|
||||
msg = "directory '{}' cannot be used as metrics."
|
||||
raise DvcException(msg.format(self.path_info))
|
||||
msg = "directory '{}' cannot be used as {}."
|
||||
raise DvcException(msg.format(self.path_info, name))
|
||||
|
||||
if not istextfile(path):
|
||||
msg = "binary file '{}' cannot be used as metrics."
|
||||
raise DvcException(msg.format(self.path_info))
|
||||
msg = "binary file '{}' cannot be used as {}."
|
||||
raise DvcException(msg.format(self.path_info, name))
|
||||
|
||||
+7
-8
@@ -8,7 +8,7 @@ from funcy import cached_property
|
||||
from dvc.utils import relpath
|
||||
|
||||
|
||||
class _BasePath(object):
|
||||
class _BasePath:
|
||||
def overlaps(self, other):
|
||||
if isinstance(other, (str, bytes)):
|
||||
other = self.__class__(other)
|
||||
@@ -50,7 +50,6 @@ class PathInfo(pathlib.PurePath, _BasePath):
|
||||
return "{}: '{}'".format(type(self).__name__, self)
|
||||
|
||||
# This permits passing it to file utils directly in Python 3.6+
|
||||
# With Python 2.7, Python 3.5+ we are stuck with path_info.fspath for now
|
||||
def __fspath__(self):
|
||||
return pathlib.PurePath.__str__(self)
|
||||
|
||||
@@ -88,7 +87,7 @@ class _URLPathInfo(PosixPathInfo):
|
||||
__unicode__ = __str__
|
||||
|
||||
|
||||
class _URLPathParents(object):
|
||||
class _URLPathParents:
|
||||
def __init__(self, src):
|
||||
self.src = src
|
||||
self._parents = self.src._path.parents
|
||||
@@ -100,7 +99,7 @@ class _URLPathParents(object):
|
||||
return self.src.replace(path=self._parents[idx])
|
||||
|
||||
def __repr__(self):
|
||||
return "<{}.parents>".format(self.src)
|
||||
return f"<{self.src}.parents>"
|
||||
|
||||
|
||||
class URLInfo(_BasePath):
|
||||
@@ -120,7 +119,7 @@ class URLInfo(_BasePath):
|
||||
assert bool(host) ^ bool(netloc)
|
||||
|
||||
if netloc is not None:
|
||||
return cls("{}://{}{}".format(scheme, netloc, path))
|
||||
return cls(f"{scheme}://{netloc}{path}")
|
||||
|
||||
obj = cls.__new__(cls)
|
||||
obj._fill_parts(scheme, host, user, port, path)
|
||||
@@ -154,7 +153,7 @@ class URLInfo(_BasePath):
|
||||
|
||||
@cached_property
|
||||
def url(self):
|
||||
return "{}://{}{}".format(self.scheme, self.netloc, self._spath)
|
||||
return f"{self.scheme}://{self.netloc}{self._spath}"
|
||||
|
||||
def __str__(self):
|
||||
return self.url
|
||||
@@ -216,10 +215,10 @@ class URLInfo(_BasePath):
|
||||
if isinstance(other, (str, bytes)):
|
||||
other = self.__class__(other)
|
||||
if self.__class__ != other.__class__:
|
||||
msg = "'{}' has incompatible class with '{}'".format(self, other)
|
||||
msg = f"'{self}' has incompatible class with '{other}'"
|
||||
raise ValueError(msg)
|
||||
if self._base_parts != other._base_parts:
|
||||
msg = "'{}' does not start with '{}'".format(self, other)
|
||||
msg = f"'{self}' does not start with '{other}'"
|
||||
raise ValueError(msg)
|
||||
return self._path.relative_to(other._path)
|
||||
|
||||
|
||||
+2
-2
@@ -38,7 +38,7 @@ def confirm(statement):
|
||||
Returns:
|
||||
bool: whether or not specified statement was confirmed.
|
||||
"""
|
||||
prompt = "{statement} [y/n]".format(statement=statement)
|
||||
prompt = f"{statement} [y/n]"
|
||||
answer = _ask(prompt, limited_to=["yes", "no", "y", "n"])
|
||||
return answer and answer.startswith("y")
|
||||
|
||||
@@ -52,5 +52,5 @@ def password(statement):
|
||||
Returns:
|
||||
str: password entered by the user.
|
||||
"""
|
||||
logger.info("{statement}: ".format(statement=statement))
|
||||
logger.info(f"{statement}: ")
|
||||
return getpass("")
|
||||
|
||||
+19
-35
@@ -1,10 +1,8 @@
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
@@ -19,12 +17,6 @@ logger = logging.getLogger(__name__)
|
||||
class AzureRemote(BaseRemote):
|
||||
scheme = Schemes.AZURE
|
||||
path_cls = CloudURLInfo
|
||||
REGEX = (
|
||||
r"azure://((?P<path>[^=;]*)?|("
|
||||
# backward compatibility
|
||||
r"(ContainerName=(?P<container_name>[^;]+);?)?"
|
||||
r"(?P<connection_string>.+)?)?)$"
|
||||
)
|
||||
REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
@@ -34,31 +26,14 @@ class AzureRemote(BaseRemote):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url", "azure://")
|
||||
self.path_info = self.path_cls(url)
|
||||
|
||||
match = re.match(self.REGEX, url) # backward compatibility
|
||||
path = match.group("path")
|
||||
bucket = (
|
||||
urlparse(url if path else "").netloc
|
||||
or match.group("container_name") # backward compatibility
|
||||
or os.getenv("AZURE_STORAGE_CONTAINER_NAME")
|
||||
)
|
||||
if not self.path_info.bucket:
|
||||
container = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
|
||||
self.path_info = self.path_cls(f"azure://{container}")
|
||||
|
||||
self.connection_string = (
|
||||
config.get("connection_string")
|
||||
or match.group("connection_string") # backward compatibility
|
||||
or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
|
||||
)
|
||||
|
||||
if not bucket:
|
||||
raise ValueError("azure storage container name missing")
|
||||
|
||||
if not self.connection_string:
|
||||
raise ValueError("azure storage connection string missing")
|
||||
|
||||
self.path_info = (
|
||||
self.path_cls(url)
|
||||
if path
|
||||
else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
|
||||
self.connection_string = config.get("connection_string") or os.getenv(
|
||||
"AZURE_STORAGE_CONNECTION_STRING"
|
||||
)
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@@ -67,12 +42,12 @@ class AzureRemote(BaseRemote):
|
||||
from azure.storage.blob import BlockBlobService
|
||||
from azure.common import AzureMissingResourceHttpError
|
||||
|
||||
logger.debug("URL {}".format(self.path_info))
|
||||
logger.debug("Connection string {}".format(self.connection_string))
|
||||
logger.debug(f"URL {self.path_info}")
|
||||
logger.debug(f"Connection string {self.connection_string}")
|
||||
blob_service = BlockBlobService(
|
||||
connection_string=self.connection_string
|
||||
)
|
||||
logger.debug("Container name {}".format(self.path_info.bucket))
|
||||
logger.debug(f"Container name {self.path_info.bucket}")
|
||||
try: # verify that container exists
|
||||
blob_service.list_blobs(
|
||||
self.path_info.bucket, delimiter="/", num_results=1
|
||||
@@ -81,11 +56,20 @@ class AzureRemote(BaseRemote):
|
||||
blob_service.create_container(self.path_info.bucket)
|
||||
return blob_service
|
||||
|
||||
def get_etag(self, path_info):
|
||||
etag = self.blob_service.get_blob_properties(
|
||||
path_info.bucket, path_info.path
|
||||
).properties.etag
|
||||
return etag.strip('"')
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
return self.get_etag(path_info)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug("Removing {}".format(path_info))
|
||||
logger.debug(f"Removing {path_info}")
|
||||
self.blob_service.delete_blob(path_info.bucket, path_info.path)
|
||||
|
||||
def _list_paths(self, bucket, prefix, progress_callback=None):
|
||||
|
||||
+133
-85
@@ -26,6 +26,7 @@ from dvc.path_info import PathInfo, URLInfo, WindowsPathInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.index import RemoteIndex, RemoteIndexNoop
|
||||
from dvc.remote.slow_link_detection import slow_link_guard
|
||||
from dvc.scm.tree import is_working_tree
|
||||
from dvc.state import StateNoop
|
||||
from dvc.utils import tmp_fname
|
||||
from dvc.utils.fs import makedirs, move
|
||||
@@ -57,7 +58,7 @@ class RemoteCmdError(DvcException):
|
||||
|
||||
class RemoteActionNotImplemented(DvcException):
|
||||
def __init__(self, action, scheme):
|
||||
m = "{} is not supported for {} remotes".format(action, scheme)
|
||||
m = f"{action} is not supported for {scheme} remotes"
|
||||
super().__init__(m)
|
||||
|
||||
|
||||
@@ -68,7 +69,7 @@ class RemoteMissingDepsError(DvcException):
|
||||
class DirCacheError(DvcException):
|
||||
def __init__(self, checksum):
|
||||
super().__init__(
|
||||
"Failed to load dir cache for hash value: '{}'.".format(checksum)
|
||||
f"Failed to load dir cache for hash value: '{checksum}'."
|
||||
)
|
||||
|
||||
|
||||
@@ -82,7 +83,7 @@ def index_locked(f):
|
||||
return wrapper
|
||||
|
||||
|
||||
class BaseRemote(object):
|
||||
class BaseRemote:
|
||||
scheme = "base"
|
||||
path_cls = URLInfo
|
||||
REQUIRES = {}
|
||||
@@ -151,7 +152,7 @@ class BaseRemote(object):
|
||||
if not missing:
|
||||
return
|
||||
|
||||
url = config.get("url", "{}://".format(self.scheme))
|
||||
url = config.get("url", f"{self.scheme}://")
|
||||
msg = (
|
||||
"URL '{}' is supported but requires these missing "
|
||||
"dependencies: {}. If you have installed dvc using pip, "
|
||||
@@ -211,23 +212,33 @@ class BaseRemote(object):
|
||||
checksums = dict(zip(file_infos, tasks))
|
||||
return checksums
|
||||
|
||||
def _collect_dir(self, path_info):
|
||||
def _collect_dir(self, path_info, tree=None, save_tree=False, **kwargs):
|
||||
file_infos = set()
|
||||
|
||||
for fname in self.walk_files(path_info):
|
||||
if tree:
|
||||
walk_files = tree.walk_files
|
||||
else:
|
||||
walk_files = self.walk_files
|
||||
|
||||
for fname in walk_files(path_info, **kwargs):
|
||||
if DvcIgnore.DVCIGNORE_FILE == fname.name:
|
||||
raise DvcIgnoreInCollectedDirError(fname.parent)
|
||||
|
||||
file_infos.add(fname)
|
||||
|
||||
checksums = {fi: self.state.get(fi) for fi in file_infos}
|
||||
not_in_state = {
|
||||
fi for fi, checksum in checksums.items() if checksum is None
|
||||
}
|
||||
if tree:
|
||||
checksums = {fi: tree.get_file_checksum(fi) for fi in file_infos}
|
||||
if save_tree:
|
||||
for fi, checksum in checksums.items():
|
||||
self._save_file(fi, checksum, tree=tree, **kwargs)
|
||||
else:
|
||||
checksums = {fi: self.state.get(fi) for fi in file_infos}
|
||||
not_in_state = {
|
||||
fi for fi, checksum in checksums.items() if checksum is None
|
||||
}
|
||||
|
||||
new_checksums = self._calculate_checksums(not_in_state)
|
||||
|
||||
checksums.update(new_checksums)
|
||||
new_checksums = self._calculate_checksums(not_in_state)
|
||||
checksums.update(new_checksums)
|
||||
|
||||
result = [
|
||||
{
|
||||
@@ -248,18 +259,25 @@ class BaseRemote(object):
|
||||
# Sorting the list by path to ensure reproducibility
|
||||
return sorted(result, key=itemgetter(self.PARAM_RELPATH))
|
||||
|
||||
def get_dir_checksum(self, path_info):
|
||||
def get_dir_checksum(self, path_info, tree=None):
|
||||
if not self.cache:
|
||||
raise RemoteCacheRequiredError(path_info)
|
||||
|
||||
dir_info = self._collect_dir(path_info)
|
||||
dir_info = self._collect_dir(path_info, tree=None)
|
||||
if tree:
|
||||
# don't save state entry for path_info if it is a tree path
|
||||
path_info = None
|
||||
return self._save_dir_info(dir_info, path_info)
|
||||
|
||||
def _save_dir_info(self, dir_info, path_info=None):
|
||||
checksum, tmp_info = self._get_dir_info_checksum(dir_info)
|
||||
new_info = self.cache.checksum_to_path_info(checksum)
|
||||
if self.cache.changed_cache_file(checksum):
|
||||
self.cache.makedirs(new_info.parent)
|
||||
self.cache.move(tmp_info, new_info, mode=self.CACHE_MODE)
|
||||
|
||||
self.state.save(path_info, checksum)
|
||||
if path_info:
|
||||
self.state.save(path_info, checksum)
|
||||
self.state.save(new_info, checksum)
|
||||
|
||||
return checksum
|
||||
@@ -319,6 +337,8 @@ class BaseRemote(object):
|
||||
|
||||
@classmethod
|
||||
def is_dir_checksum(cls, checksum):
|
||||
if not checksum:
|
||||
return False
|
||||
return checksum.endswith(cls.CHECKSUM_DIR_SUFFIX)
|
||||
|
||||
def get_checksum(self, path_info):
|
||||
@@ -418,10 +438,10 @@ class BaseRemote(object):
|
||||
if self.cache_type_confirmed:
|
||||
return
|
||||
|
||||
is_link = getattr(self, "is_{}".format(link_type), None)
|
||||
is_link = getattr(self, f"is_{link_type}", None)
|
||||
if is_link and not is_link(path_info):
|
||||
self.remove(path_info)
|
||||
raise DvcException("failed to verify {}".format(link_type))
|
||||
raise DvcException(f"failed to verify {link_type}")
|
||||
|
||||
self.cache_type_confirmed = True
|
||||
|
||||
@@ -444,7 +464,7 @@ class BaseRemote(object):
|
||||
|
||||
def _do_link(self, from_info, to_info, link_method):
|
||||
if self.exists(to_info):
|
||||
raise DvcException("Link '{}' already exists!".format(to_info))
|
||||
raise DvcException(f"Link '{to_info}' already exists!")
|
||||
|
||||
link_method(from_info, to_info)
|
||||
|
||||
@@ -452,28 +472,45 @@ class BaseRemote(object):
|
||||
"Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
|
||||
)
|
||||
|
||||
def _save_file(self, path_info, checksum, save_link=True):
|
||||
def _save_file(
|
||||
self, path_info, checksum, save_link=True, tree=None, **kwargs
|
||||
):
|
||||
assert checksum
|
||||
|
||||
cache_info = self.checksum_to_path_info(checksum)
|
||||
if self.changed_cache(checksum):
|
||||
self.move(path_info, cache_info, mode=self.CACHE_MODE)
|
||||
self.link(cache_info, path_info)
|
||||
elif self.iscopy(path_info) and self._cache_is_copy(path_info):
|
||||
# Default relink procedure involves unneeded copy
|
||||
self.unprotect(path_info)
|
||||
if tree:
|
||||
if self.changed_cache(checksum):
|
||||
with tree.open(path_info, mode="rb") as fobj:
|
||||
# if tree has fetch enabled, DVC out will be fetched on
|
||||
# open and we do not need to read/copy any data
|
||||
if not (
|
||||
tree.isdvc(path_info, strict=False) and tree.fetch
|
||||
):
|
||||
self.copy_fobj(fobj, cache_info)
|
||||
callback = kwargs.get("download_callback")
|
||||
if callback:
|
||||
callback(1)
|
||||
else:
|
||||
self.remove(path_info)
|
||||
self.link(cache_info, path_info)
|
||||
if self.changed_cache(checksum):
|
||||
self.move(path_info, cache_info, mode=self.CACHE_MODE)
|
||||
self.link(cache_info, path_info)
|
||||
elif self.iscopy(path_info) and self._cache_is_copy(path_info):
|
||||
# Default relink procedure involves unneeded copy
|
||||
self.unprotect(path_info)
|
||||
else:
|
||||
self.remove(path_info)
|
||||
self.link(cache_info, path_info)
|
||||
|
||||
if save_link:
|
||||
self.state.save_link(path_info)
|
||||
if save_link:
|
||||
self.state.save_link(path_info)
|
||||
|
||||
# we need to update path and cache, since in case of reflink,
|
||||
# or copy cache type moving original file results in updates on
|
||||
# next executed command, which causes md5 recalculation
|
||||
self.state.save(path_info, checksum)
|
||||
if not tree or is_working_tree(tree):
|
||||
self.state.save(path_info, checksum)
|
||||
self.state.save(cache_info, checksum)
|
||||
return {self.PARAM_CHECKSUM: checksum}
|
||||
|
||||
def _cache_is_copy(self, path_info):
|
||||
"""Checks whether cache uses copies."""
|
||||
@@ -497,22 +534,32 @@ class BaseRemote(object):
|
||||
self.cache_type_confirmed = True
|
||||
return self.cache_types[0] == "copy"
|
||||
|
||||
def _save_dir(self, path_info, checksum, save_link=True):
|
||||
def _save_dir(
|
||||
self, path_info, checksum, save_link=True, tree=None, **kwargs
|
||||
):
|
||||
if tree:
|
||||
dir_info = self._collect_dir(
|
||||
path_info, tree=tree, save_tree=True, **kwargs
|
||||
)
|
||||
checksum = self._save_dir_info(dir_info)
|
||||
else:
|
||||
dir_info = self.get_dir_cache(checksum)
|
||||
|
||||
for entry in Tqdm(
|
||||
dir_info, desc="Saving " + path_info.name, unit="file"
|
||||
):
|
||||
entry_info = path_info / entry[self.PARAM_RELPATH]
|
||||
entry_checksum = entry[self.PARAM_CHECKSUM]
|
||||
self._save_file(entry_info, entry_checksum, save_link=False)
|
||||
|
||||
if save_link:
|
||||
self.state.save_link(path_info)
|
||||
|
||||
cache_info = self.checksum_to_path_info(checksum)
|
||||
dir_info = self.get_dir_cache(checksum)
|
||||
|
||||
for entry in Tqdm(
|
||||
dir_info, desc="Saving " + path_info.name, unit="file"
|
||||
):
|
||||
entry_info = path_info / entry[self.PARAM_RELPATH]
|
||||
entry_checksum = entry[self.PARAM_CHECKSUM]
|
||||
self._save_file(entry_info, entry_checksum, save_link=False)
|
||||
|
||||
if save_link:
|
||||
self.state.save_link(path_info)
|
||||
|
||||
self.state.save(cache_info, checksum)
|
||||
self.state.save(path_info, checksum)
|
||||
if not tree or is_working_tree(tree):
|
||||
self.state.save(path_info, checksum)
|
||||
return {self.PARAM_CHECKSUM: checksum}
|
||||
|
||||
def is_empty(self, path_info):
|
||||
return False
|
||||
@@ -541,23 +588,42 @@ class BaseRemote(object):
|
||||
def protect(path_info):
|
||||
pass
|
||||
|
||||
def save(self, path_info, checksum_info, save_link=True):
|
||||
def save(
|
||||
self, path_info, checksum_info, save_link=True, tree=None, **kwargs
|
||||
):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise RemoteActionNotImplemented(
|
||||
"save {} -> {}".format(path_info.scheme, self.scheme),
|
||||
self.scheme,
|
||||
f"save {path_info.scheme} -> {self.scheme}", self.scheme,
|
||||
)
|
||||
|
||||
checksum = checksum_info[self.PARAM_CHECKSUM]
|
||||
self._save(path_info, checksum, save_link)
|
||||
if tree:
|
||||
if tree.isdir(path_info):
|
||||
# save checksum will be computed during tree walk
|
||||
checksum = None
|
||||
else:
|
||||
checksum = tree.get_file_checksum(path_info)
|
||||
else:
|
||||
checksum = checksum_info[self.PARAM_CHECKSUM]
|
||||
return self._save(path_info, checksum, save_link, tree, **kwargs)
|
||||
|
||||
def _save(self, path_info, checksum, save_link=True):
|
||||
to_info = self.checksum_to_path_info(checksum)
|
||||
logger.debug("Saving '%s' to '%s'.", path_info, to_info)
|
||||
if self.isdir(path_info):
|
||||
self._save_dir(path_info, checksum, save_link)
|
||||
return
|
||||
self._save_file(path_info, checksum, save_link)
|
||||
def _save(self, path_info, checksum, save_link=True, tree=None, **kwargs):
|
||||
if tree:
|
||||
logger.debug("Saving tree path '%s' to cache.", path_info)
|
||||
else:
|
||||
to_info = self.checksum_to_path_info(checksum)
|
||||
logger.debug("Saving '%s' to '%s'.", path_info, to_info)
|
||||
|
||||
if tree:
|
||||
isdir = tree.isdir
|
||||
save_link = False
|
||||
else:
|
||||
isdir = self.isdir
|
||||
|
||||
if isdir(path_info):
|
||||
return self._save_dir(
|
||||
path_info, checksum, save_link, tree, **kwargs
|
||||
)
|
||||
return self._save_file(path_info, checksum, save_link, tree, **kwargs)
|
||||
|
||||
def _handle_transfer_exception(
|
||||
self, from_info, to_info, exception, operation
|
||||
@@ -696,6 +762,9 @@ class BaseRemote(object):
|
||||
def copy(self, from_info, to_info):
|
||||
raise RemoteActionNotImplemented("copy", self.scheme)
|
||||
|
||||
def copy_fobj(self, fobj, to_info):
|
||||
raise RemoteActionNotImplemented("copy_fobj", self.scheme)
|
||||
|
||||
def symlink(self, from_info, to_info):
|
||||
raise RemoteActionNotImplemented("symlink", self.scheme)
|
||||
|
||||
@@ -712,7 +781,7 @@ class BaseRemote(object):
|
||||
parts = self.path_cls(path).parts[-2:]
|
||||
|
||||
if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
|
||||
raise ValueError("Bad cache file path '{}'".format(path))
|
||||
raise ValueError(f"Bad cache file path '{path}'")
|
||||
|
||||
return "".join(parts)
|
||||
|
||||
@@ -762,7 +831,7 @@ class BaseRemote(object):
|
||||
|
||||
@index_locked
|
||||
def gc(self, named_cache, jobs=None):
|
||||
used = self.extract_used_local_checksums(named_cache)
|
||||
used = set(named_cache.scheme_keys("local"))
|
||||
|
||||
if self.scheme != "":
|
||||
used.update(named_cache.scheme_keys(self.scheme))
|
||||
@@ -778,6 +847,7 @@ class BaseRemote(object):
|
||||
continue
|
||||
path_info = self.checksum_to_path_info(checksum)
|
||||
if self.is_dir_checksum(checksum):
|
||||
# backward compatibility
|
||||
self._remove_unpacked_dir(checksum)
|
||||
self.remove(path_info)
|
||||
removed = True
|
||||
@@ -836,11 +906,6 @@ class BaseRemote(object):
|
||||
if self.changed_cache_file(checksum):
|
||||
return True
|
||||
|
||||
if not (path_info and filter_info) and not self._changed_unpacked_dir(
|
||||
checksum
|
||||
):
|
||||
return False
|
||||
|
||||
for entry in self.get_dir_cache(checksum):
|
||||
entry_checksum = entry[self.PARAM_CHECKSUM]
|
||||
|
||||
@@ -852,9 +917,6 @@ class BaseRemote(object):
|
||||
if self.changed_cache_file(entry_checksum):
|
||||
return True
|
||||
|
||||
if not (path_info and filter_info):
|
||||
self._update_unpacked_dir(checksum)
|
||||
|
||||
return False
|
||||
|
||||
def changed_cache(self, checksum, path_info=None, filter_info=None):
|
||||
@@ -991,7 +1053,7 @@ class BaseRemote(object):
|
||||
|
||||
with Tqdm(
|
||||
desc="Estimating size of "
|
||||
+ ("cache in '{}'".format(name) if name else "remote cache"),
|
||||
+ (f"cache in '{name}'" if name else "remote cache"),
|
||||
unit="file",
|
||||
) as pbar:
|
||||
|
||||
@@ -1010,7 +1072,7 @@ class BaseRemote(object):
|
||||
remote_size = total_prefixes * len(remote_checksums)
|
||||
else:
|
||||
remote_size = total_prefixes
|
||||
logger.debug("Estimated remote size: {} files".format(remote_size))
|
||||
logger.debug(f"Estimated remote size: {remote_size} files")
|
||||
return remote_size, remote_checksums
|
||||
|
||||
def _cache_checksums_traverse(
|
||||
@@ -1043,7 +1105,7 @@ class BaseRemote(object):
|
||||
else:
|
||||
yield from remote_checksums
|
||||
initial = len(remote_checksums)
|
||||
traverse_prefixes = ["{:02x}".format(i) for i in range(1, 256)]
|
||||
traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
|
||||
if self.TRAVERSE_PREFIX_LEN > 2:
|
||||
traverse_prefixes += [
|
||||
"{0:0{1}x}".format(i, self.TRAVERSE_PREFIX_LEN)
|
||||
@@ -1051,7 +1113,7 @@ class BaseRemote(object):
|
||||
]
|
||||
with Tqdm(
|
||||
desc="Querying "
|
||||
+ ("cache in '{}'".format(name) if name else "remote cache"),
|
||||
+ (f"cache in '{name}'" if name else "remote cache"),
|
||||
total=remote_size,
|
||||
initial=initial,
|
||||
unit="file",
|
||||
@@ -1293,19 +1355,5 @@ class BaseRemote(object):
|
||||
def unprotect(path_info):
|
||||
pass
|
||||
|
||||
def _get_unpacked_dir_names(self, checksums):
|
||||
return set()
|
||||
|
||||
def extract_used_local_checksums(self, named_cache):
|
||||
used = set(named_cache.scheme_keys("local"))
|
||||
unpacked = self._get_unpacked_dir_names(used)
|
||||
return used | unpacked
|
||||
|
||||
def _changed_unpacked_dir(self, checksum):
|
||||
return True
|
||||
|
||||
def _update_unpacked_dir(self, checksum):
|
||||
pass
|
||||
|
||||
def _remove_unpacked_dir(self, checksum):
|
||||
pass
|
||||
|
||||
@@ -22,10 +22,8 @@ FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
|
||||
|
||||
class GDrivePathNotFound(DvcException):
|
||||
def __init__(self, path_info, hint):
|
||||
hint = "" if hint is None else " {}".format(hint)
|
||||
super().__init__(
|
||||
"GDrive path '{}' not found.{}".format(path_info, hint)
|
||||
)
|
||||
hint = "" if hint is None else f" {hint}"
|
||||
super().__init__(f"GDrive path '{path_info}' not found.{hint}")
|
||||
|
||||
|
||||
class GDriveAuthError(DvcException):
|
||||
@@ -62,7 +60,7 @@ def _gdrive_retry(func):
|
||||
"rateLimitExceeded",
|
||||
]
|
||||
if result:
|
||||
logger.debug("Retrying GDrive API call, error: {}.".format(exc))
|
||||
logger.debug(f"Retrying GDrive API call, error: {exc}.")
|
||||
|
||||
return result
|
||||
|
||||
@@ -465,7 +463,7 @@ class GDriveRemote(BaseRemote):
|
||||
return None
|
||||
query = "trashed=false and ({})".format(
|
||||
" or ".join(
|
||||
"'{}' in parents".format(parent_id) for parent_id in parent_ids
|
||||
f"'{parent_id}' in parents" for parent_id in parent_ids
|
||||
)
|
||||
)
|
||||
query += " and title='{}'".format(title.replace("'", "\\'"))
|
||||
@@ -542,9 +540,9 @@ class GDriveRemote(BaseRemote):
|
||||
else:
|
||||
dir_ids = self._ids_cache["ids"]
|
||||
parents_query = " or ".join(
|
||||
"'{}' in parents".format(dir_id) for dir_id in dir_ids
|
||||
f"'{dir_id}' in parents" for dir_id in dir_ids
|
||||
)
|
||||
query = "({}) and trashed=false".format(parents_query)
|
||||
query = f"({parents_query}) and trashed=false"
|
||||
|
||||
for item in self._gdrive_list(query):
|
||||
if progress_callback:
|
||||
|
||||
+4
-5
@@ -1,4 +1,3 @@
|
||||
import io
|
||||
import logging
|
||||
import os.path
|
||||
import posixpath
|
||||
@@ -55,7 +54,7 @@ def _upload_to_bucket(
|
||||
no_progress_bar=False,
|
||||
):
|
||||
blob = bucket.blob(to_info.path, chunk_size=chunk_size)
|
||||
with io.open(from_file, mode="rb") as fobj:
|
||||
with open(from_file, mode="rb") as fobj:
|
||||
with Tqdm.wrapattr(
|
||||
fobj,
|
||||
"read",
|
||||
@@ -110,7 +109,7 @@ class GSRemote(BaseRemote):
|
||||
from_bucket = self.gs.bucket(from_info.bucket)
|
||||
blob = from_bucket.get_blob(from_info.path)
|
||||
if not blob:
|
||||
msg = "'{}' doesn't exist in the cloud".format(from_info.path)
|
||||
msg = f"'{from_info.path}' doesn't exist in the cloud"
|
||||
raise DvcException(msg)
|
||||
|
||||
to_bucket = self.gs.bucket(to_info.bucket)
|
||||
@@ -120,7 +119,7 @@ class GSRemote(BaseRemote):
|
||||
if path_info.scheme != "gs":
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug("Removing gs://{}".format(path_info))
|
||||
logger.debug(f"Removing gs://{path_info}")
|
||||
blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
|
||||
if not blob:
|
||||
return
|
||||
@@ -193,7 +192,7 @@ class GSRemote(BaseRemote):
|
||||
def _download(self, from_info, to_file, name=None, no_progress_bar=False):
|
||||
bucket = self.gs.bucket(from_info.bucket)
|
||||
blob = bucket.get_blob(from_info.path)
|
||||
with io.open(to_file, mode="wb") as fobj:
|
||||
with open(to_file, mode="wb") as fobj:
|
||||
with Tqdm.wrapattr(
|
||||
fobj,
|
||||
"write",
|
||||
|
||||
+7
-5
@@ -55,7 +55,7 @@ class HDFSRemote(BaseRemote):
|
||||
def hadoop_fs(self, cmd, user=None):
|
||||
cmd = "hadoop fs -" + cmd
|
||||
if user:
|
||||
cmd = "HADOOP_USER_NAME={} ".format(user) + cmd
|
||||
cmd = f"HADOOP_USER_NAME={user} " + cmd
|
||||
|
||||
# NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
|
||||
# See https://github.com/iterative/dvc/issues/1197.
|
||||
@@ -87,7 +87,7 @@ class HDFSRemote(BaseRemote):
|
||||
# NOTE: pyarrow doesn't support checksum, so we need to use hadoop
|
||||
regex = r".*\t.*\t(?P<checksum>.*)"
|
||||
stdout = self.hadoop_fs(
|
||||
"checksum {}".format(path_info.path), user=path_info.user
|
||||
f"checksum {path_info.path}", user=path_info.user
|
||||
)
|
||||
return self._group(regex, stdout, "checksum")
|
||||
|
||||
@@ -112,7 +112,7 @@ class HDFSRemote(BaseRemote):
|
||||
raise NotImplementedError
|
||||
|
||||
if self.exists(path_info):
|
||||
logger.debug("Removing {}".format(path_info.path))
|
||||
logger.debug(f"Removing {path_info.path}")
|
||||
with self.hdfs(path_info) as hdfs:
|
||||
hdfs.rm(path_info.path)
|
||||
|
||||
@@ -147,7 +147,7 @@ class HDFSRemote(BaseRemote):
|
||||
yield fd
|
||||
else:
|
||||
yield io.TextIOWrapper(fd, encoding=encoding)
|
||||
except IOError as e:
|
||||
except OSError as e:
|
||||
# Empty .errno and not specific enough error class in pyarrow,
|
||||
# see https://issues.apache.org/jira/browse/ARROW-6248
|
||||
if "file does not exist" in str(e):
|
||||
@@ -165,6 +165,8 @@ class HDFSRemote(BaseRemote):
|
||||
dirs = deque([root])
|
||||
|
||||
with self.hdfs(self.path_info) as hdfs:
|
||||
if prefix and not hdfs.exists(root):
|
||||
return
|
||||
while dirs:
|
||||
try:
|
||||
entries = hdfs.ls(dirs.pop(), detail=True)
|
||||
@@ -175,7 +177,7 @@ class HDFSRemote(BaseRemote):
|
||||
if progress_callback:
|
||||
progress_callback()
|
||||
yield urlparse(entry["name"]).path
|
||||
except IOError as e:
|
||||
except OSError as e:
|
||||
# When searching for a specific prefix pyarrow raises an
|
||||
# exception if the specified cache dir does not exist
|
||||
if not prefix:
|
||||
|
||||
+1
-1
@@ -185,7 +185,7 @@ class HTTPRemote(BaseRemote):
|
||||
return res
|
||||
|
||||
except requests.exceptions.RequestException:
|
||||
raise DvcException("could not perform a {} request".format(method))
|
||||
raise DvcException(f"could not perform a {method} request")
|
||||
|
||||
def gc(self):
|
||||
raise NotImplementedError
|
||||
|
||||
+4
-6
@@ -69,9 +69,7 @@ class RemoteIndex:
|
||||
INDEX_TABLE_LAYOUT = "checksum TEXT PRIMARY KEY, " "dir INTEGER NOT NULL"
|
||||
|
||||
def __init__(self, repo, name, dir_suffix=".dir"):
|
||||
self.path = os.path.join(
|
||||
repo.index_dir, "{}{}".format(name, self.INDEX_SUFFIX)
|
||||
)
|
||||
self.path = os.path.join(repo.index_dir, f"{name}{self.INDEX_SUFFIX}")
|
||||
|
||||
self.dir_suffix = dir_suffix
|
||||
self.database = None
|
||||
@@ -80,7 +78,7 @@ class RemoteIndex:
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def __iter__(self):
|
||||
cmd = "SELECT checksum FROM {}".format(self.INDEX_TABLE)
|
||||
cmd = f"SELECT checksum FROM {self.INDEX_TABLE}"
|
||||
for (checksum,) in self._execute(cmd):
|
||||
yield checksum
|
||||
|
||||
@@ -105,7 +103,7 @@ class RemoteIndex:
|
||||
|
||||
def dir_checksums(self):
|
||||
"""Iterate over .dir checksums stored in the index."""
|
||||
cmd = "SELECT checksum FROM {} WHERE dir = 1".format(self.INDEX_TABLE)
|
||||
cmd = f"SELECT checksum FROM {self.INDEX_TABLE} WHERE dir = 1"
|
||||
for (checksum,) in self._execute(cmd):
|
||||
yield checksum
|
||||
|
||||
@@ -184,7 +182,7 @@ class RemoteIndex:
|
||||
|
||||
Changes to the index will not committed until dump() is called.
|
||||
"""
|
||||
cmd = "DELETE FROM {}".format(self.INDEX_TABLE)
|
||||
cmd = f"DELETE FROM {self.INDEX_TABLE}"
|
||||
self._execute(cmd)
|
||||
|
||||
def update(self, dir_checksums, file_checksums):
|
||||
|
||||
+84
-93
@@ -8,7 +8,6 @@ from functools import partial
|
||||
from funcy import cached_property, concat
|
||||
from shortuuid import uuid
|
||||
|
||||
from dvc.compat import fspath_py35
|
||||
from dvc.exceptions import DownloadError, DvcException, UploadError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.progress import Tqdm
|
||||
@@ -22,10 +21,17 @@ from dvc.remote.base import (
|
||||
)
|
||||
from dvc.remote.index import RemoteIndexNoop
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.scm.tree import is_working_tree
|
||||
from dvc.scm.tree import WorkingTree, is_working_tree
|
||||
from dvc.system import System
|
||||
from dvc.utils import file_md5, relpath, tmp_fname
|
||||
from dvc.utils.fs import copyfile, makedirs, move, remove, walk_files
|
||||
from dvc.utils.fs import (
|
||||
copy_fobj_to_file,
|
||||
copyfile,
|
||||
makedirs,
|
||||
move,
|
||||
remove,
|
||||
walk_files,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -62,6 +68,21 @@ class LocalRemote(BaseRemote):
|
||||
def cache_dir(self, value):
|
||||
self.path_info = PathInfo(value) if value else None
|
||||
|
||||
@cached_property
|
||||
def _work_tree(self):
|
||||
if self.repo:
|
||||
return WorkingTree(self.repo.root_dir)
|
||||
return None
|
||||
|
||||
@property
|
||||
def work_tree(self):
|
||||
# When using repo.brancher, repo.tree may change to/from WorkingTree to
|
||||
# GitTree arbitarily. When repo.tree is GitTree, local cache needs to
|
||||
# use its own WorkingTree instance.
|
||||
if self.repo and not is_working_tree(self.repo.tree):
|
||||
return self._work_tree
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def supported(cls, config):
|
||||
return True
|
||||
@@ -71,12 +92,19 @@ class LocalRemote(BaseRemote):
|
||||
return os.path.abspath(self.cache_dir)
|
||||
|
||||
def checksum_to_path(self, checksum):
|
||||
return os.path.join(self.cache_path, checksum[0:2], checksum[2:])
|
||||
# NOTE: `self.cache_path` is already normalized so we can simply use
|
||||
# `os.sep` instead of `os.path.join`. This results in this helper
|
||||
# being ~5.5 times faster.
|
||||
return (
|
||||
f"{self.cache_path}{os.sep}{checksum[0:2]}{os.sep}{checksum[2:]}"
|
||||
)
|
||||
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
assert self.path_info is not None
|
||||
if prefix:
|
||||
path_info = self.path_info / prefix[:2]
|
||||
if not self.exists(path_info):
|
||||
return
|
||||
else:
|
||||
path_info = self.path_info
|
||||
if progress_callback:
|
||||
@@ -93,9 +121,12 @@ class LocalRemote(BaseRemote):
|
||||
return self.checksum_to_path_info(md5).url
|
||||
|
||||
def exists(self, path_info):
|
||||
assert is_working_tree(self.repo.tree)
|
||||
assert isinstance(path_info, str) or path_info.scheme == "local"
|
||||
return self.repo.tree.exists(fspath_py35(path_info))
|
||||
if not self.repo:
|
||||
return os.path.exists(path_info)
|
||||
if self.work_tree and self.work_tree.exists(path_info):
|
||||
return True
|
||||
return self.repo.tree.exists(path_info)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
makedirs(path_info, exist_ok=True, mode=self._dir_mode)
|
||||
@@ -127,13 +158,19 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def isfile(path_info):
|
||||
return os.path.isfile(fspath_py35(path_info))
|
||||
def isfile(self, path_info):
|
||||
if not self.repo:
|
||||
return os.path.isfile(path_info)
|
||||
if self.work_tree and self.work_tree.isfile(path_info):
|
||||
return True
|
||||
return self.repo.tree.isfile(path_info)
|
||||
|
||||
@staticmethod
|
||||
def isdir(path_info):
|
||||
return os.path.isdir(fspath_py35(path_info))
|
||||
def isdir(self, path_info):
|
||||
if not self.repo:
|
||||
return os.path.isdir(path_info)
|
||||
if self.work_tree and self.work_tree.isdir(path_info):
|
||||
return True
|
||||
return self.repo.tree.isdir(path_info)
|
||||
|
||||
def iscopy(self, path_info):
|
||||
return not (
|
||||
@@ -142,12 +179,14 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
@staticmethod
|
||||
def getsize(path_info):
|
||||
return os.path.getsize(fspath_py35(path_info))
|
||||
return os.path.getsize(path_info)
|
||||
|
||||
def walk_files(self, path_info):
|
||||
assert is_working_tree(self.repo.tree)
|
||||
|
||||
for fname in self.repo.tree.walk_files(path_info):
|
||||
if self.work_tree:
|
||||
tree = self.work_tree
|
||||
else:
|
||||
tree = self.repo.tree
|
||||
for fname in tree.walk_files(path_info):
|
||||
yield PathInfo(fname)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
@@ -182,8 +221,19 @@ class LocalRemote(BaseRemote):
|
||||
tmp_info = to_info.parent / tmp_fname(to_info.name)
|
||||
try:
|
||||
System.copy(from_info, tmp_info)
|
||||
os.chmod(fspath_py35(tmp_info), self._file_mode)
|
||||
os.rename(fspath_py35(tmp_info), fspath_py35(to_info))
|
||||
os.chmod(tmp_info, self._file_mode)
|
||||
os.rename(tmp_info, to_info)
|
||||
except Exception:
|
||||
self.remove(tmp_info)
|
||||
raise
|
||||
|
||||
def copy_fobj(self, fobj, to_info):
|
||||
self.makedirs(to_info.parent)
|
||||
tmp_info = to_info.parent / tmp_fname(to_info.name)
|
||||
try:
|
||||
copy_fobj_to_file(fobj, tmp_info)
|
||||
os.chmod(tmp_info, self._file_mode)
|
||||
os.rename(tmp_info, to_info)
|
||||
except Exception:
|
||||
self.remove(tmp_info)
|
||||
raise
|
||||
@@ -231,8 +281,8 @@ class LocalRemote(BaseRemote):
|
||||
System.reflink(from_info, tmp_info)
|
||||
# NOTE: reflink has its own separate inode, so you can set permissions
|
||||
# that are different from the source.
|
||||
os.chmod(fspath_py35(tmp_info), self._file_mode)
|
||||
os.rename(fspath_py35(tmp_info), fspath_py35(to_info))
|
||||
os.chmod(tmp_info, self._file_mode)
|
||||
os.rename(tmp_info, to_info)
|
||||
|
||||
def cache_exists(self, checksums, jobs=None, name=None):
|
||||
return [
|
||||
@@ -255,7 +305,9 @@ class LocalRemote(BaseRemote):
|
||||
copyfile(
|
||||
from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
|
||||
)
|
||||
os.rename(tmp_file, fspath_py35(to_info))
|
||||
|
||||
self.protect(tmp_file)
|
||||
os.rename(tmp_file, to_info)
|
||||
|
||||
def _download(
|
||||
self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
|
||||
@@ -266,7 +318,7 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
@staticmethod
|
||||
def open(path_info, mode="r", encoding=None):
|
||||
return open(fspath_py35(path_info), mode=mode, encoding=encoding)
|
||||
return open(path_info, mode=mode, encoding=encoding)
|
||||
|
||||
@index_locked
|
||||
def status(
|
||||
@@ -302,9 +354,7 @@ class LocalRemote(BaseRemote):
|
||||
{dir_checksum: set(file_checksum, ...)} which can be used to map
|
||||
a .dir file to its file contents.
|
||||
"""
|
||||
logger.debug(
|
||||
"Preparing to collect status from {}".format(remote.path_info)
|
||||
)
|
||||
logger.debug(f"Preparing to collect status from {remote.path_info}")
|
||||
md5s = set(named_cache.scheme_keys(self.scheme))
|
||||
|
||||
logger.debug("Collecting information from local cache...")
|
||||
@@ -550,9 +600,7 @@ class LocalRemote(BaseRemote):
|
||||
"failed to upload full contents of '{}', "
|
||||
"aborting .dir file upload".format(name)
|
||||
)
|
||||
logger.error(
|
||||
"failed to upload '{}' to '{}'".format(from_info, to_info)
|
||||
)
|
||||
logger.error(f"failed to upload '{from_info}' to '{to_info}'")
|
||||
return 1
|
||||
return func(from_info, to_info, name)
|
||||
|
||||
@@ -596,7 +644,7 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
def _unprotect_file(self, path):
|
||||
if System.is_symlink(path) or System.is_hardlink(path):
|
||||
logger.debug("Unprotecting '{}'".format(path))
|
||||
logger.debug(f"Unprotecting '{path}'")
|
||||
tmp = os.path.join(os.path.dirname(path), "." + uuid())
|
||||
|
||||
# The operations order is important here - if some application
|
||||
@@ -625,9 +673,7 @@ class LocalRemote(BaseRemote):
|
||||
def unprotect(self, path_info):
|
||||
path = path_info.fspath
|
||||
if not os.path.exists(path):
|
||||
raise DvcException(
|
||||
"can't unprotect non-existing data '{}'".format(path)
|
||||
)
|
||||
raise DvcException(f"can't unprotect non-existing data '{path}'")
|
||||
|
||||
if os.path.isdir(path):
|
||||
self._unprotect_dir(path)
|
||||
@@ -635,7 +681,7 @@ class LocalRemote(BaseRemote):
|
||||
self._unprotect_file(path)
|
||||
|
||||
def protect(self, path_info):
|
||||
path = fspath_py35(path_info)
|
||||
path = os.fspath(path_info)
|
||||
mode = self.CACHE_MODE
|
||||
|
||||
try:
|
||||
@@ -654,70 +700,15 @@ class LocalRemote(BaseRemote):
|
||||
if actual != mode:
|
||||
raise
|
||||
|
||||
def _get_unpacked_dir_path_info(self, checksum):
|
||||
info = self.checksum_to_path_info(checksum)
|
||||
return info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
|
||||
|
||||
def _remove_unpacked_dir(self, checksum):
|
||||
path_info = self._get_unpacked_dir_path_info(checksum)
|
||||
info = self.checksum_to_path_info(checksum)
|
||||
path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
|
||||
self.remove(path_info)
|
||||
|
||||
def _path_info_changed(self, path_info):
|
||||
if self.exists(path_info) and self.state.get(path_info):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _update_unpacked_dir(self, checksum):
|
||||
unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
|
||||
|
||||
if not self._path_info_changed(unpacked_dir_info):
|
||||
return
|
||||
|
||||
self.remove(unpacked_dir_info)
|
||||
|
||||
try:
|
||||
dir_info = self.get_dir_cache(checksum)
|
||||
self._create_unpacked_dir(checksum, dir_info, unpacked_dir_info)
|
||||
except DvcException:
|
||||
logger.warning("Could not create '{}'".format(unpacked_dir_info))
|
||||
|
||||
self.remove(unpacked_dir_info)
|
||||
|
||||
def _create_unpacked_dir(self, checksum, dir_info, unpacked_dir_info):
|
||||
self.makedirs(unpacked_dir_info)
|
||||
|
||||
for entry in Tqdm(dir_info, desc="Creating unpacked dir", unit="file"):
|
||||
entry_cache_info = self.checksum_to_path_info(
|
||||
entry[self.PARAM_CHECKSUM]
|
||||
)
|
||||
relative_path = entry[self.PARAM_RELPATH]
|
||||
# In shared cache mode some cache files might not be owned by the
|
||||
# user, so we need to use symlinks because, unless
|
||||
# /proc/sys/fs/protected_hardlinks is disabled, the user is not
|
||||
# allowed to create hardlinks to files that he doesn't own.
|
||||
link_types = ["hardlink", "symlink"]
|
||||
self._link(
|
||||
entry_cache_info, unpacked_dir_info / relative_path, link_types
|
||||
)
|
||||
|
||||
self.state.save(unpacked_dir_info, checksum)
|
||||
|
||||
def _changed_unpacked_dir(self, checksum):
|
||||
status_unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
|
||||
|
||||
return not self.state.get(status_unpacked_dir_info)
|
||||
|
||||
def _get_unpacked_dir_names(self, checksums):
|
||||
unpacked = set()
|
||||
for c in checksums:
|
||||
if self.is_dir_checksum(c):
|
||||
unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
|
||||
return unpacked
|
||||
|
||||
def is_protected(self, path_info):
|
||||
if not self.exists(path_info):
|
||||
try:
|
||||
mode = os.stat(path_info).st_mode
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
mode = os.stat(fspath_py35(path_info)).st_mode
|
||||
|
||||
return stat.S_IMODE(mode) == self.CACHE_MODE
|
||||
|
||||
+4
-4
@@ -64,9 +64,9 @@ class OSSRemote(BaseRemote):
|
||||
def oss_service(self):
|
||||
import oss2
|
||||
|
||||
logger.debug("URL: {}".format(self.path_info))
|
||||
logger.debug("key id: {}".format(self.key_id))
|
||||
logger.debug("key secret: {}".format(self.key_secret))
|
||||
logger.debug(f"URL: {self.path_info}")
|
||||
logger.debug(f"key id: {self.key_id}")
|
||||
logger.debug(f"key secret: {self.key_secret}")
|
||||
|
||||
auth = oss2.Auth(self.key_id, self.key_secret)
|
||||
bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)
|
||||
@@ -87,7 +87,7 @@ class OSSRemote(BaseRemote):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug("Removing oss://{}".format(path_info))
|
||||
logger.debug(f"Removing oss://{path_info}")
|
||||
self.oss_service.delete_object(path_info.path)
|
||||
|
||||
def _list_paths(self, prefix, progress_callback=None):
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ def close_pools():
|
||||
get_pool.memory.clear()
|
||||
|
||||
|
||||
class Pool(object):
|
||||
class Pool:
|
||||
def __init__(self, conn_func, *conn_args, **conn_kwargs):
|
||||
self._conn_func = conn_func
|
||||
self._conn_args = conn_args
|
||||
|
||||
+7
-7
@@ -1,5 +1,3 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
@@ -46,6 +44,10 @@ class S3Remote(BaseRemote):
|
||||
if self.sse:
|
||||
self.extra_args["ServerSideEncryption"] = self.sse
|
||||
|
||||
self.sse_kms_key_id = config.get("sse_kms_key_id")
|
||||
if self.sse_kms_key_id:
|
||||
self.extra_args["SSEKMSKeyId"] = self.sse_kms_key_id
|
||||
|
||||
self.acl = config.get("acl")
|
||||
if self.acl:
|
||||
self.extra_args["ACL"] = self.acl
|
||||
@@ -84,9 +86,7 @@ class S3Remote(BaseRemote):
|
||||
try:
|
||||
obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
|
||||
except Exception as exc:
|
||||
raise DvcException(
|
||||
"s3://{}/{} does not exist".format(bucket, path)
|
||||
) from exc
|
||||
raise DvcException(f"s3://{bucket}/{path} does not exist") from exc
|
||||
return obj
|
||||
|
||||
@classmethod
|
||||
@@ -109,7 +109,7 @@ class S3Remote(BaseRemote):
|
||||
if lastbyte > size:
|
||||
lastbyte = size - 1
|
||||
|
||||
srange = "bytes={}-{}".format(byte_position, lastbyte)
|
||||
srange = f"bytes={byte_position}-{lastbyte}"
|
||||
|
||||
part = s3.upload_part_copy(
|
||||
Bucket=to_info.bucket,
|
||||
@@ -187,7 +187,7 @@ class S3Remote(BaseRemote):
|
||||
if path_info.scheme != "s3":
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug("Removing {}".format(path_info))
|
||||
logger.debug(f"Removing {path_info}")
|
||||
self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
|
||||
|
||||
def _list_objects(
|
||||
|
||||
@@ -265,6 +265,8 @@ class SSHRemote(BaseRemote):
|
||||
else:
|
||||
root = self.path_info.path
|
||||
with self.ssh(self.path_info) as ssh:
|
||||
if prefix and not ssh.exists(root):
|
||||
return
|
||||
# If we simply return an iterator then with above closes instantly
|
||||
if progress_callback:
|
||||
for path in ssh.walk_files(root):
|
||||
@@ -290,7 +292,7 @@ class SSHRemote(BaseRemote):
|
||||
try:
|
||||
channel.stat(path)
|
||||
ret.append(True)
|
||||
except IOError as exc:
|
||||
except OSError as exc:
|
||||
if exc.errno != errno.ENOENT:
|
||||
raise
|
||||
ret.append(False)
|
||||
|
||||
@@ -25,9 +25,9 @@ def sizeof_fmt(num, suffix="B"):
|
||||
""" Convert number of bytes to human-readable string """
|
||||
for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
|
||||
if abs(num) < 1024.0:
|
||||
return "%3.1f%s%s" % (num, unit, suffix)
|
||||
return f"{num:3.1f}{unit}{suffix}"
|
||||
num /= 1024.0
|
||||
return "%.1f%s%s" % (num, "Y", suffix)
|
||||
return "{:.1f}{}{}".format(num, "Y", suffix)
|
||||
|
||||
|
||||
class SSHConnection:
|
||||
@@ -100,7 +100,7 @@ class SSHConnection:
|
||||
|
||||
if stat.S_ISREG(st_mode) or stat.S_ISLNK(st_mode):
|
||||
raise DvcException(
|
||||
"a file with the same name '{}' already exists".format(path)
|
||||
f"a file with the same name '{path}' already exists"
|
||||
)
|
||||
|
||||
head, tail = posixpath.split(path)
|
||||
@@ -111,12 +111,12 @@ class SSHConnection:
|
||||
if tail:
|
||||
try:
|
||||
self.sftp.mkdir(path)
|
||||
except IOError as exc:
|
||||
except OSError as exc:
|
||||
# Since paramiko errors are very vague we need to recheck
|
||||
# whether it's because path already exists or something else
|
||||
if exc.errno == errno.EACCES or not self.exists(path):
|
||||
raise DvcException(
|
||||
"unable to create remote directory '{}'".format(path)
|
||||
f"unable to create remote directory '{path}'"
|
||||
) from exc
|
||||
|
||||
def walk(self, directory, topdown=True):
|
||||
@@ -126,7 +126,7 @@ class SSHConnection:
|
||||
# [1] https://github.com/python/cpython/blob/master/Lib/os.py
|
||||
try:
|
||||
dir_entries = self.sftp.listdir_attr(directory)
|
||||
except IOError as exc:
|
||||
except OSError as exc:
|
||||
raise DvcException(
|
||||
"couldn't get the '{}' remote directory files list".format(
|
||||
directory
|
||||
@@ -292,14 +292,14 @@ class SSHConnection:
|
||||
md5 = self.execute("md5 " + path).split()[-1]
|
||||
else:
|
||||
raise DvcException(
|
||||
"'{}' is not supported as a SSH remote".format(self.uname)
|
||||
f"'{self.uname}' is not supported as a SSH remote"
|
||||
)
|
||||
|
||||
assert len(md5) == 32
|
||||
return md5
|
||||
|
||||
def copy(self, src, dest):
|
||||
self.execute("cp {} {}".format(src, dest))
|
||||
self.execute(f"cp {src} {dest}")
|
||||
|
||||
def open_max_sftp_channels(self):
|
||||
# If there are more than 1 it means we've already opened max amount
|
||||
@@ -321,14 +321,12 @@ class SSHConnection:
|
||||
|
||||
def reflink(self, src, dest):
|
||||
if self.uname == "Linux":
|
||||
return self.execute("cp --reflink {} {}".format(src, dest))
|
||||
return self.execute(f"cp --reflink {src} {dest}")
|
||||
|
||||
if self.uname == "Darwin":
|
||||
return self.execute("cp -c {} {}".format(src, dest))
|
||||
return self.execute(f"cp -c {src} {dest}")
|
||||
|
||||
raise DvcException(
|
||||
"'{}' is not supported as a SSH remote".format(self.uname)
|
||||
)
|
||||
raise DvcException(f"'{self.uname}' is not supported as a SSH remote")
|
||||
|
||||
def hardlink(self, src, dest):
|
||||
self.execute("ln {} {}".format(src, dest))
|
||||
self.execute(f"ln {src} {dest}")
|
||||
|
||||
+148
-90
@@ -5,22 +5,26 @@ from functools import wraps
|
||||
|
||||
from funcy import cached_property, cat, first
|
||||
|
||||
from dvc.compat import fspath_py35
|
||||
from dvc.config import Config
|
||||
from dvc.dvcfile import PIPELINE_FILE, Dvcfile, is_valid_filename
|
||||
from dvc.exceptions import FileMissingError
|
||||
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
|
||||
from dvc.exceptions import (
|
||||
FileMissingError,
|
||||
IsADirectoryError,
|
||||
NoOutputOrStageError,
|
||||
NotDvcRepoError,
|
||||
OutputNotFoundError,
|
||||
)
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.remote.base import RemoteActionNotImplemented
|
||||
from dvc.repo.tree import RepoTree
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
|
||||
from ..utils import parse_target
|
||||
from .graph import check_acyclic, get_pipeline, get_pipelines
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def locked(f):
|
||||
@wraps(f)
|
||||
@@ -35,7 +39,7 @@ def locked(f):
|
||||
return wrapper
|
||||
|
||||
|
||||
class Repo(object):
|
||||
class Repo:
|
||||
DVC_DIR = ".dvc"
|
||||
|
||||
from dvc.repo.destroy import destroy
|
||||
@@ -61,31 +65,38 @@ class Repo(object):
|
||||
from dvc.repo.get import get
|
||||
from dvc.repo.get_url import get_url
|
||||
from dvc.repo.update import update
|
||||
from dvc.repo.plot import plot
|
||||
|
||||
def __init__(self, root_dir=None):
|
||||
from dvc.state import State
|
||||
def __init__(self, root_dir=None, scm=None, rev=None):
|
||||
from dvc.state import State, StateNoop
|
||||
from dvc.lock import make_lock
|
||||
from dvc.scm import SCM
|
||||
from dvc.cache import Cache
|
||||
from dvc.data_cloud import DataCloud
|
||||
from dvc.repo.metrics import Metrics
|
||||
from dvc.repo.plots import Plots
|
||||
from dvc.repo.params import Params
|
||||
from dvc.scm.tree import WorkingTree
|
||||
from dvc.utils.fs import makedirs
|
||||
from dvc.stage.cache import StageCache
|
||||
|
||||
root_dir = self.find_root(root_dir)
|
||||
if scm:
|
||||
# use GitTree instead of WorkingTree as default repo tree instance
|
||||
tree = scm.get_tree(rev)
|
||||
self.root_dir = self.find_root(root_dir, tree)
|
||||
self.scm = scm
|
||||
self.tree = tree
|
||||
self.state = StateNoop()
|
||||
else:
|
||||
root_dir = self.find_root(root_dir)
|
||||
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
|
||||
|
||||
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
|
||||
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
|
||||
|
||||
self.config = Config(self.dvc_dir)
|
||||
|
||||
no_scm = self.config["core"].get("no_scm", False)
|
||||
self.scm = SCM(self.root_dir, no_scm=no_scm)
|
||||
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
if not scm:
|
||||
no_scm = self.config["core"].get("no_scm", False)
|
||||
self.scm = SCM(self.root_dir, no_scm=no_scm)
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
|
||||
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
|
||||
self.index_dir = os.path.join(self.tmp_dir, "index")
|
||||
@@ -99,9 +110,11 @@ class Repo(object):
|
||||
friendly=True,
|
||||
)
|
||||
|
||||
# NOTE: storing state and link_state in the repository itself to avoid
|
||||
# any possible state corruption in 'shared cache dir' scenario.
|
||||
self.state = State(self)
|
||||
if not scm:
|
||||
# NOTE: storing state and link_state in the repository itself to
|
||||
# avoid any possible state corruption in 'shared cache dir'
|
||||
# scenario.
|
||||
self.state = State(self)
|
||||
|
||||
self.cache = Cache(self)
|
||||
self.cloud = DataCloud(self)
|
||||
@@ -109,6 +122,7 @@ class Repo(object):
|
||||
self.stage_cache = StageCache(self)
|
||||
|
||||
self.metrics = Metrics(self)
|
||||
self.plots = Plots(self)
|
||||
self.params = Params(self)
|
||||
|
||||
self._ignore()
|
||||
@@ -125,14 +139,19 @@ class Repo(object):
|
||||
self._reset()
|
||||
|
||||
def __repr__(self):
|
||||
return "{}: '{}'".format(self.__class__.__name__, self.root_dir)
|
||||
return f"{self.__class__.__name__}: '{self.root_dir}'"
|
||||
|
||||
@classmethod
|
||||
def find_root(cls, root=None):
|
||||
def find_root(cls, root=None, tree=None):
|
||||
root_dir = os.path.realpath(root or os.curdir)
|
||||
|
||||
if tree:
|
||||
if tree.isdir(os.path.join(root_dir, cls.DVC_DIR)):
|
||||
return root_dir
|
||||
raise NotDvcRepoError(f"'{root}' does not contain DVC directory")
|
||||
|
||||
if not os.path.isdir(root_dir):
|
||||
raise NotDvcRepoError("directory '{}' does not exist".format(root))
|
||||
raise NotDvcRepoError(f"directory '{root}' does not exist")
|
||||
|
||||
while True:
|
||||
dvc_dir = os.path.join(root_dir, cls.DVC_DIR)
|
||||
@@ -171,6 +190,25 @@ class Repo(object):
|
||||
|
||||
self.scm.ignore_list(flist)
|
||||
|
||||
def get_stage(self, path=None, name=None):
|
||||
if not path:
|
||||
path = PIPELINE_FILE
|
||||
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
|
||||
|
||||
dvcfile = Dvcfile(self, path)
|
||||
return dvcfile.stages[name]
|
||||
|
||||
def get_stages(self, path=None, name=None):
|
||||
if not path:
|
||||
path = PIPELINE_FILE
|
||||
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
|
||||
|
||||
if name:
|
||||
return [self.get_stage(path, name)]
|
||||
|
||||
dvcfile = Dvcfile(self, path)
|
||||
return list(dvcfile.stages.values())
|
||||
|
||||
def check_modified_graph(self, new_stages):
|
||||
"""Generate graph including the new stage to check for errors"""
|
||||
# Building graph might be costly for the ones with many DVC-files,
|
||||
@@ -194,10 +232,9 @@ class Repo(object):
|
||||
stages = nx.dfs_postorder_nodes(graph)
|
||||
return [stage for stage in stages if path_isin(stage.path, path)]
|
||||
|
||||
def collect(self, target, with_deps=False, recursive=False, graph=None):
|
||||
import networkx as nx
|
||||
from ..dvcfile import Dvcfile
|
||||
|
||||
def collect(
|
||||
self, target=None, with_deps=False, recursive=False, graph=None
|
||||
):
|
||||
if not target:
|
||||
return list(graph) if graph else self.stages
|
||||
|
||||
@@ -207,36 +244,81 @@ class Repo(object):
|
||||
)
|
||||
|
||||
path, name = parse_target(target)
|
||||
dvcfile = Dvcfile(self, path)
|
||||
stages = list(dvcfile.stages.filter(name).values())
|
||||
stages = self.get_stages(path, name)
|
||||
if not with_deps:
|
||||
return stages
|
||||
|
||||
res = set()
|
||||
for stage in stages:
|
||||
pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
|
||||
res.update(nx.dfs_postorder_nodes(pipeline, stage))
|
||||
res.update(self._collect_pipeline(stage, graph=graph))
|
||||
return res
|
||||
|
||||
def collect_granular(self, target, *args, **kwargs):
|
||||
from ..dvcfile import Dvcfile, is_valid_filename
|
||||
def _collect_pipeline(self, stage, graph=None):
|
||||
import networkx as nx
|
||||
|
||||
pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
|
||||
return nx.dfs_postorder_nodes(pipeline, stage)
|
||||
|
||||
def _collect_from_default_dvcfile(self, target):
|
||||
dvcfile = Dvcfile(self, PIPELINE_FILE)
|
||||
if dvcfile.exists():
|
||||
return dvcfile.stages.get(target)
|
||||
|
||||
def collect_granular(
|
||||
self, target=None, with_deps=False, recursive=False, graph=None
|
||||
):
|
||||
"""
|
||||
Priority is in the order of following in case of ambiguity:
|
||||
- .dvc file or .yaml file
|
||||
- dir if recursive and directory exists
|
||||
- stage_name
|
||||
- output file
|
||||
"""
|
||||
if not target:
|
||||
return [(stage, None) for stage in self.stages]
|
||||
|
||||
file, name = parse_target(target)
|
||||
if is_valid_filename(file) and not kwargs.get("with_deps"):
|
||||
# Optimization: do not collect the graph for a specific .dvc target
|
||||
stages = Dvcfile(self, file).stages.filter(name)
|
||||
return [(stage, None) for stage in stages.values()]
|
||||
stages = []
|
||||
|
||||
try:
|
||||
(out,) = self.find_outs_by_path(file, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(file))
|
||||
return [(out.stage, filter_info)]
|
||||
except OutputNotFoundError:
|
||||
stages = self.collect(target, *args, **kwargs)
|
||||
return [(stage, None) for stage in stages]
|
||||
# Optimization: do not collect the graph for a specific target
|
||||
if not file:
|
||||
# parsing is ambiguous when it does not have a colon
|
||||
# or if it's not a dvcfile, as it can be a stage name
|
||||
# in `dvc.yaml` or, an output in a stage.
|
||||
logger.debug(
|
||||
"Checking if stage '%s' is in '%s'", target, PIPELINE_FILE
|
||||
)
|
||||
if not (recursive and os.path.isdir(target)):
|
||||
stage = self._collect_from_default_dvcfile(target)
|
||||
if stage:
|
||||
stages = (
|
||||
self._collect_pipeline(stage) if with_deps else [stage]
|
||||
)
|
||||
elif not with_deps and is_valid_filename(file):
|
||||
stages = self.get_stages(file, name)
|
||||
|
||||
if not stages:
|
||||
if not (recursive and os.path.isdir(target)):
|
||||
try:
|
||||
(out,) = self.find_outs_by_path(target, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(target))
|
||||
return [(out.stage, filter_info)]
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
|
||||
try:
|
||||
stages = self.collect(target, with_deps, recursive, graph)
|
||||
except StageFileDoesNotExistError as exc:
|
||||
# collect() might try to use `target` as a stage name
|
||||
# and throw error that dvc.yaml does not exist, whereas it
|
||||
# should say that both stage name and file does not exist.
|
||||
if file and is_valid_filename(file):
|
||||
raise
|
||||
raise NoOutputOrStageError(target, exc.file) from exc
|
||||
except StageNotFound as exc:
|
||||
raise NoOutputOrStageError(target, exc.file) from exc
|
||||
|
||||
return [(stage, None) for stage in stages]
|
||||
|
||||
def used_cache(
|
||||
self,
|
||||
@@ -249,6 +331,7 @@ class Repo(object):
|
||||
force=False,
|
||||
jobs=None,
|
||||
recursive=False,
|
||||
used_run_cache=None,
|
||||
):
|
||||
"""Get the stages related to the given target and collect
|
||||
the `info` of its outputs.
|
||||
@@ -282,7 +365,7 @@ class Repo(object):
|
||||
for target in targets
|
||||
)
|
||||
|
||||
suffix = "({})".format(branch) if branch else ""
|
||||
suffix = f"({branch})" if branch else ""
|
||||
for stage, filter_info in pairs:
|
||||
used_cache = stage.get_used_cache(
|
||||
remote=remote,
|
||||
@@ -292,6 +375,12 @@ class Repo(object):
|
||||
)
|
||||
cache.update(used_cache, suffix=suffix)
|
||||
|
||||
if used_run_cache:
|
||||
used_cache = self.stage_cache.get_used_cache(
|
||||
used_run_cache, remote=remote, force=force, jobs=jobs,
|
||||
)
|
||||
cache.update(used_cache)
|
||||
|
||||
return cache
|
||||
|
||||
def _collect_graph(self, stages):
|
||||
@@ -421,22 +510,18 @@ class Repo(object):
|
||||
|
||||
@cached_property
|
||||
def plot_templates(self):
|
||||
from dvc.repo.plot.template import PlotTemplates
|
||||
from .plots.template import PlotTemplates
|
||||
|
||||
return PlotTemplates(self.dvc_dir)
|
||||
|
||||
def _collect_stages(self):
|
||||
from dvc.dvcfile import Dvcfile, is_valid_filename
|
||||
|
||||
stages = []
|
||||
outs = set()
|
||||
|
||||
for root, dirs, files in self.tree.walk(self.root_dir):
|
||||
for file_name in filter(is_valid_filename, files):
|
||||
path = os.path.join(root, file_name)
|
||||
stage_loader = Dvcfile(self, path).stages
|
||||
with stage_loader.log_level(at=logging.DEBUG):
|
||||
stages.extend(stage_loader.values())
|
||||
stages.extend(self.get_stages(path))
|
||||
outs.update(
|
||||
out.fspath
|
||||
for stage in stages
|
||||
@@ -482,49 +567,22 @@ class Repo(object):
|
||||
@contextmanager
|
||||
def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
|
||||
"""Opens a specified resource as a file descriptor"""
|
||||
cause = None
|
||||
|
||||
tree = RepoTree(self, stream=True)
|
||||
path = os.path.join(self.root_dir, path)
|
||||
try:
|
||||
out = self.find_out_by_relpath(path)
|
||||
except OutputNotFoundError as exc:
|
||||
out = None
|
||||
cause = exc
|
||||
|
||||
if out and out.use_cache:
|
||||
try:
|
||||
with self._open_cached(out, remote, mode, encoding) as fd:
|
||||
yield fd
|
||||
return
|
||||
except FileNotFoundError as exc:
|
||||
raise FileMissingError(path) from exc
|
||||
|
||||
abs_path = os.path.join(self.root_dir, path)
|
||||
if os.path.exists(abs_path):
|
||||
with open(abs_path, mode=mode, encoding=encoding) as fd:
|
||||
yield fd
|
||||
return
|
||||
|
||||
raise FileMissingError(path) from cause
|
||||
|
||||
def _open_cached(self, out, remote=None, mode="r", encoding=None):
|
||||
if out.isdir():
|
||||
raise IsADirectoryError("Can't open a dir")
|
||||
|
||||
cache_file = self.cache.local.checksum_to_path_info(out.checksum)
|
||||
cache_file = fspath_py35(cache_file)
|
||||
|
||||
if os.path.exists(cache_file):
|
||||
return open(cache_file, mode=mode, encoding=encoding)
|
||||
|
||||
try:
|
||||
remote_obj = self.cloud.get_remote(remote)
|
||||
remote_info = remote_obj.checksum_to_path_info(out.checksum)
|
||||
return remote_obj.open(remote_info, mode=mode, encoding=encoding)
|
||||
except RemoteActionNotImplemented:
|
||||
with self.state:
|
||||
cache_info = out.get_used_cache(remote=remote)
|
||||
self.cloud.pull(cache_info, remote=remote)
|
||||
|
||||
return open(cache_file, mode=mode, encoding=encoding)
|
||||
with tree.open(
|
||||
os.path.join(self.root_dir, path),
|
||||
mode=mode,
|
||||
encoding=encoding,
|
||||
remote=remote,
|
||||
) as fobj:
|
||||
yield fobj
|
||||
except FileNotFoundError as exc:
|
||||
raise FileMissingError(path) from exc
|
||||
except IsADirectoryError as exc:
|
||||
raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
|
||||
|
||||
def close(self):
|
||||
self.scm.close()
|
||||
|
||||
@@ -25,13 +25,16 @@ def brancher( # noqa: E302
|
||||
return
|
||||
|
||||
saved_tree = self.tree
|
||||
revs = revs or []
|
||||
revs = revs.copy() if revs else []
|
||||
|
||||
scm = self.scm
|
||||
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
yield "working tree"
|
||||
|
||||
if revs and "working tree" in revs:
|
||||
revs.remove("working tree")
|
||||
|
||||
if all_commits:
|
||||
revs = scm.list_all_commits()
|
||||
else:
|
||||
|
||||
+18
-10
@@ -1,8 +1,11 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.compat import fspath
|
||||
from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit
|
||||
from dvc.exceptions import (
|
||||
CheckoutError,
|
||||
CheckoutErrorSuggestGit,
|
||||
NoOutputOrStageError,
|
||||
)
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.utils import relpath
|
||||
|
||||
@@ -19,11 +22,11 @@ def _get_unused_links(repo):
|
||||
return repo.state.get_unused_links(used)
|
||||
|
||||
|
||||
def _fspath_dir(path, root):
|
||||
def _fspath_dir(path):
|
||||
if not os.path.exists(str(path)):
|
||||
return str(path)
|
||||
|
||||
path = relpath(fspath(path), root)
|
||||
path = relpath(path)
|
||||
return os.path.join(path, "") if os.path.isdir(path) else path
|
||||
|
||||
|
||||
@@ -57,9 +60,12 @@ def _checkout(
|
||||
targets = [None]
|
||||
unused = _get_unused_links(self)
|
||||
|
||||
stats["deleted"] = [_fspath_dir(u, self.root_dir) for u in unused]
|
||||
stats["deleted"] = [_fspath_dir(u) for u in unused]
|
||||
self.state.remove_links(unused)
|
||||
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
|
||||
pairs = set()
|
||||
for target in targets:
|
||||
try:
|
||||
@@ -68,7 +74,11 @@ def _checkout(
|
||||
target, with_deps=with_deps, recursive=recursive
|
||||
)
|
||||
)
|
||||
except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
|
||||
except (
|
||||
StageFileDoesNotExistError,
|
||||
StageFileBadNameError,
|
||||
NoOutputOrStageError,
|
||||
) as exc:
|
||||
if not target:
|
||||
raise
|
||||
raise CheckoutErrorSuggestGit(target) from exc
|
||||
@@ -84,10 +94,8 @@ def _checkout(
|
||||
relink=relink,
|
||||
filter_info=filter_info,
|
||||
)
|
||||
for data in ["failed", "added", "modified"]:
|
||||
stats[data].extend(
|
||||
_fspath_dir(path, self.root_dir) for path in result[data]
|
||||
)
|
||||
for key, items in result.items():
|
||||
stats[key].extend(_fspath_dir(path) for path in items)
|
||||
|
||||
if stats.get("failed"):
|
||||
raise CheckoutError(stats["failed"], stats)
|
||||
|
||||
+37
-1
@@ -1,13 +1,49 @@
|
||||
from dvc import prompt
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.stage.exceptions import StageCommitError
|
||||
|
||||
from . import locked
|
||||
|
||||
|
||||
def _prepare_message(stage, changes):
|
||||
changed_deps, changed_outs, changed_stage = changes
|
||||
if changed_deps and changed_outs:
|
||||
msg = "dependencies {deps} and outputs {outs} of {stage} changed."
|
||||
elif changed_deps:
|
||||
msg = "dependencies {deps} of {stage} changed."
|
||||
elif changed_outs:
|
||||
msg = "outputs {outs} of {stage} changed."
|
||||
else:
|
||||
msg = "{stage_changed}"
|
||||
|
||||
msg += " Are you sure you want to commit it?"
|
||||
|
||||
kw = {
|
||||
"stage": stage,
|
||||
"deps": changed_deps,
|
||||
"outs": changed_outs,
|
||||
"stage_changed": changed_stage,
|
||||
}
|
||||
return msg.format_map(kw)
|
||||
|
||||
|
||||
def prompt_to_commit(stage, changes, force=False):
|
||||
if not (force or prompt.confirm(_prepare_message(stage, changes))):
|
||||
raise StageCommitError(
|
||||
"unable to commit changed {}. Use `-f|--force` to "
|
||||
"force.".format(stage)
|
||||
)
|
||||
|
||||
|
||||
@locked
|
||||
def commit(self, target, with_deps=False, recursive=False, force=False):
|
||||
stages = self.collect(target, with_deps=with_deps, recursive=recursive)
|
||||
for stage in stages:
|
||||
stage.check_can_commit(force=force)
|
||||
changes = stage.changed_entries()
|
||||
if any(changes):
|
||||
prompt_to_commit(stage, changes, force=force)
|
||||
stage.save()
|
||||
stage.commit()
|
||||
|
||||
Dvcfile(self, stage.path).dump(stage)
|
||||
return stages
|
||||
|
||||
@@ -52,6 +52,9 @@ def diff(self, a_rev="HEAD", b_rev=None):
|
||||
if _exists(output)
|
||||
}
|
||||
|
||||
if self.scm.no_commits:
|
||||
return {}
|
||||
|
||||
working_tree = self.tree
|
||||
a_tree = self.scm.get_tree(a_rev)
|
||||
b_tree = self.scm.get_tree(b_rev) if b_rev else working_tree
|
||||
|
||||
+14
-57
@@ -1,9 +1,7 @@
|
||||
import logging
|
||||
|
||||
from dvc.cache import NamedCache
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.exceptions import DownloadError, OutputNotFoundError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.exceptions import DownloadError
|
||||
from dvc.scm.base import CloneError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -34,6 +32,12 @@ def _fetch(
|
||||
config.NoRemoteError: thrown when downloading only local files and no
|
||||
remote is configured
|
||||
"""
|
||||
|
||||
used_run_cache = self.stage_cache.pull(remote) if run_cache else []
|
||||
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
|
||||
used = self.used_cache(
|
||||
targets,
|
||||
all_branches=all_branches,
|
||||
@@ -44,6 +48,7 @@ def _fetch(
|
||||
remote=remote,
|
||||
jobs=jobs,
|
||||
recursive=recursive,
|
||||
used_run_cache=used_run_cache,
|
||||
)
|
||||
|
||||
downloaded = 0
|
||||
@@ -51,11 +56,7 @@ def _fetch(
|
||||
|
||||
try:
|
||||
downloaded += self.cloud.pull(
|
||||
used,
|
||||
jobs,
|
||||
remote=remote,
|
||||
show_checksums=show_checksums,
|
||||
run_cache=run_cache,
|
||||
used, jobs, remote=remote, show_checksums=show_checksums,
|
||||
)
|
||||
except NoRemoteError:
|
||||
if not used.external and used["local"]:
|
||||
@@ -75,35 +76,15 @@ def _fetch(
|
||||
|
||||
|
||||
def _fetch_external(self, repo_url, repo_rev, files, jobs):
|
||||
from dvc.external_repo import external_repo, ExternalRepo
|
||||
from dvc.external_repo import external_repo
|
||||
|
||||
failed, downloaded = 0, 0
|
||||
try:
|
||||
with external_repo(repo_url, repo_rev) as repo:
|
||||
is_dvc_repo = isinstance(repo, ExternalRepo)
|
||||
# gather git-only tracked files if dvc repo
|
||||
git_files = [] if is_dvc_repo else files
|
||||
if is_dvc_repo:
|
||||
repo.cache.local.cache_dir = self.cache.local.cache_dir
|
||||
with repo.state:
|
||||
cache = NamedCache()
|
||||
for name in files:
|
||||
try:
|
||||
out = repo.find_out_by_relpath(name)
|
||||
except OutputNotFoundError:
|
||||
# try to add to cache if they are git-tracked files
|
||||
git_files.append(name)
|
||||
else:
|
||||
cache.update(out.get_used_cache())
|
||||
|
||||
try:
|
||||
downloaded += repo.cloud.pull(cache, jobs=jobs)
|
||||
except DownloadError as exc:
|
||||
failed += exc.amount
|
||||
|
||||
d, f = _git_to_cache(self.cache.local, repo.root_dir, git_files)
|
||||
downloaded += d
|
||||
failed += f
|
||||
with repo.use_cache(self.cache.local):
|
||||
d, f, _ = repo.fetch_external(files, jobs=jobs)
|
||||
downloaded += d
|
||||
failed += f
|
||||
except CloneError:
|
||||
failed += 1
|
||||
logger.exception(
|
||||
@@ -111,27 +92,3 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs):
|
||||
)
|
||||
|
||||
return downloaded, failed
|
||||
|
||||
|
||||
def _git_to_cache(cache, repo_root, files):
|
||||
"""Save files from a git repo directly to the cache."""
|
||||
failed = set()
|
||||
num_downloads = 0
|
||||
repo_root = PathInfo(repo_root)
|
||||
for file in files:
|
||||
info = cache.save_info(repo_root / file)
|
||||
if info.get(cache.PARAM_CHECKSUM) is None:
|
||||
failed.add(file)
|
||||
continue
|
||||
|
||||
if cache.changed_cache(info[cache.PARAM_CHECKSUM]):
|
||||
logger.debug("fetched '%s' from '%s' repo", file, repo_root)
|
||||
num_downloads += 1
|
||||
cache.save(repo_root / file, info, save_link=False)
|
||||
|
||||
if failed:
|
||||
logger.exception(
|
||||
"failed to fetch data for {}".format(", ".join(failed))
|
||||
)
|
||||
|
||||
return num_downloads, len(failed)
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
|
||||
def _do_gc(typ, func, clist, jobs=None):
|
||||
removed = func(clist, jobs=jobs)
|
||||
if not removed:
|
||||
logger.info("No unused '{}' cache to remove.".format(typ))
|
||||
logger.info(f"No unused '{typ}' cache to remove.")
|
||||
|
||||
|
||||
def _raise_error_if_all_disabled(**kwargs):
|
||||
|
||||
+1
-3
@@ -4,7 +4,6 @@ import os
|
||||
import shortuuid
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.utils import resolve_output
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
@@ -51,7 +50,6 @@ def get(url, path, out=None, rev=None):
|
||||
# Also, we can't use theoretical "move" link type here, because
|
||||
# the same cache file might be used a few times in a directory.
|
||||
repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]
|
||||
|
||||
repo.pull_to(path, PathInfo(out))
|
||||
repo.get_external(path, out)
|
||||
finally:
|
||||
remove(tmp_dir)
|
||||
|
||||
+6
-2
@@ -17,8 +17,12 @@ def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
|
||||
out = resolve_output(url, out)
|
||||
path, wdir, out = resolve_paths(self, out)
|
||||
|
||||
# NOTE: when user is importing something from within his own repository
|
||||
if os.path.exists(url) and path_isin(os.path.abspath(url), self.root_dir):
|
||||
# NOTE: when user is importing something from within their own repository
|
||||
if (
|
||||
erepo is None
|
||||
and os.path.exists(url)
|
||||
and path_isin(os.path.abspath(url), self.root_dir)
|
||||
):
|
||||
url = relpath(url, wdir)
|
||||
|
||||
stage = create_stage(
|
||||
|
||||
+2
-4
@@ -3,13 +3,11 @@ from . import locked
|
||||
|
||||
@locked
|
||||
def lock(self, target, unlock=False):
|
||||
from .. import dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
dvcfile = dvcfile.Dvcfile(self, path)
|
||||
stage = dvcfile.stages[name]
|
||||
stage = self.get_stage(path, name)
|
||||
stage.locked = False if unlock else True
|
||||
dvcfile.dump(stage, update_pipeline=True)
|
||||
stage.dvcfile.dump(stage, update_pipeline=True)
|
||||
|
||||
return stage
|
||||
|
||||
+37
-40
@@ -4,7 +4,7 @@ from dvc.path_info import PathInfo
|
||||
|
||||
@staticmethod
|
||||
def ls(
|
||||
url, path=None, rev=None, recursive=None, outs_only=False,
|
||||
url, path=None, rev=None, recursive=None, dvc_only=False,
|
||||
):
|
||||
"""Methods for getting files and outputs for the repo.
|
||||
|
||||
@@ -13,7 +13,7 @@ def ls(
|
||||
path (str, optional): relative path into the repo
|
||||
rev (str, optional): SHA commit, branch or tag name
|
||||
recursive (bool, optional): recursively walk the repo
|
||||
outs_only (bool, optional): show only DVC-artifacts
|
||||
dvc_only (bool, optional): show only DVC-artifacts
|
||||
|
||||
Returns:
|
||||
list of `entry`
|
||||
@@ -28,25 +28,16 @@ def ls(
|
||||
}
|
||||
"""
|
||||
from dvc.external_repo import external_repo
|
||||
from dvc.repo import Repo
|
||||
|
||||
with external_repo(url, rev) as repo:
|
||||
path_info = PathInfo(repo.root_dir)
|
||||
if path:
|
||||
path_info /= path
|
||||
|
||||
ret = {}
|
||||
if isinstance(repo, Repo):
|
||||
ret = _ls(repo, path_info, recursive, True)
|
||||
|
||||
nondvc = {}
|
||||
if not outs_only:
|
||||
nondvc = _ls(repo, path_info, recursive, False)
|
||||
|
||||
ret.update(nondvc)
|
||||
ret = _ls(repo, path_info, recursive, dvc_only)
|
||||
|
||||
if path and not ret:
|
||||
raise PathMissingError(path, repo, output_only=outs_only)
|
||||
raise PathMissingError(path, repo, dvc_only=dvc_only)
|
||||
|
||||
ret_list = []
|
||||
for path, info in ret.items():
|
||||
@@ -56,46 +47,52 @@ def ls(
|
||||
return ret_list
|
||||
|
||||
|
||||
def _ls(repo, path_info, recursive=None, dvc=False):
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.repo.tree import DvcTree
|
||||
from dvc.scm.tree import WorkingTree
|
||||
def _ls(repo, path_info, recursive=None, dvc_only=False):
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
if dvc:
|
||||
tree = DvcTree(repo)
|
||||
else:
|
||||
tree = CleanTree(WorkingTree(repo.root_dir))
|
||||
# use our own RepoTree instance instead of repo.repo_tree since we do not
|
||||
# want fetch/stream enabled for ls
|
||||
tree = RepoTree(repo)
|
||||
|
||||
ret = {}
|
||||
try:
|
||||
for root, dirs, files in tree.walk(path_info.fspath):
|
||||
for root, dirs, files in tree.walk(path_info.fspath, dvcfiles=True):
|
||||
for fname in files:
|
||||
info = PathInfo(root) / fname
|
||||
path = str(info.relative_to(path_info))
|
||||
ret[path] = {
|
||||
"isout": dvc,
|
||||
"isdir": False,
|
||||
"isexec": False if dvc else tree.isexec(info.fspath),
|
||||
}
|
||||
dvc = tree.isdvc(info)
|
||||
if dvc or not dvc_only:
|
||||
path = str(info.relative_to(path_info))
|
||||
ret[path] = {
|
||||
"isout": dvc,
|
||||
"isdir": False,
|
||||
"isexec": False if dvc else tree.isexec(info),
|
||||
}
|
||||
|
||||
if not recursive:
|
||||
for dname in dirs:
|
||||
info = PathInfo(root) / dname
|
||||
path = str(info.relative_to(path_info))
|
||||
ret[path] = {
|
||||
"isout": tree.isdvc(info.fspath) if dvc else False,
|
||||
"isdir": True,
|
||||
"isexec": False if dvc else tree.isexec(info.fspath),
|
||||
}
|
||||
if not dvc_only or (
|
||||
tree.dvctree and tree.dvctree.exists(info)
|
||||
):
|
||||
dvc = tree.isdvc(info)
|
||||
path = str(info.relative_to(path_info))
|
||||
ret[path] = {
|
||||
"isout": dvc,
|
||||
"isdir": True,
|
||||
"isexec": False if dvc else tree.isexec(info),
|
||||
}
|
||||
break
|
||||
except NotADirectoryError:
|
||||
return {
|
||||
path_info.name: {
|
||||
"isout": dvc,
|
||||
"isdir": False,
|
||||
"isexec": False if dvc else tree.isexec(path_info.fspath),
|
||||
dvc = tree.isdvc(path_info)
|
||||
if dvc or not dvc_only:
|
||||
return {
|
||||
path_info.name: {
|
||||
"isout": dvc,
|
||||
"isdir": False,
|
||||
"isexec": False if dvc else tree.isexec(path_info),
|
||||
}
|
||||
}
|
||||
}
|
||||
return {}
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
class Metrics(object):
|
||||
class Metrics:
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
|
||||
|
||||
@@ -14,6 +14,9 @@ def _get_metrics(repo, *args, rev=None, **kwargs):
|
||||
|
||||
|
||||
def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
|
||||
if repo.scm.no_commits:
|
||||
return {}
|
||||
|
||||
with_unchanged = kwargs.pop("all", False)
|
||||
|
||||
old = _get_metrics(repo, *args, **kwargs, rev=(a_rev or "HEAD"))
|
||||
|
||||
@@ -3,7 +3,6 @@ import os
|
||||
|
||||
import yaml
|
||||
|
||||
from dvc.compat import fspath_py35
|
||||
from dvc.exceptions import NoMetricsError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import locked
|
||||
@@ -58,10 +57,10 @@ def _read_metrics(repo, metrics, rev):
|
||||
|
||||
res = {}
|
||||
for metric in metrics:
|
||||
if not tree.exists(fspath_py35(metric)):
|
||||
if not tree.exists(metric):
|
||||
continue
|
||||
|
||||
with tree.open(fspath_py35(metric), "r") as fobj:
|
||||
with tree.open(metric, "r") as fobj:
|
||||
try:
|
||||
# NOTE this also supports JSON
|
||||
val = yaml.safe_load(fobj)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
class Params(object):
|
||||
class Params:
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
|
||||
|
||||
@@ -13,6 +13,9 @@ def _get_params(repo, *args, rev=None, **kwargs):
|
||||
|
||||
|
||||
def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
|
||||
if repo.scm.no_commits:
|
||||
return {}
|
||||
|
||||
with_unchanged = kwargs.pop("all", False)
|
||||
|
||||
old = _get_params(repo, *args, **kwargs, rev=(a_rev or "HEAD"))
|
||||
|
||||
@@ -2,7 +2,6 @@ import logging
|
||||
|
||||
import yaml
|
||||
|
||||
from dvc.compat import fspath_py35
|
||||
from dvc.dependency.param import ParamsDependency
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import PathInfo
|
||||
@@ -30,10 +29,10 @@ def _collect_configs(repo):
|
||||
def _read_params(repo, configs, rev):
|
||||
res = {}
|
||||
for config in configs:
|
||||
if not repo.tree.exists(fspath_py35(config)):
|
||||
if not repo.tree.exists(config):
|
||||
continue
|
||||
|
||||
with repo.tree.open(fspath_py35(config), "r") as fobj:
|
||||
with repo.tree.open(config, "r") as fobj:
|
||||
try:
|
||||
res[str(config)] = yaml.safe_load(fobj)
|
||||
except yaml.YAMLError:
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
class Plots:
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
|
||||
def show(self, *args, **kwargs):
|
||||
from .show import show
|
||||
|
||||
return show(self.repo, *args, **kwargs)
|
||||
|
||||
def diff(self, *args, **kwargs):
|
||||
from .diff import diff
|
||||
|
||||
return diff(self.repo, *args, **kwargs)
|
||||
@@ -14,8 +14,6 @@ from dvc.exceptions import DvcException, PathMissingError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
WORKSPACE_REVISION_NAME = "workspace"
|
||||
|
||||
|
||||
class PlotMetricTypeError(DvcException):
|
||||
def __init__(self, file):
|
||||
@@ -29,7 +27,7 @@ class PlotDataStructureError(DvcException):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"Plot data extraction failed. Please see "
|
||||
"documentation for supported data formats."
|
||||
"https://man.dvc.org/plot for supported data formats."
|
||||
)
|
||||
|
||||
|
||||
@@ -45,14 +43,12 @@ class NoMetricOnRevisionError(DvcException):
|
||||
def __init__(self, path, revision):
|
||||
self.path = path
|
||||
self.revision = revision
|
||||
super().__init__(
|
||||
"Could not find '{}' on revision '{}'".format(path, revision)
|
||||
)
|
||||
super().__init__(f"Could not find '{path}' on revision '{revision}'")
|
||||
|
||||
|
||||
class NoMetricInHistoryError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__("Could not find '{}'.".format(path))
|
||||
super().__init__(f"Could not find '{path}'.")
|
||||
|
||||
|
||||
def plot_data(filename, revision, content):
|
||||
@@ -194,13 +190,13 @@ class JSONPlotData(PlotData):
|
||||
return json.loads(self.content, object_pairs_hook=OrderedDict)
|
||||
|
||||
def _processors(self):
|
||||
parent_processors = super(JSONPlotData, self)._processors()
|
||||
parent_processors = super()._processors()
|
||||
return [_apply_path, _find_data] + parent_processors
|
||||
|
||||
|
||||
class CSVPlotData(PlotData):
|
||||
def __init__(self, filename, revision, content, delimiter=","):
|
||||
super(CSVPlotData, self).__init__(filename, revision, content)
|
||||
super().__init__(filename, revision, content)
|
||||
self.delimiter = delimiter
|
||||
|
||||
def raw(self, csv_header=True, **kwargs):
|
||||
@@ -243,20 +239,12 @@ class YAMLPlotData(PlotData):
|
||||
|
||||
|
||||
def _load_from_revision(repo, datafile, revision):
|
||||
if revision is WORKSPACE_REVISION_NAME:
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
def open_datafile():
|
||||
return repo.tree.open(datafile, "r")
|
||||
|
||||
else:
|
||||
|
||||
def open_datafile():
|
||||
from dvc import api
|
||||
|
||||
return api.open(datafile, repo.root_dir, revision)
|
||||
tree = RepoTree(repo)
|
||||
|
||||
try:
|
||||
with open_datafile() as fobj:
|
||||
with tree.open(datafile) as fobj:
|
||||
datafile_content = fobj.read()
|
||||
|
||||
except (FileNotFoundError, PathMissingError):
|
||||
@@ -269,7 +257,10 @@ def _load_from_revisions(repo, datafile, revisions):
|
||||
data = []
|
||||
exceptions = []
|
||||
|
||||
for rev in revisions:
|
||||
for rev in repo.brancher(revs=revisions):
|
||||
if rev == "working tree" and rev not in revisions:
|
||||
continue
|
||||
|
||||
try:
|
||||
data.append(_load_from_revision(repo, datafile, rev))
|
||||
except NoMetricOnRevisionError as e:
|
||||
@@ -277,7 +268,7 @@ def _load_from_revisions(repo, datafile, revisions):
|
||||
except PlotMetricTypeError:
|
||||
raise
|
||||
except (yaml.error.YAMLError, json.decoder.JSONDecodeError, csv.Error):
|
||||
logger.error("Failed to parse '{}' at '{}'.".format(datafile, rev))
|
||||
logger.error(f"Failed to parse '{datafile}' at '{rev}'.")
|
||||
raise
|
||||
|
||||
if not data and exceptions:
|
||||
@@ -0,0 +1,13 @@
|
||||
def _revisions(revs, is_dirty):
|
||||
revisions = revs or []
|
||||
if len(revisions) <= 1:
|
||||
if len(revisions) == 0 and is_dirty:
|
||||
revisions.append("HEAD")
|
||||
revisions.append("working tree")
|
||||
return revisions
|
||||
|
||||
|
||||
def diff(repo, *args, revs=None, **kwargs):
|
||||
return repo.plots.show(
|
||||
*args, revs=_revisions(revs, repo.scm.is_dirty()), **kwargs
|
||||
)
|
||||
@@ -1,33 +1,17 @@
|
||||
import logging
|
||||
import os
|
||||
from collections import defaultdict
|
||||
|
||||
from funcy import first, last
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.repo import locked
|
||||
from dvc.repo.plot.data import PlotData
|
||||
from dvc.repo.plot.template import NoDataForTemplateError, Template
|
||||
|
||||
from .data import PlotData
|
||||
from .template import NoDataForTemplateError, Template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PAGE_HTML = """<html>
|
||||
<head>
|
||||
<title>DVC Plot</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
|
||||
</head>
|
||||
<body>
|
||||
{divs}
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
DIV_HTML = """<div id = "{id}"></div>
|
||||
<script type = "text/javascript">
|
||||
var spec = {vega_json};
|
||||
vegaEmbed('#{id}', spec);
|
||||
</script>"""
|
||||
|
||||
|
||||
class TooManyDataSourcesError(DvcException):
|
||||
def __init__(self, datafile, template_datafiles):
|
||||
@@ -44,6 +28,10 @@ class NoDataOrTemplateProvided(DvcException):
|
||||
super().__init__("Datafile or template is not specified.")
|
||||
|
||||
|
||||
class TooManyTemplatesError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
def _evaluate_templatepath(repo, template=None):
|
||||
if not template:
|
||||
return repo.plot_templates.default_template
|
||||
@@ -64,7 +52,7 @@ def fill_template(
|
||||
csv_header=True,
|
||||
x_field=None,
|
||||
y_field=None,
|
||||
**kwargs
|
||||
**kwargs,
|
||||
):
|
||||
if x_field and fields:
|
||||
fields.add(x_field)
|
||||
@@ -81,7 +69,7 @@ def fill_template(
|
||||
|
||||
template_data = {}
|
||||
for template_datafile in template_datafiles:
|
||||
from dvc.repo.plot.data import _load_from_revisions
|
||||
from .data import _load_from_revisions
|
||||
|
||||
plot_datas = _load_from_revisions(repo, template_datafile, revisions)
|
||||
tmp_data = []
|
||||
@@ -102,15 +90,19 @@ def fill_template(
|
||||
if len(template_data) == 0:
|
||||
raise NoDataForTemplateError(template_path)
|
||||
|
||||
return Template.fill(
|
||||
content = Template.fill(
|
||||
template_path,
|
||||
template_data,
|
||||
priority_datafile=datafile,
|
||||
x_field=x_field,
|
||||
y_field=y_field,
|
||||
**kwargs
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
path = datafile or ",".join(template_datafiles)
|
||||
|
||||
return path, content
|
||||
|
||||
|
||||
def _infer_y_field(rev_data_points, x_field):
|
||||
all_fields = list(first(rev_data_points).keys())
|
||||
@@ -121,32 +113,24 @@ def _infer_y_field(rev_data_points, x_field):
|
||||
return y_field
|
||||
|
||||
|
||||
def plot(
|
||||
repo, datafile=None, template=None, revisions=None, embed=False, **kwargs
|
||||
):
|
||||
if revisions is None:
|
||||
from dvc.repo.plot.data import WORKSPACE_REVISION_NAME
|
||||
|
||||
revisions = [WORKSPACE_REVISION_NAME]
|
||||
def _show(repo, datafile=None, template=None, revs=None, **kwargs):
|
||||
if revs is None:
|
||||
revs = ["working tree"]
|
||||
|
||||
if not datafile and not template:
|
||||
raise NoDataOrTemplateProvided()
|
||||
|
||||
template_path = _evaluate_templatepath(repo, template)
|
||||
|
||||
plot_content = fill_template(
|
||||
repo, datafile, template_path, revisions, **kwargs
|
||||
plot_datafile, plot_content = fill_template(
|
||||
repo, datafile, template_path, revs, **kwargs
|
||||
)
|
||||
|
||||
if embed:
|
||||
div = DIV_HTML.format(id="plot", vega_json=plot_content)
|
||||
plot_content = PAGE_HTML.format(divs=div)
|
||||
|
||||
return plot_content
|
||||
return plot_datafile, plot_content
|
||||
|
||||
|
||||
def _parse_template(template_path, priority_datafile):
|
||||
with open(template_path, "r") as fobj:
|
||||
with open(template_path) as fobj:
|
||||
tempalte_content = fobj.read()
|
||||
|
||||
template_datafiles = Template.parse_data_anchors(tempalte_content)
|
||||
@@ -162,3 +146,69 @@ def _parse_template(template_path, priority_datafile):
|
||||
Template.X_ANCHOR in tempalte_content,
|
||||
Template.Y_ANCHOR in tempalte_content,
|
||||
)
|
||||
|
||||
|
||||
def _collect_plots(repo):
|
||||
plots = defaultdict(set)
|
||||
|
||||
for stage in repo.stages:
|
||||
for out in stage.outs:
|
||||
if not out.plot:
|
||||
continue
|
||||
|
||||
if isinstance(out.plot, dict):
|
||||
template = out.plot[out.PARAM_PLOT_TEMPLATE]
|
||||
else:
|
||||
template = None
|
||||
|
||||
plots[str(out)].add(template)
|
||||
|
||||
return plots
|
||||
|
||||
|
||||
def show(repo, targets=None, template=None, revs=None, **kwargs) -> dict:
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
|
||||
if targets:
|
||||
for target in targets:
|
||||
return {
|
||||
target: _show(
|
||||
repo,
|
||||
datafile=target,
|
||||
template=template,
|
||||
revs=revs,
|
||||
**kwargs,
|
||||
)[1]
|
||||
}
|
||||
|
||||
if not revs:
|
||||
plots = _collect_plots(repo)
|
||||
else:
|
||||
plots = defaultdict(set)
|
||||
for rev in repo.brancher(revs=revs):
|
||||
for plot, templates in _collect_plots(repo).items():
|
||||
plots[plot].update(templates)
|
||||
|
||||
if not plots:
|
||||
datafile, plot = _show(
|
||||
repo, datafile=None, template=template, revs=revs, **kwargs
|
||||
)
|
||||
return {datafile: plot}
|
||||
|
||||
ret = {}
|
||||
for plot, templates in plots.items():
|
||||
tmplt = template
|
||||
if len(templates) == 1:
|
||||
tmplt = list(templates)[0]
|
||||
elif not template:
|
||||
raise TooManyTemplatesError(
|
||||
f"'{plot}' uses multiple templates '{templates}'. "
|
||||
"Use `-t|--template` to specify the template to use. "
|
||||
)
|
||||
|
||||
ret[plot] = _show(
|
||||
repo, datafile=plot, template=tmplt, revs=revs, **kwargs
|
||||
)[1]
|
||||
|
||||
return ret
|
||||
@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class TemplateNotFoundError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__("Template '{}' not found.".format(path))
|
||||
super().__init__(f"Template '{path}' not found.")
|
||||
|
||||
|
||||
class NoDataForTemplateError(DvcException):
|
||||
@@ -26,7 +26,7 @@ class NoDataForTemplateError(DvcException):
|
||||
class NoFieldInDataError(DvcException):
|
||||
def __init__(self, field_name):
|
||||
super().__init__(
|
||||
"Field '{}' does not exist in provided data.".format(field_name)
|
||||
f"Field '{field_name}' does not exist in provided data."
|
||||
)
|
||||
|
||||
|
||||
@@ -59,6 +59,7 @@ class Template:
|
||||
indent=self.INDENT,
|
||||
separators=self.SEPARATORS,
|
||||
)
|
||||
fobj.write("\n")
|
||||
|
||||
@staticmethod
|
||||
def get_data_anchor(template_content):
|
||||
@@ -94,7 +95,7 @@ class Template:
|
||||
x_title=None,
|
||||
y_title=None,
|
||||
):
|
||||
with open(template_path, "r") as fobj:
|
||||
with open(template_path) as fobj:
|
||||
result_content = fobj.read()
|
||||
|
||||
if x_field:
|
||||
@@ -191,6 +192,7 @@ class DefaultLinearTemplate(Template):
|
||||
"field": Template.Y_ANCHOR,
|
||||
"type": "quantitative",
|
||||
"title": Template.Y_TITLE_ANCHOR,
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
},
|
||||
@@ -240,6 +242,7 @@ class DefaultScatterTemplate(Template):
|
||||
"field": Template.Y_ANCHOR,
|
||||
"type": "quantitative",
|
||||
"title": Template.Y_TITLE_ANCHOR,
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
},
|
||||
@@ -247,7 +250,7 @@ class DefaultScatterTemplate(Template):
|
||||
|
||||
|
||||
class PlotTemplates:
|
||||
TEMPLATES_DIR = "plot"
|
||||
TEMPLATES_DIR = "plots"
|
||||
TEMPLATES = [
|
||||
DefaultLinearTemplate,
|
||||
DefaultConfusionTemplate,
|
||||
@@ -19,6 +19,9 @@ def pull(
|
||||
all_commits=False,
|
||||
run_cache=False,
|
||||
):
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
|
||||
processed_files_count = self._fetch(
|
||||
targets,
|
||||
jobs,
|
||||
|
||||
+7
-1
@@ -14,6 +14,11 @@ def push(
|
||||
all_commits=False,
|
||||
run_cache=False,
|
||||
):
|
||||
used_run_cache = self.stage_cache.push(remote) if run_cache else []
|
||||
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
|
||||
used = self.used_cache(
|
||||
targets,
|
||||
all_branches=all_branches,
|
||||
@@ -24,6 +29,7 @@ def push(
|
||||
remote=remote,
|
||||
jobs=jobs,
|
||||
recursive=recursive,
|
||||
used_run_cache=used_run_cache,
|
||||
)
|
||||
|
||||
return self.cloud.push(used, jobs, remote=remote, run_cache=run_cache)
|
||||
return self.cloud.push(used, jobs, remote=remote)
|
||||
|
||||
+5
-6
@@ -7,16 +7,15 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@locked
|
||||
def remove(self, target, outs_only=False):
|
||||
from ..dvcfile import Dvcfile
|
||||
def remove(self, target, dvc_only=False):
|
||||
from ..dvcfile import Dvcfile, is_valid_filename
|
||||
|
||||
path, name = parse_target(target)
|
||||
dvcfile = Dvcfile(self, path)
|
||||
stages = list(dvcfile.stages.filter(name).values())
|
||||
stages = self.get_stages(path, name)
|
||||
for stage in stages:
|
||||
stage.remove_outs(force=True)
|
||||
|
||||
if not outs_only:
|
||||
dvcfile.remove()
|
||||
if path and is_valid_filename(path) and not dvc_only:
|
||||
Dvcfile(self, path).remove()
|
||||
|
||||
return stages
|
||||
|
||||
@@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _reproduce_stage(stage, **kwargs):
|
||||
if stage.locked:
|
||||
if stage.locked and not stage.is_import:
|
||||
logger.warning(
|
||||
"{} is locked. Its dependencies are"
|
||||
" not going to be reproduced.".format(stage)
|
||||
@@ -61,7 +61,6 @@ def reproduce(
|
||||
all_pipelines=False,
|
||||
**kwargs
|
||||
):
|
||||
from ..dvcfile import Dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
if not target and not all_pipelines:
|
||||
@@ -81,8 +80,7 @@ def reproduce(
|
||||
if all_pipelines:
|
||||
pipelines = active_pipelines
|
||||
else:
|
||||
dvcfile = Dvcfile(self, path)
|
||||
stage = dvcfile.stages[name]
|
||||
stage = self.get_stage(path, name)
|
||||
pipelines = [get_pipeline(active_pipelines, stage)]
|
||||
|
||||
targets = []
|
||||
@@ -162,12 +160,18 @@ def _reproduce_stages(
|
||||
if stage not in pipeline:
|
||||
pipeline.append(stage)
|
||||
|
||||
force_downstream = kwargs.pop("force_downstream", False)
|
||||
result = []
|
||||
# `ret` is used to add a cosmetic newline.
|
||||
ret = []
|
||||
for stage in pipeline:
|
||||
if ret:
|
||||
logger.info("")
|
||||
|
||||
try:
|
||||
ret = _reproduce_stage(stage, **kwargs)
|
||||
|
||||
if len(ret) != 0 and kwargs.get("ignore_build_cache", False):
|
||||
if len(ret) != 0 and force_downstream:
|
||||
# NOTE: we are walking our pipeline from the top to the
|
||||
# bottom. If one stage is changed, it will be reproduced,
|
||||
# which tells us that we should force reproducing all of
|
||||
|
||||
+11
-5
@@ -11,7 +11,9 @@ from .scm_context import scm_context
|
||||
|
||||
|
||||
def is_valid_name(name: str):
|
||||
return not {"\\", "/", "@", ":"} & set(name)
|
||||
from ..stage import INVALID_STAGENAME_CHARS
|
||||
|
||||
return not INVALID_STAGENAME_CHARS & set(name)
|
||||
|
||||
|
||||
def _get_file_path(kwargs):
|
||||
@@ -23,6 +25,8 @@ def _get_file_path(kwargs):
|
||||
kwargs.get("outs_no_cache", []),
|
||||
kwargs.get("metrics", []),
|
||||
kwargs.get("metrics_no_cache", []),
|
||||
kwargs.get("plots", []),
|
||||
kwargs.get("plots_no_cache", []),
|
||||
kwargs.get("outs_persist", []),
|
||||
kwargs.get("outs_persist_no_cache", []),
|
||||
)
|
||||
@@ -41,6 +45,9 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
|
||||
from dvc.stage import PipelineStage, Stage, create_stage
|
||||
from dvc.dvcfile import Dvcfile, PIPELINE_FILE
|
||||
|
||||
if not kwargs.get("cmd"):
|
||||
raise InvalidArgumentError("command is not specified")
|
||||
|
||||
stage_cls = PipelineStage
|
||||
path = PIPELINE_FILE
|
||||
stage_name = kwargs.get("name")
|
||||
@@ -78,13 +85,12 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
|
||||
raise OutputDuplicationError(exc.output, set(exc.stages) - {stage})
|
||||
|
||||
if no_exec:
|
||||
for out in stage.outs:
|
||||
out.ignore()
|
||||
stage.ignore_outs()
|
||||
else:
|
||||
stage.run(
|
||||
no_commit=kwargs.get("no_commit", False),
|
||||
ignore_build_cache=kwargs.get("ignore_build_cache", False),
|
||||
run_cache=kwargs.get("run_cache", True),
|
||||
)
|
||||
|
||||
dvcfile.dump(stage, update_pipeline=True)
|
||||
dvcfile.dump(stage, update_pipeline=True, no_lock=no_exec)
|
||||
return stage
|
||||
|
||||
@@ -116,6 +116,9 @@ def status(
|
||||
all_commits=False,
|
||||
recursive=False,
|
||||
):
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
|
||||
if cloud or remote:
|
||||
return _cloud_status(
|
||||
self,
|
||||
|
||||
+303
-31
@@ -1,14 +1,35 @@
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.dvcfile import is_valid_filename
|
||||
from dvc.exceptions import OutputNotFoundError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.scm.tree import BaseTree, WorkingTree
|
||||
from dvc.remote.base import RemoteActionNotImplemented
|
||||
from dvc.scm.tree import BaseTree
|
||||
from dvc.utils import file_md5
|
||||
from dvc.utils.fs import copy_fobj_to_file, makedirs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DvcTree(BaseTree):
|
||||
def __init__(self, repo):
|
||||
"""DVC repo tree.
|
||||
|
||||
Args:
|
||||
repo: DVC repo.
|
||||
fetch: if True, uncached DVC outs will be fetched on `open()`.
|
||||
stream: if True, uncached DVC outs will be streamed directly from
|
||||
remote on `open()`.
|
||||
|
||||
`stream` takes precedence over `fetch`. If `stream` is enabled and
|
||||
a remote does not support streaming, uncached DVC outs will be fetched
|
||||
as a fallback.
|
||||
"""
|
||||
|
||||
def __init__(self, repo, fetch=False, stream=False):
|
||||
self.repo = repo
|
||||
self.fetch = fetch
|
||||
self.stream = stream
|
||||
|
||||
def _find_outs(self, path, *args, **kwargs):
|
||||
outs = self.repo.find_outs_by_path(path, *args, **kwargs)
|
||||
@@ -22,27 +43,56 @@ class DvcTree(BaseTree):
|
||||
|
||||
return outs
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8"):
|
||||
def _get_granular_checksum(self, path, out, remote=None):
|
||||
if not self.fetch and not self.stream:
|
||||
raise FileNotFoundError
|
||||
dir_cache = out.get_dir_cache(remote=remote)
|
||||
for entry in dir_cache:
|
||||
entry_relpath = entry[out.remote.PARAM_RELPATH]
|
||||
if os.name == "nt":
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
if path == out.path_info / entry_relpath:
|
||||
return entry[out.remote.PARAM_CHECKSUM]
|
||||
raise FileNotFoundError
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8", remote=None):
|
||||
try:
|
||||
outs = self._find_outs(path, strict=False)
|
||||
except OutputNotFoundError as exc:
|
||||
raise FileNotFoundError from exc
|
||||
|
||||
if len(outs) != 1 or outs[0].is_dir_checksum:
|
||||
raise IOError(errno.EISDIR)
|
||||
if len(outs) != 1 or (
|
||||
outs[0].is_dir_checksum and path == outs[0].path_info
|
||||
):
|
||||
raise IsADirectoryError
|
||||
|
||||
out = outs[0]
|
||||
# temporary hack to make cache use WorkingTree and not GitTree, because
|
||||
# cache dir doesn't exist in the latter.
|
||||
saved_tree = self.repo.tree
|
||||
self.repo.tree = WorkingTree(self.repo.root_dir)
|
||||
try:
|
||||
if out.changed_cache():
|
||||
if out.changed_cache(filter_info=path):
|
||||
if not self.fetch and not self.stream:
|
||||
raise FileNotFoundError
|
||||
finally:
|
||||
self.repo.tree = saved_tree
|
||||
|
||||
return open(out.cache_path, mode=mode, encoding=encoding)
|
||||
remote_obj = self.repo.cloud.get_remote(remote)
|
||||
if self.stream:
|
||||
if out.is_dir_checksum:
|
||||
checksum = self._get_granular_checksum(path, out)
|
||||
else:
|
||||
checksum = out.checksum
|
||||
try:
|
||||
remote_info = remote_obj.checksum_to_path_info(checksum)
|
||||
return remote_obj.open(
|
||||
remote_info, mode=mode, encoding=encoding
|
||||
)
|
||||
except RemoteActionNotImplemented:
|
||||
pass
|
||||
cache_info = out.get_used_cache(filter_info=path, remote=remote)
|
||||
self.repo.cloud.pull(cache_info, remote=remote)
|
||||
|
||||
if out.is_dir_checksum:
|
||||
checksum = self._get_granular_checksum(path, out)
|
||||
cache_path = out.cache.checksum_to_path_info(checksum).url
|
||||
else:
|
||||
cache_path = out.cache_path
|
||||
return open(cache_path, mode=mode, encoding=encoding)
|
||||
|
||||
def exists(self, path):
|
||||
try:
|
||||
@@ -57,10 +107,25 @@ class DvcTree(BaseTree):
|
||||
|
||||
path_info = PathInfo(os.path.abspath(path))
|
||||
outs = self._find_outs(path, strict=False, recursive=True)
|
||||
if len(outs) != 1 or outs[0].path_info != path_info:
|
||||
if len(outs) != 1:
|
||||
return True
|
||||
|
||||
return outs[0].is_dir_checksum
|
||||
out = outs[0]
|
||||
if not out.is_dir_checksum:
|
||||
if out.path_info != path_info:
|
||||
return True
|
||||
return False
|
||||
|
||||
if out.path_info == path_info:
|
||||
return True
|
||||
|
||||
# for dir checksum, we need to check if this is a file inside the
|
||||
# directory
|
||||
try:
|
||||
self._get_granular_checksum(path, out)
|
||||
return False
|
||||
except FileNotFoundError:
|
||||
return True
|
||||
|
||||
def isfile(self, path):
|
||||
if not self.exists(path):
|
||||
@@ -78,21 +143,22 @@ class DvcTree(BaseTree):
|
||||
continue
|
||||
|
||||
name = key[root_len]
|
||||
if len(key) > root_len + 1 or out.is_dir_checksum:
|
||||
if len(key) > root_len + 1 or (out and out.is_dir_checksum):
|
||||
dirs.add(name)
|
||||
continue
|
||||
|
||||
files.append(name)
|
||||
|
||||
if topdown:
|
||||
yield root.fspath, list(dirs), files
|
||||
dirs = list(dirs)
|
||||
yield root.fspath, dirs, files
|
||||
|
||||
for dname in dirs:
|
||||
yield from self._walk(root / dname, trie)
|
||||
else:
|
||||
assert False
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
def walk(self, top, topdown=True, download_callback=None, **kwargs):
|
||||
from pygtrie import Trie
|
||||
|
||||
assert topdown
|
||||
@@ -111,11 +177,30 @@ class DvcTree(BaseTree):
|
||||
for out in outs:
|
||||
trie[out.path_info.parts] = out
|
||||
|
||||
if out.is_dir_checksum and (self.fetch or self.stream):
|
||||
# pull dir cache if needed
|
||||
dir_cache = out.get_dir_cache(**kwargs)
|
||||
|
||||
# pull dir contents if needed
|
||||
if self.fetch:
|
||||
if out.changed_cache(filter_info=top):
|
||||
used_cache = out.get_used_cache(filter_info=top)
|
||||
downloaded = self.repo.cloud.pull(used_cache, **kwargs)
|
||||
if download_callback:
|
||||
download_callback(downloaded)
|
||||
|
||||
for entry in dir_cache:
|
||||
entry_relpath = entry[out.remote.PARAM_RELPATH]
|
||||
if os.name == "nt":
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
path_info = out.path_info / entry_relpath
|
||||
trie[path_info.parts] = None
|
||||
|
||||
yield from self._walk(root, trie, topdown=topdown)
|
||||
|
||||
def isdvc(self, path):
|
||||
def isdvc(self, path, **kwargs):
|
||||
try:
|
||||
return len(self._find_outs(path)) == 1
|
||||
return len(self._find_outs(path, **kwargs)) == 1
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
return False
|
||||
@@ -123,19 +208,206 @@ class DvcTree(BaseTree):
|
||||
def isexec(self, path):
|
||||
return False
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
outs = self._find_outs(path_info, strict=False)
|
||||
if len(outs) != 1:
|
||||
raise OutputNotFoundError
|
||||
out = outs[0]
|
||||
if out.is_dir_checksum:
|
||||
return self._get_granular_checksum(path_info, out)
|
||||
return out.checksum
|
||||
|
||||
|
||||
class RepoTree(BaseTree):
|
||||
def __init__(self, repo):
|
||||
"""DVC + git-tracked files tree.
|
||||
|
||||
Args:
|
||||
repo: DVC or git repo.
|
||||
|
||||
Any kwargs will be passed to `DvcTree()`.
|
||||
"""
|
||||
|
||||
def __init__(self, repo, **kwargs):
|
||||
self.repo = repo
|
||||
self.dvctree = DvcTree(repo)
|
||||
if hasattr(repo, "dvc_dir"):
|
||||
self.dvctree = DvcTree(repo, **kwargs)
|
||||
else:
|
||||
# git-only erepo's do not need dvctree
|
||||
self.dvctree = None
|
||||
|
||||
def open(self, *args, **kwargs):
|
||||
try:
|
||||
return self.dvctree.open(*args, **kwargs)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
@property
|
||||
def fetch(self):
|
||||
if self.dvctree:
|
||||
return self.dvctree.fetch
|
||||
return False
|
||||
|
||||
return self.repo.tree.open(*args, **kwargs)
|
||||
@property
|
||||
def stream(self):
|
||||
if self.dvctree:
|
||||
return self.dvctree.stream
|
||||
return False
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8", **kwargs):
|
||||
if "b" in mode:
|
||||
encoding = None
|
||||
|
||||
if self.dvctree and self.dvctree.exists(path):
|
||||
try:
|
||||
return self.dvctree.open(
|
||||
path, mode=mode, encoding=encoding, **kwargs
|
||||
)
|
||||
except FileNotFoundError:
|
||||
if self.isdvc(path):
|
||||
raise
|
||||
return self.repo.tree.open(path, mode=mode, encoding=encoding)
|
||||
|
||||
def exists(self, path):
|
||||
return self.repo.tree.exists(path) or self.dvctree.exists(path)
|
||||
return self.repo.tree.exists(path) or (
|
||||
self.dvctree and self.dvctree.exists(path)
|
||||
)
|
||||
|
||||
def isdir(self, path):
|
||||
return self.repo.tree.isdir(path) or (
|
||||
self.dvctree and self.dvctree.isdir(path)
|
||||
)
|
||||
|
||||
def isdvc(self, path, **kwargs):
|
||||
return self.dvctree is not None and self.dvctree.isdvc(path, **kwargs)
|
||||
|
||||
def isfile(self, path):
|
||||
return self.repo.tree.isfile(path) or (
|
||||
self.dvctree and self.dvctree.isfile(path)
|
||||
)
|
||||
|
||||
def isexec(self, path):
|
||||
if self.dvctree and self.dvctree.exists(path):
|
||||
return self.dvctree.isexec(path)
|
||||
return self.repo.tree.isexec(path)
|
||||
|
||||
def stat(self, path):
|
||||
return self.repo.tree.stat(path)
|
||||
|
||||
def _walk_one(self, walk):
|
||||
try:
|
||||
root, dirs, files = next(walk)
|
||||
except StopIteration:
|
||||
return
|
||||
yield root, dirs, files
|
||||
for _ in dirs:
|
||||
yield from self._walk_one(walk)
|
||||
|
||||
def _walk(self, dvc_walk, repo_walk, dvcfiles=False):
|
||||
try:
|
||||
_, dvc_dirs, dvc_fnames = next(dvc_walk)
|
||||
repo_root, repo_dirs, repo_fnames = next(repo_walk)
|
||||
except StopIteration:
|
||||
return
|
||||
|
||||
# separate subdirs into shared dirs, dvc-only dirs, repo-only dirs
|
||||
dvc_set = set(dvc_dirs)
|
||||
repo_set = set(repo_dirs)
|
||||
dvc_only = list(dvc_set - repo_set)
|
||||
repo_only = list(repo_set - dvc_set)
|
||||
shared = list(dvc_set & repo_set)
|
||||
dirs = shared + dvc_only + repo_only
|
||||
|
||||
# merge file lists
|
||||
files = {
|
||||
fname
|
||||
for fname in dvc_fnames + repo_fnames
|
||||
if dvcfiles or not is_valid_filename(fname)
|
||||
}
|
||||
|
||||
yield repo_root, dirs, list(files)
|
||||
|
||||
# set dir order for next recursion level - shared dirs first so that
|
||||
# next() for both generators recurses into the same shared directory
|
||||
dvc_dirs[:] = [dirname for dirname in dirs if dirname in dvc_set]
|
||||
repo_dirs[:] = [dirname for dirname in dirs if dirname in repo_set]
|
||||
|
||||
for dirname in dirs:
|
||||
if dirname in shared:
|
||||
yield from self._walk(dvc_walk, repo_walk, dvcfiles=dvcfiles)
|
||||
elif dirname in dvc_set:
|
||||
yield from self._walk_one(dvc_walk)
|
||||
elif dirname in repo_set:
|
||||
yield from self._walk_one(repo_walk)
|
||||
|
||||
def walk(self, top, topdown=True, dvcfiles=False, **kwargs):
|
||||
"""Walk and merge both DVC and repo trees.
|
||||
|
||||
Args:
|
||||
top: path to walk from
|
||||
topdown: if True, tree will be walked from top down.
|
||||
dvcfiles: if True, dvcfiles will be included in the files list
|
||||
for walked directories.
|
||||
|
||||
Any kwargs will be passed into methods used for fetching and/or
|
||||
streaming DVC outs from remotes.
|
||||
"""
|
||||
assert topdown
|
||||
|
||||
if not self.exists(top):
|
||||
raise FileNotFoundError
|
||||
|
||||
if not self.isdir(top):
|
||||
raise NotADirectoryError
|
||||
|
||||
dvc_exists = self.dvctree and self.dvctree.exists(top)
|
||||
repo_exists = self.repo.tree.exists(top)
|
||||
if dvc_exists and not repo_exists:
|
||||
yield from self.dvctree.walk(top, topdown=topdown, **kwargs)
|
||||
return
|
||||
if repo_exists and not dvc_exists:
|
||||
yield from self.repo.tree.walk(top, topdown=topdown)
|
||||
return
|
||||
if not dvc_exists and not repo_exists:
|
||||
raise FileNotFoundError
|
||||
|
||||
dvc_walk = self.dvctree.walk(top, topdown=topdown, **kwargs)
|
||||
repo_walk = self.repo.tree.walk(top, topdown=topdown)
|
||||
yield from self._walk(dvc_walk, repo_walk, dvcfiles=dvcfiles)
|
||||
|
||||
def walk_files(self, top, **kwargs):
|
||||
for root, _, files in self.walk(top, **kwargs):
|
||||
for fname in files:
|
||||
yield PathInfo(root) / fname
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
"""Return file checksum for specified path.
|
||||
|
||||
If path_info is a DVC out, the pre-computed checksum for the file
|
||||
will be used. If path_info is a git file, MD5 will be computed for
|
||||
the git object.
|
||||
"""
|
||||
if not self.exists(path_info):
|
||||
raise FileNotFoundError
|
||||
if self.dvctree and self.dvctree.exists(path_info):
|
||||
try:
|
||||
return self.dvctree.get_file_checksum(path_info)
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
return file_md5(path_info, self)[0]
|
||||
|
||||
def copytree(self, top, dest):
|
||||
top = PathInfo(top)
|
||||
dest = PathInfo(dest)
|
||||
|
||||
if not self.exists(top):
|
||||
raise FileNotFoundError
|
||||
|
||||
if self.isfile(top):
|
||||
makedirs(dest.parent, exist_ok=True)
|
||||
with self.open(top, mode="rb") as fobj:
|
||||
copy_fobj_to_file(fobj, dest)
|
||||
return
|
||||
|
||||
for root, _, files in self.walk(top):
|
||||
root = PathInfo(root)
|
||||
dest_dir = root.relative_to(top)
|
||||
makedirs(dest_dir, exist_ok=True)
|
||||
for fname in files:
|
||||
src = root / fname
|
||||
dest = dest_dir / fname
|
||||
with self.open(src, mode="rb") as fobj:
|
||||
copy_fobj_to_file(fobj, dest)
|
||||
|
||||
+16
-7
@@ -1,14 +1,23 @@
|
||||
from ..dvcfile import Dvcfile
|
||||
from . import locked
|
||||
|
||||
|
||||
@locked
|
||||
def update(self, target, rev=None):
|
||||
from ..dvcfile import Dvcfile
|
||||
def update(self, targets, rev=None, recursive=False):
|
||||
if not targets:
|
||||
targets = [None]
|
||||
|
||||
dvcfile = Dvcfile(self, target)
|
||||
stage = dvcfile.stage
|
||||
stage.update(rev)
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
|
||||
dvcfile.dump(stage)
|
||||
stages = set()
|
||||
for target in targets:
|
||||
stages.update(self.collect(target, recursive=recursive))
|
||||
|
||||
return stage
|
||||
for stage in stages:
|
||||
stage.update(rev)
|
||||
dvcfile = Dvcfile(self, stage.path)
|
||||
dvcfile.dump(stage)
|
||||
stages.add(stage)
|
||||
|
||||
return list(stages)
|
||||
|
||||
+1
-1
@@ -38,7 +38,7 @@ class RWLockFileFormatError(DvcException):
|
||||
def _edit_rwlock(lock_dir):
|
||||
path = os.path.join(lock_dir, "rwlock")
|
||||
try:
|
||||
with open(path, "r") as fobj:
|
||||
with open(path) as fobj:
|
||||
lock = SCHEMA(json.load(fobj))
|
||||
except FileNotFoundError:
|
||||
lock = SCHEMA({})
|
||||
|
||||
+21
-4
@@ -1,8 +1,8 @@
|
||||
from voluptuous import Any, Optional, Required, Schema
|
||||
|
||||
from dvc import dependency, output
|
||||
from dvc.output import CHECKSUMS_SCHEMA
|
||||
from dvc.stage.params import OutputParams, StageParams
|
||||
from dvc.output import CHECKSUMS_SCHEMA, BaseOutput
|
||||
from dvc.stage.params import StageParams
|
||||
|
||||
STAGES = "stages"
|
||||
SINGLE_STAGE_SCHEMA = {
|
||||
@@ -25,16 +25,33 @@ LOCK_FILE_STAGE_SCHEMA = {
|
||||
}
|
||||
LOCKFILE_SCHEMA = {str: LOCK_FILE_STAGE_SCHEMA}
|
||||
|
||||
OUT_PSTAGE_DETAILED_SCHEMA = {
|
||||
str: {BaseOutput.PARAM_CACHE: bool, BaseOutput.PARAM_PERSIST: bool}
|
||||
}
|
||||
|
||||
PLOT_PSTAGE_SCHEMA = OUT_PSTAGE_DETAILED_SCHEMA.copy()
|
||||
PLOT_PSTAGE_SCHEMA[str][BaseOutput.PARAM_PLOT_TEMPLATE] = str
|
||||
|
||||
PARAM_PSTAGE_NON_DEFAULT_SCHEMA = {str: [str]}
|
||||
|
||||
SINGLE_PIPELINE_STAGE_SCHEMA = {
|
||||
str: {
|
||||
StageParams.PARAM_CMD: str,
|
||||
Optional(StageParams.PARAM_WDIR): str,
|
||||
Optional(StageParams.PARAM_DEPS): [str],
|
||||
Optional(StageParams.PARAM_PARAMS): [Any(str, {str: [str]})],
|
||||
Optional(StageParams.PARAM_PARAMS): [
|
||||
Any(str, PARAM_PSTAGE_NON_DEFAULT_SCHEMA)
|
||||
],
|
||||
Optional(StageParams.PARAM_LOCKED): bool,
|
||||
Optional(StageParams.PARAM_META): object,
|
||||
Optional(StageParams.PARAM_ALWAYS_CHANGED): bool,
|
||||
**{Optional(p.value): [str] for p in OutputParams},
|
||||
Optional(StageParams.PARAM_OUTS): [
|
||||
Any(str, OUT_PSTAGE_DETAILED_SCHEMA)
|
||||
],
|
||||
Optional(StageParams.PARAM_METRICS): [
|
||||
Any(str, OUT_PSTAGE_DETAILED_SCHEMA)
|
||||
],
|
||||
Optional(StageParams.PARAM_PLOTS): [Any(str, PLOT_PSTAGE_SCHEMA)],
|
||||
}
|
||||
}
|
||||
MULTI_STAGE_SCHEMA = {STAGES: SINGLE_PIPELINE_STAGE_SCHEMA}
|
||||
|
||||
+2
-2
@@ -17,7 +17,7 @@ class FileNotInRepoError(SCMError):
|
||||
|
||||
class CloneError(SCMError):
|
||||
def __init__(self, url, path):
|
||||
super().__init__("Failed to clone repo '{}' to '{}'".format(url, path))
|
||||
super().__init__(f"Failed to clone repo '{url}' to '{path}'")
|
||||
|
||||
|
||||
class RevError(SCMError):
|
||||
@@ -34,7 +34,7 @@ class NoSCMError(SCMError):
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class Base(object):
|
||||
class Base:
|
||||
"""Base class for source control management driver implementations."""
|
||||
|
||||
def __init__(self, root_dir=os.curdir):
|
||||
|
||||
+14
-10
@@ -30,7 +30,7 @@ class TqdmGit(Tqdm):
|
||||
if op_code:
|
||||
message = (op_code + " | " + message) if message else op_code
|
||||
if message:
|
||||
self.postfix["info"] = " {} | ".format(message)
|
||||
self.postfix["info"] = f" {message} | "
|
||||
self.update_to(cur_count, max_count)
|
||||
|
||||
@staticmethod
|
||||
@@ -211,7 +211,7 @@ class Git(Base):
|
||||
fobj.seek(fobj.tell() - 1, os.SEEK_SET)
|
||||
last = fobj.read(1)
|
||||
prefix = "" if last == "\n" else "\n"
|
||||
fobj.write("{}{}\n".format(prefix, entry))
|
||||
fobj.write(f"{prefix}{entry}\n")
|
||||
|
||||
def ignore_remove(self, path):
|
||||
entry, gitignore = self._get_gitignore(path)
|
||||
@@ -219,7 +219,7 @@ class Git(Base):
|
||||
if not os.path.exists(gitignore):
|
||||
return
|
||||
|
||||
with open(gitignore, "r") as fobj:
|
||||
with open(gitignore) as fobj:
|
||||
lines = fobj.readlines()
|
||||
|
||||
filtered = list(filter(lambda x: x.strip() != entry.strip(), lines))
|
||||
@@ -257,13 +257,13 @@ class Git(Base):
|
||||
infos = self.repo.remote().pull()
|
||||
for info in infos:
|
||||
if info.flags & info.ERROR:
|
||||
raise SCMError("pull failed: {}".format(info.note))
|
||||
raise SCMError(f"pull failed: {info.note}")
|
||||
|
||||
def push(self):
|
||||
infos = self.repo.remote().push()
|
||||
for info in infos:
|
||||
if info.flags & info.ERROR:
|
||||
raise SCMError("push failed: {}".format(info.summary))
|
||||
raise SCMError(f"push failed: {info.summary}")
|
||||
|
||||
def branch(self, branch):
|
||||
self.repo.git.branch(branch)
|
||||
@@ -296,7 +296,7 @@ class Git(Base):
|
||||
def _install_hook(self, name):
|
||||
hook = self._hook_path(name)
|
||||
with open(hook, "w+") as fobj:
|
||||
fobj.write("#!/bin/sh\nexec dvc git-hook {} $@\n".format(name))
|
||||
fobj.write(f"#!/bin/sh\nexec dvc git-hook {name} $@\n")
|
||||
|
||||
os.chmod(hook, 0o777)
|
||||
|
||||
@@ -312,7 +312,7 @@ class Git(Base):
|
||||
|
||||
config = {}
|
||||
if os.path.exists(config_path):
|
||||
with open(config_path, "r") as fobj:
|
||||
with open(config_path) as fobj:
|
||||
config = yaml.safe_load(fobj)
|
||||
|
||||
entry = {
|
||||
@@ -404,15 +404,15 @@ class Git(Base):
|
||||
# Try all the remotes and if it resolves unambiguously then take it
|
||||
if not Git.is_sha(rev):
|
||||
shas = {
|
||||
_resolve_rev("{}/{}".format(remote.name, rev))
|
||||
_resolve_rev(f"{remote.name}/{rev}")
|
||||
for remote in self.repo.remotes
|
||||
} - {None}
|
||||
if len(shas) > 1:
|
||||
raise RevError("ambiguous Git revision '{}'".format(rev))
|
||||
raise RevError(f"ambiguous Git revision '{rev}'")
|
||||
if len(shas) == 1:
|
||||
return shas.pop()
|
||||
|
||||
raise RevError("unknown Git revision '{}'".format(rev))
|
||||
raise RevError(f"unknown Git revision '{rev}'")
|
||||
|
||||
def has_rev(self, rev):
|
||||
try:
|
||||
@@ -439,3 +439,7 @@ class Git(Base):
|
||||
self._verify_hook("post-checkout")
|
||||
self._verify_hook("pre-commit")
|
||||
self._verify_hook("pre-push")
|
||||
|
||||
@property
|
||||
def no_commits(self):
|
||||
return not self.list_all_commits()
|
||||
|
||||
+45
-5
@@ -1,6 +1,7 @@
|
||||
import errno
|
||||
import io
|
||||
import os
|
||||
import stat
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.scm.tree import BaseTree
|
||||
@@ -42,10 +43,10 @@ class GitTree(BaseTree):
|
||||
|
||||
obj = self.git_object_by_path(path)
|
||||
if obj is None:
|
||||
msg = "No such file in branch '{}'".format(self.rev)
|
||||
raise IOError(errno.ENOENT, msg, relative_path)
|
||||
msg = f"No such file in branch '{self.rev}'"
|
||||
raise OSError(errno.ENOENT, msg, relative_path)
|
||||
if obj.mode == GIT_MODE_DIR:
|
||||
raise IOError(errno.EISDIR, "Is a directory", relative_path)
|
||||
raise OSError(errno.EISDIR, "Is a directory", relative_path)
|
||||
|
||||
# GitPython's obj.data_stream is a fragile thing, it is better to
|
||||
# read it immediately, also it needs to be to decoded if we follow
|
||||
@@ -87,7 +88,9 @@ class GitTree(BaseTree):
|
||||
import git
|
||||
|
||||
path = relpath(os.path.realpath(path), self.git.working_dir)
|
||||
assert path.split(os.sep, 1)[0] != ".."
|
||||
if path.split(os.sep, 1)[0] == "..":
|
||||
# path points outside of git repository
|
||||
return None
|
||||
|
||||
try:
|
||||
tree = self.git.tree(self.rev)
|
||||
@@ -135,6 +138,43 @@ class GitTree(BaseTree):
|
||||
|
||||
tree = self.git_object_by_path(top)
|
||||
if tree is None:
|
||||
raise IOError(errno.ENOENT, "No such file")
|
||||
raise OSError(errno.ENOENT, "No such file")
|
||||
|
||||
yield from self._walk(tree, topdown=topdown)
|
||||
|
||||
def isexec(self, path):
|
||||
if not self.exists(path):
|
||||
return False
|
||||
|
||||
mode = self.stat(path).st_mode
|
||||
return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
def stat(self, path):
|
||||
import git
|
||||
|
||||
def to_ctime(git_time):
|
||||
sec, nano_sec = git_time
|
||||
return sec + nano_sec / 1000000000
|
||||
|
||||
obj = self.git_object_by_path(path)
|
||||
if obj is None:
|
||||
raise OSError(errno.ENOENT, "No such file")
|
||||
entry = git.index.IndexEntry.from_blob(obj)
|
||||
|
||||
# os.stat_result takes a tuple in the form:
|
||||
# (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime)
|
||||
return os.stat_result(
|
||||
(
|
||||
entry.mode,
|
||||
entry.inode,
|
||||
entry.dev,
|
||||
0,
|
||||
entry.uid,
|
||||
entry.gid,
|
||||
entry.size,
|
||||
# git index has no atime equivalent, use mtime
|
||||
to_ctime(entry.mtime),
|
||||
to_ctime(entry.mtime),
|
||||
to_ctime(entry.ctime),
|
||||
)
|
||||
)
|
||||
|
||||
+13
-5
@@ -1,10 +1,8 @@
|
||||
import os
|
||||
import stat
|
||||
|
||||
from dvc.compat import fspath
|
||||
|
||||
|
||||
class BaseTree(object):
|
||||
class BaseTree:
|
||||
"""Abstract class to represent access to files"""
|
||||
|
||||
@property
|
||||
@@ -31,6 +29,12 @@ class BaseTree(object):
|
||||
- it could raise exceptions, there is no onerror argument
|
||||
"""
|
||||
|
||||
def walk_files(self, top):
|
||||
for root, _, files in self.walk(top):
|
||||
for file in files:
|
||||
# NOTE: os.path.join is ~5.5 times slower
|
||||
yield f"{root}{os.sep}{file}"
|
||||
|
||||
|
||||
class WorkingTree(BaseTree):
|
||||
"""Proxies the repo file access methods to working tree files"""
|
||||
@@ -44,6 +48,8 @@ class WorkingTree(BaseTree):
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8"):
|
||||
"""Open file and return a stream."""
|
||||
if "b" in mode:
|
||||
encoding = None
|
||||
return open(path, mode=mode, encoding=encoding)
|
||||
|
||||
def exists(self, path):
|
||||
@@ -66,8 +72,6 @@ class WorkingTree(BaseTree):
|
||||
- it could raise exceptions, there is no onerror argument
|
||||
"""
|
||||
|
||||
top = fspath(top)
|
||||
|
||||
def onerror(e):
|
||||
raise e
|
||||
|
||||
@@ -80,6 +84,10 @@ class WorkingTree(BaseTree):
|
||||
mode = os.stat(path).st_mode
|
||||
return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
@staticmethod
|
||||
def stat(path):
|
||||
return os.stat(path)
|
||||
|
||||
|
||||
def is_working_tree(tree):
|
||||
return isinstance(tree, WorkingTree) or isinstance(
|
||||
|
||||
+60
-23
@@ -6,32 +6,45 @@ from typing import TYPE_CHECKING, List
|
||||
from funcy import lsplit, rpartial
|
||||
|
||||
from dvc.dependency import ParamsDependency
|
||||
from dvc.output import BaseOutput
|
||||
from dvc.stage.params import StageParams
|
||||
from dvc.stage.utils import resolve_wdir
|
||||
from dvc.utils.collections import apply_diff
|
||||
from dvc.utils.stage import parse_stage_for_update
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.stage import PipelineStage, Stage
|
||||
|
||||
PARAM_PATH = ParamsDependency.PARAM_PATH
|
||||
PARAM_PARAMS = ParamsDependency.PARAM_PARAMS
|
||||
PARAM_PATH = ParamsDependency.PARAM_PATH
|
||||
|
||||
PARAM_DEPS = StageParams.PARAM_DEPS
|
||||
PARAM_OUTS = StageParams.PARAM_OUTS
|
||||
|
||||
PARAM_CACHE = BaseOutput.PARAM_CACHE
|
||||
PARAM_METRIC = BaseOutput.PARAM_METRIC
|
||||
PARAM_PLOT = BaseOutput.PARAM_PLOT
|
||||
PARAM_PERSIST = BaseOutput.PARAM_PERSIST
|
||||
|
||||
DEFAULT_PARAMS_FILE = ParamsDependency.DEFAULT_PARAMS_FILE
|
||||
|
||||
|
||||
sort_by_path = partial(sorted, key=attrgetter("def_path"))
|
||||
|
||||
|
||||
def _get_outs(stage: "PipelineStage"):
|
||||
outs_bucket = {}
|
||||
for o in sort_by_path(stage.outs):
|
||||
bucket_key = ["metrics"] if o.metric else ["outs"]
|
||||
def _get_out(out):
|
||||
res = OrderedDict()
|
||||
if not out.use_cache:
|
||||
res[PARAM_CACHE] = False
|
||||
if out.persist:
|
||||
res[PARAM_PERSIST] = True
|
||||
if out.plot and isinstance(out.plot, dict):
|
||||
res.update(out.plot)
|
||||
return out.def_path if not res else {out.def_path: res}
|
||||
|
||||
if not o.metric and o.persist:
|
||||
bucket_key += ["persist"]
|
||||
if not o.use_cache:
|
||||
bucket_key += ["no_cache"]
|
||||
key = "_".join(bucket_key)
|
||||
outs_bucket[key] = outs_bucket.get(key, []) + [o.def_path]
|
||||
return [(key, outs_bucket[key]) for key in sorted(outs_bucket.keys())]
|
||||
|
||||
def _get_outs(outs):
|
||||
return [_get_out(out) for out in sort_by_path(outs)]
|
||||
|
||||
|
||||
def get_params_deps(stage: "PipelineStage"):
|
||||
@@ -56,14 +69,22 @@ def _serialize_params(params: List[ParamsDependency]):
|
||||
for param_dep in sort_by_path(params):
|
||||
dump = param_dep.dumpd()
|
||||
path, params = dump[PARAM_PATH], dump[PARAM_PARAMS]
|
||||
k = list(params.keys())
|
||||
if not k:
|
||||
continue
|
||||
key_vals[path] = OrderedDict([(key, params[key]) for key in sorted(k)])
|
||||
if isinstance(params, dict):
|
||||
k = list(params.keys())
|
||||
if not k:
|
||||
continue
|
||||
key_vals[path] = OrderedDict(
|
||||
[(key, params[key]) for key in sorted(k)]
|
||||
)
|
||||
else:
|
||||
assert isinstance(params, list)
|
||||
k = params
|
||||
key_vals = OrderedDict()
|
||||
# params from default file is always kept at the start of the `params:`
|
||||
if path == DEFAULT_PARAMS_FILE:
|
||||
keys = k + keys
|
||||
key_vals.move_to_end(path, last=False)
|
||||
if key_vals:
|
||||
key_vals.move_to_end(path, last=False)
|
||||
else:
|
||||
# if it's not a default file, change the shape
|
||||
# to: {path: k}
|
||||
@@ -77,10 +98,23 @@ def to_pipeline_file(stage: "PipelineStage"):
|
||||
|
||||
res = [
|
||||
(stage.PARAM_CMD, stage.cmd),
|
||||
(stage.PARAM_WDIR, stage.resolve_wdir()),
|
||||
(stage.PARAM_WDIR, resolve_wdir(stage.wdir, stage.path)),
|
||||
(stage.PARAM_DEPS, sorted([d.def_path for d in deps])),
|
||||
(stage.PARAM_PARAMS, serialized_params),
|
||||
*_get_outs(stage),
|
||||
(
|
||||
PARAM_OUTS,
|
||||
_get_outs(
|
||||
[out for out in stage.outs if not (out.metric or out.plot)]
|
||||
),
|
||||
),
|
||||
(
|
||||
stage.PARAM_METRICS,
|
||||
_get_outs([out for out in stage.outs if out.metric]),
|
||||
),
|
||||
(
|
||||
stage.PARAM_PLOTS,
|
||||
_get_outs([out for out in stage.outs if out.plot]),
|
||||
),
|
||||
(stage.PARAM_LOCKED, stage.locked),
|
||||
(stage.PARAM_ALWAYS_CHANGED, stage.always_changed),
|
||||
]
|
||||
@@ -97,18 +131,21 @@ def to_single_stage_lockfile(stage: "Stage") -> dict:
|
||||
deps, outs = [
|
||||
[
|
||||
OrderedDict(
|
||||
[("path", item.def_path), (item.checksum_type, item.checksum)]
|
||||
[
|
||||
(PARAM_PATH, item.def_path),
|
||||
(item.checksum_type, item.checksum),
|
||||
]
|
||||
)
|
||||
for item in sort_by_path(items)
|
||||
]
|
||||
for items in [deps, stage.outs]
|
||||
]
|
||||
if deps:
|
||||
res["deps"] = deps
|
||||
res[PARAM_DEPS] = deps
|
||||
if params:
|
||||
_, res["params"] = _serialize_params(params)
|
||||
_, res[PARAM_PARAMS] = _serialize_params(params)
|
||||
if outs:
|
||||
res["outs"] = outs
|
||||
res[PARAM_OUTS] = outs
|
||||
|
||||
return res
|
||||
|
||||
|
||||
+169
-450
@@ -1,34 +1,35 @@
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import signal
|
||||
import subprocess
|
||||
import threading
|
||||
from itertools import chain, product
|
||||
import string
|
||||
from collections import defaultdict
|
||||
|
||||
from funcy import project
|
||||
|
||||
import dvc.dependency as dependency
|
||||
import dvc.output as output
|
||||
import dvc.prompt as prompt
|
||||
from dvc.exceptions import CheckoutError, DvcException
|
||||
from dvc.utils import dict_md5, fix_env, relpath
|
||||
from dvc.utils.fs import path_isin
|
||||
from dvc.utils import relpath
|
||||
|
||||
from . import params
|
||||
from .decorators import rwlocked, unlocked_repo
|
||||
from .exceptions import (
|
||||
MissingDataSource,
|
||||
StageCmdFailedError,
|
||||
StageCommitError,
|
||||
StagePathNotDirectoryError,
|
||||
StagePathNotFoundError,
|
||||
StagePathOutsideError,
|
||||
StageUpdateError,
|
||||
from .decorators import rwlocked
|
||||
from .exceptions import StageUpdateError
|
||||
from .imports import sync_import, update_import
|
||||
from .run import run_stage
|
||||
from .utils import (
|
||||
check_circular_dependency,
|
||||
check_duplicated_arguments,
|
||||
check_missing_outputs,
|
||||
check_stage_path,
|
||||
compute_md5,
|
||||
fill_stage_dependencies,
|
||||
fill_stage_outputs,
|
||||
get_dump,
|
||||
stage_dump_eq,
|
||||
)
|
||||
from .params import OutputParams
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
# Disallow all punctuation characters except hyphen and underscore
|
||||
INVALID_STAGENAME_CHARS = set(string.punctuation) - {"_", "-"}
|
||||
|
||||
|
||||
def loads_from(cls, repo, path, wdir, data):
|
||||
@@ -56,25 +57,27 @@ def create_stage(cls, repo, path, **kwargs):
|
||||
wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
|
||||
path = os.path.abspath(path)
|
||||
check_dvc_filename(path)
|
||||
cls._check_stage_path(repo, wdir, is_wdir=kwargs.get("wdir"))
|
||||
cls._check_stage_path(repo, os.path.dirname(path))
|
||||
check_stage_path(repo, wdir, is_wdir=kwargs.get("wdir"))
|
||||
check_stage_path(repo, os.path.dirname(path))
|
||||
|
||||
stage = loads_from(cls, repo, path, wdir, kwargs)
|
||||
stage._fill_stage_outputs(**kwargs)
|
||||
stage._fill_stage_dependencies(**kwargs)
|
||||
stage._check_circular_dependency()
|
||||
stage._check_duplicated_arguments()
|
||||
fill_stage_outputs(stage, **kwargs)
|
||||
fill_stage_dependencies(
|
||||
stage, **project(kwargs, ["deps", "erepo", "params"])
|
||||
)
|
||||
check_circular_dependency(stage)
|
||||
check_duplicated_arguments(stage)
|
||||
|
||||
if stage and stage.dvcfile.exists():
|
||||
has_persist_outs = any(out.persist for out in stage.outs)
|
||||
ignore_build_cache = (
|
||||
kwargs.get("ignore_build_cache", False) or has_persist_outs
|
||||
ignore_run_cache = (
|
||||
not kwargs.get("run_cache", True) or has_persist_outs
|
||||
)
|
||||
if has_persist_outs:
|
||||
logger.warning("Build cache is ignored when persisting outputs.")
|
||||
|
||||
if not ignore_build_cache and stage.can_be_skipped:
|
||||
logger.info("Stage is cached, skipping.")
|
||||
if not ignore_run_cache and stage.can_be_skipped:
|
||||
logger.info("Stage is cached, skipping")
|
||||
return None
|
||||
|
||||
return stage
|
||||
@@ -141,14 +144,10 @@ class Stage(params.StageParams):
|
||||
self._dvcfile = dvcfile
|
||||
|
||||
def __repr__(self):
|
||||
return "Stage: '{path}'".format(
|
||||
path=self.path_in_repo if self.path else "No path"
|
||||
)
|
||||
return f"Stage: '{self.addressing}'"
|
||||
|
||||
def __str__(self):
|
||||
return "stage: '{path}'".format(
|
||||
path=self.relpath if self.path else "No path"
|
||||
)
|
||||
return f"stage: '{self.addressing}'"
|
||||
|
||||
@property
|
||||
def addressing(self):
|
||||
@@ -156,7 +155,7 @@ class Stage(params.StageParams):
|
||||
Useful for alternative presentations where we don't need
|
||||
`Stage:` prefix.
|
||||
"""
|
||||
return self.relpath
|
||||
return self.relpath if self.path else "No path"
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.path_in_repo)
|
||||
@@ -201,21 +200,25 @@ class Stage(params.StageParams):
|
||||
|
||||
return isinstance(self.deps[0], dependency.RepoDependency)
|
||||
|
||||
def _changed_deps(self):
|
||||
def changed_deps(self):
|
||||
if self.locked:
|
||||
return False
|
||||
|
||||
if self.is_callback:
|
||||
logger.warning(
|
||||
'{stage} is a "callback" stage '
|
||||
logger.debug(
|
||||
'%s is a "callback" stage '
|
||||
"(has a command and no dependencies) and thus always "
|
||||
"considered as changed.".format(stage=self)
|
||||
"considered as changed.",
|
||||
self,
|
||||
)
|
||||
return True
|
||||
|
||||
if self.always_changed:
|
||||
return True
|
||||
|
||||
return self._changed_deps()
|
||||
|
||||
def _changed_deps(self):
|
||||
for dep in self.deps:
|
||||
status = dep.status()
|
||||
if status:
|
||||
@@ -226,10 +229,9 @@ class Stage(params.StageParams):
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _changed_outs(self):
|
||||
def changed_outs(self):
|
||||
for out in self.outs:
|
||||
status = out.status()
|
||||
if status:
|
||||
@@ -243,28 +245,24 @@ class Stage(params.StageParams):
|
||||
|
||||
return False
|
||||
|
||||
def stage_changed(self, warn=False):
|
||||
changed = self.md5 != self._compute_md5()
|
||||
if changed and warn:
|
||||
logger.warning("DVC-file '{}' changed.".format(self.relpath))
|
||||
def changed_stage(self):
|
||||
changed = self.md5 != self.compute_md5()
|
||||
if changed:
|
||||
logger.debug(self._changed_stage_entry())
|
||||
return changed
|
||||
|
||||
@rwlocked(read=["deps", "outs"])
|
||||
def changed(self):
|
||||
if self._changed():
|
||||
logger.warning("{} changed.".format(self))
|
||||
return True
|
||||
|
||||
logger.debug("{} didn't change.".format(self))
|
||||
return False
|
||||
|
||||
def _changed(self):
|
||||
# Short-circuit order: stage md5 is fast, deps are expected to change
|
||||
return (
|
||||
self.stage_changed(warn=True)
|
||||
or self._changed_deps()
|
||||
or self._changed_outs()
|
||||
is_changed = (
|
||||
# Short-circuit order: stage md5 is fast,
|
||||
# deps are expected to change
|
||||
self.changed_stage()
|
||||
or self.changed_deps()
|
||||
or self.changed_outs()
|
||||
)
|
||||
if is_changed:
|
||||
logger.debug("%s changed.", self)
|
||||
return is_changed
|
||||
|
||||
@rwlocked(write=["outs"])
|
||||
def remove_outs(self, ignore_remove=False, force=False):
|
||||
@@ -272,13 +270,10 @@ class Stage(params.StageParams):
|
||||
for out in self.outs:
|
||||
if out.persist and not force:
|
||||
out.unprotect()
|
||||
else:
|
||||
logger.debug(
|
||||
"Removing output '{out}' of {stage}.".format(
|
||||
out=out, stage=self
|
||||
)
|
||||
)
|
||||
out.remove(ignore_remove=ignore_remove)
|
||||
continue
|
||||
|
||||
logger.debug(f"Removing output '{out}' of {self}.")
|
||||
out.remove(ignore_remove=ignore_remove)
|
||||
|
||||
def unprotect_outs(self):
|
||||
for out in self.outs:
|
||||
@@ -294,8 +289,8 @@ class Stage(params.StageParams):
|
||||
|
||||
@rwlocked(read=["deps"], write=["outs"])
|
||||
def reproduce(self, interactive=False, **kwargs):
|
||||
|
||||
if not kwargs.get("force", False) and not self.changed():
|
||||
if not (kwargs.get("force", False) or self.changed()):
|
||||
logger.info("Stage '%s' didn't change, skipping", self.addressing)
|
||||
return None
|
||||
|
||||
msg = (
|
||||
@@ -308,45 +303,14 @@ class Stage(params.StageParams):
|
||||
|
||||
self.run(**kwargs)
|
||||
|
||||
logger.debug("{stage} was reproduced".format(stage=self))
|
||||
logger.debug(f"{self} was reproduced")
|
||||
|
||||
return self
|
||||
|
||||
def update(self, rev=None):
|
||||
if not self.is_repo_import and not self.is_import:
|
||||
if not (self.is_repo_import or self.is_import):
|
||||
raise StageUpdateError(self.relpath)
|
||||
|
||||
self.deps[0].update(rev=rev)
|
||||
locked = self.locked
|
||||
self.locked = False
|
||||
try:
|
||||
self.reproduce()
|
||||
finally:
|
||||
self.locked = locked
|
||||
|
||||
@staticmethod
|
||||
def _check_stage_path(repo, path, is_wdir=False):
|
||||
assert repo is not None
|
||||
|
||||
error_msg = "{wdir_or_path} '{path}' {{}}".format(
|
||||
wdir_or_path="stage working dir" if is_wdir else "file path",
|
||||
path=path,
|
||||
)
|
||||
|
||||
real_path = os.path.realpath(path)
|
||||
if not os.path.exists(real_path):
|
||||
raise StagePathNotFoundError(error_msg.format("does not exist"))
|
||||
|
||||
if not os.path.isdir(real_path):
|
||||
raise StagePathNotDirectoryError(
|
||||
error_msg.format("is not directory")
|
||||
)
|
||||
|
||||
proj_dir = os.path.realpath(repo.root_dir)
|
||||
if real_path != proj_dir and not path_isin(real_path, proj_dir):
|
||||
raise StagePathOutsideError(
|
||||
error_msg.format("is outside of DVC repo")
|
||||
)
|
||||
update_import(self, rev=rev)
|
||||
|
||||
@property
|
||||
def can_be_skipped(self):
|
||||
@@ -359,42 +323,15 @@ class Stage(params.StageParams):
|
||||
|
||||
@property
|
||||
def is_cached(self):
|
||||
"""
|
||||
Checks if this stage has been already ran and stored
|
||||
"""
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.remote.s3 import S3Remote
|
||||
|
||||
"""Checks if this stage has been already ran and stored"""
|
||||
old = self.reload()
|
||||
if old._changed_outs():
|
||||
if old.changed_outs():
|
||||
return False
|
||||
|
||||
# NOTE: need to save checksums for deps in order to compare them
|
||||
# with what is written in the old stage.
|
||||
self._save_deps()
|
||||
|
||||
old_d = old.dumpd()
|
||||
new_d = self.dumpd()
|
||||
|
||||
# NOTE: need to remove checksums from old dict in order to compare
|
||||
# it to the new one, since the new one doesn't have checksums yet.
|
||||
old_d.pop(self.PARAM_MD5, None)
|
||||
new_d.pop(self.PARAM_MD5, None)
|
||||
outs = old_d.get(self.PARAM_OUTS, [])
|
||||
for out in outs:
|
||||
out.pop(LocalRemote.PARAM_CHECKSUM, None)
|
||||
out.pop(S3Remote.PARAM_CHECKSUM, None)
|
||||
|
||||
# outs and deps are lists of dicts. To check equality, we need to make
|
||||
# them independent of the order, so, we convert them to dicts.
|
||||
combination = product(
|
||||
[old_d, new_d], [self.PARAM_DEPS, self.PARAM_OUTS]
|
||||
)
|
||||
for coll, key in combination:
|
||||
if coll.get(key):
|
||||
coll[key] = {item["path"]: item for item in coll[key]}
|
||||
|
||||
if old_d != new_d:
|
||||
self.save_deps()
|
||||
if not stage_dump_eq(Stage, old.dumpd(), self.dumpd()):
|
||||
return False
|
||||
|
||||
# NOTE: committing to prevent potential data duplication. For example
|
||||
@@ -410,290 +347,80 @@ class Stage(params.StageParams):
|
||||
|
||||
return True
|
||||
|
||||
def _fill_stage_outputs(self, **kwargs):
|
||||
assert not self.outs
|
||||
|
||||
self.outs = []
|
||||
for key in (p.value for p in OutputParams):
|
||||
self.outs += output.loads_from(
|
||||
self,
|
||||
kwargs.get(key, []),
|
||||
use_cache="no_cache" not in key,
|
||||
persist="persist" in key,
|
||||
metric="metrics" in key,
|
||||
)
|
||||
|
||||
def _fill_stage_dependencies(self, **kwargs):
|
||||
assert not self.deps
|
||||
self.deps = []
|
||||
self.deps += dependency.loads_from(
|
||||
self, kwargs.get("deps", []), erepo=kwargs.get("erepo", None)
|
||||
)
|
||||
self.deps += dependency.loads_params(self, kwargs.get("params", []))
|
||||
|
||||
def _fix_outs_deps_path(self, wdir):
|
||||
for out in chain(self.outs, self.deps):
|
||||
if out.is_in_repo:
|
||||
out.def_path = relpath(out.path_info, wdir)
|
||||
|
||||
def resolve_wdir(self):
|
||||
rel_wdir = relpath(self.wdir, os.path.dirname(self.path))
|
||||
return (
|
||||
pathlib.PurePath(rel_wdir).as_posix() if rel_wdir != "." else None
|
||||
)
|
||||
|
||||
def dumpd(self):
|
||||
return {
|
||||
key: value
|
||||
for key, value in {
|
||||
Stage.PARAM_MD5: self.md5,
|
||||
Stage.PARAM_CMD: self.cmd,
|
||||
Stage.PARAM_WDIR: self.resolve_wdir(),
|
||||
Stage.PARAM_LOCKED: self.locked,
|
||||
Stage.PARAM_DEPS: [d.dumpd() for d in self.deps],
|
||||
Stage.PARAM_OUTS: [o.dumpd() for o in self.outs],
|
||||
Stage.PARAM_ALWAYS_CHANGED: self.always_changed,
|
||||
}.items()
|
||||
if value
|
||||
}
|
||||
return get_dump(self)
|
||||
|
||||
def _compute_md5(self):
|
||||
from dvc.output.base import BaseOutput
|
||||
|
||||
d = self.dumpd()
|
||||
|
||||
# Remove md5 and meta, these should not affect stage md5
|
||||
d.pop(self.PARAM_MD5, None)
|
||||
d.pop(self.PARAM_META, None)
|
||||
|
||||
# Ignore the wdir default value. In this case DVC-file w/o
|
||||
# wdir has the same md5 as a file with the default value specified.
|
||||
# It's important for backward compatibility with pipelines that
|
||||
# didn't have WDIR in their DVC-files.
|
||||
if d.get(self.PARAM_WDIR) == ".":
|
||||
del d[self.PARAM_WDIR]
|
||||
|
||||
# NOTE: excluding parameters that don't affect the state of the
|
||||
# pipeline. Not excluding `LocalOutput.PARAM_CACHE`, because if
|
||||
# it has changed, we might not have that output in our cache.
|
||||
m = dict_md5(
|
||||
d,
|
||||
exclude=[
|
||||
self.PARAM_LOCKED,
|
||||
BaseOutput.PARAM_METRIC,
|
||||
BaseOutput.PARAM_PERSIST,
|
||||
],
|
||||
)
|
||||
logger.debug("Computed {} md5: '{}'".format(self, m))
|
||||
def compute_md5(self):
|
||||
# `dvc add`ed files don't need stage md5
|
||||
if self.is_data_source and not (self.is_import or self.is_repo_import):
|
||||
m = None
|
||||
else:
|
||||
m = compute_md5(self)
|
||||
logger.debug(f"Computed {self} md5: '{m}'")
|
||||
return m
|
||||
|
||||
def _save_deps(self):
|
||||
for dep in self.deps:
|
||||
dep.save()
|
||||
|
||||
def save(self):
|
||||
self._save_deps()
|
||||
|
||||
for out in self.outs:
|
||||
out.save()
|
||||
|
||||
self.md5 = self._compute_md5()
|
||||
self.save_deps()
|
||||
self.save_outs()
|
||||
self.md5 = self.compute_md5()
|
||||
|
||||
self.repo.stage_cache.save(self)
|
||||
|
||||
def save_deps(self):
|
||||
for dep in self.deps:
|
||||
dep.save()
|
||||
|
||||
def save_outs(self):
|
||||
for out in self.outs:
|
||||
out.save()
|
||||
|
||||
def ignore_outs(self):
|
||||
for out in self.outs:
|
||||
out.ignore()
|
||||
|
||||
@staticmethod
|
||||
def _changed_entries(entries):
|
||||
return [
|
||||
str(entry)
|
||||
for entry in entries
|
||||
if entry.checksum and entry.changed_checksum()
|
||||
]
|
||||
return [str(entry) for entry in entries if entry.changed_checksum()]
|
||||
|
||||
def check_can_commit(self, force):
|
||||
def _changed_stage_entry(self):
|
||||
return f"'md5' of {self} changed."
|
||||
|
||||
def changed_entries(self):
|
||||
changed_deps = self._changed_entries(self.deps)
|
||||
changed_outs = self._changed_entries(self.outs)
|
||||
|
||||
if changed_deps or changed_outs or self.stage_changed():
|
||||
msg = (
|
||||
"dependencies {}".format(changed_deps) if changed_deps else ""
|
||||
)
|
||||
msg += " and " if (changed_deps and changed_outs) else ""
|
||||
msg += "outputs {}".format(changed_outs) if changed_outs else ""
|
||||
msg += "md5" if not (changed_deps or changed_outs) else ""
|
||||
msg += " of {} changed. ".format(self)
|
||||
msg += "Are you sure you want to commit it?"
|
||||
if not force and not prompt.confirm(msg):
|
||||
raise StageCommitError(
|
||||
"unable to commit changed {}. Use `-f|--force` to "
|
||||
"force.".format(self)
|
||||
)
|
||||
self.save()
|
||||
return (
|
||||
changed_deps,
|
||||
changed_outs,
|
||||
self._changed_stage_entry() if self.changed_stage() else None,
|
||||
)
|
||||
|
||||
@rwlocked(write=["outs"])
|
||||
def commit(self):
|
||||
for out in self.outs:
|
||||
out.commit()
|
||||
|
||||
@staticmethod
|
||||
def _warn_if_fish(executable): # pragma: no cover
|
||||
if (
|
||||
executable is None
|
||||
or os.path.basename(os.path.realpath(executable)) != "fish"
|
||||
):
|
||||
return
|
||||
|
||||
logger.warning(
|
||||
"DVC detected that you are using fish as your default "
|
||||
"shell. Be aware that it might cause problems by overwriting "
|
||||
"your current environment variables with values defined "
|
||||
"in '.fishrc', which might affect your command. See "
|
||||
"https://github.com/iterative/dvc/issues/1307. "
|
||||
)
|
||||
|
||||
def _check_circular_dependency(self):
|
||||
from dvc.exceptions import CircularDependencyError
|
||||
|
||||
circular_dependencies = set(d.path_info for d in self.deps) & set(
|
||||
o.path_info for o in self.outs
|
||||
)
|
||||
|
||||
if circular_dependencies:
|
||||
raise CircularDependencyError(str(circular_dependencies.pop()))
|
||||
|
||||
def _check_duplicated_arguments(self):
|
||||
from dvc.exceptions import ArgumentDuplicationError
|
||||
from collections import Counter
|
||||
|
||||
path_counts = Counter(edge.path_info for edge in self.deps + self.outs)
|
||||
|
||||
for path, occurrence in path_counts.items():
|
||||
if occurrence > 1:
|
||||
raise ArgumentDuplicationError(str(path))
|
||||
|
||||
@unlocked_repo
|
||||
def _run(self):
|
||||
kwargs = {"cwd": self.wdir, "env": fix_env(None), "close_fds": True}
|
||||
|
||||
if os.name == "nt":
|
||||
kwargs["shell"] = True
|
||||
cmd = self.cmd
|
||||
else:
|
||||
# NOTE: when you specify `shell=True`, `Popen` [1] will default to
|
||||
# `/bin/sh` on *nix and will add ["/bin/sh", "-c"] to your command.
|
||||
# But we actually want to run the same shell that we are running
|
||||
# from right now, which is usually determined by the `SHELL` env
|
||||
# var. So instead, we compose our command on our own, making sure
|
||||
# to include special flags to prevent shell from reading any
|
||||
# configs and modifying env, which may change the behavior or the
|
||||
# command we are running. See [2] for more info.
|
||||
#
|
||||
# [1] https://github.com/python/cpython/blob/3.7/Lib/subprocess.py
|
||||
# #L1426
|
||||
# [2] https://github.com/iterative/dvc/issues/2506
|
||||
# #issuecomment-535396799
|
||||
kwargs["shell"] = False
|
||||
executable = os.getenv("SHELL") or "/bin/sh"
|
||||
|
||||
self._warn_if_fish(executable)
|
||||
|
||||
opts = {"zsh": ["--no-rcs"], "bash": ["--noprofile", "--norc"]}
|
||||
name = os.path.basename(executable).lower()
|
||||
cmd = [executable] + opts.get(name, []) + ["-c", self.cmd]
|
||||
|
||||
main_thread = isinstance(
|
||||
threading.current_thread(), threading._MainThread
|
||||
)
|
||||
old_handler = None
|
||||
p = None
|
||||
|
||||
try:
|
||||
p = subprocess.Popen(cmd, **kwargs)
|
||||
if main_thread:
|
||||
old_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||
p.communicate()
|
||||
finally:
|
||||
if old_handler:
|
||||
signal.signal(signal.SIGINT, old_handler)
|
||||
|
||||
retcode = None if not p else p.returncode
|
||||
if retcode != 0:
|
||||
raise StageCmdFailedError(self, retcode)
|
||||
|
||||
@rwlocked(read=["deps"], write=["outs"])
|
||||
def run(
|
||||
self, dry=False, no_commit=False, force=False, ignore_build_cache=False
|
||||
):
|
||||
def run(self, dry=False, no_commit=False, force=False, run_cache=True):
|
||||
if (self.cmd or self.is_import) and not self.locked and not dry:
|
||||
self.remove_outs(ignore_remove=False, force=False)
|
||||
|
||||
if self.locked:
|
||||
logger.info(
|
||||
"Verifying outputs in locked {stage}".format(stage=self)
|
||||
)
|
||||
if not dry:
|
||||
self.check_missing_outputs()
|
||||
|
||||
elif self.is_import:
|
||||
logger.info(
|
||||
"Importing '{dep}' -> '{out}'".format(
|
||||
dep=self.deps[0], out=self.outs[0]
|
||||
)
|
||||
)
|
||||
if not dry:
|
||||
if (
|
||||
not force
|
||||
and not self.stage_changed(warn=True)
|
||||
and self._already_cached()
|
||||
):
|
||||
self.outs[0].checkout()
|
||||
else:
|
||||
self.deps[0].download(self.outs[0])
|
||||
elif self.is_data_source:
|
||||
msg = "Verifying data sources in {}".format(self)
|
||||
logger.info(msg)
|
||||
if not dry:
|
||||
self.check_missing_outputs()
|
||||
|
||||
if not self.locked and self.is_import:
|
||||
sync_import(self, dry, force)
|
||||
elif not self.locked and self.cmd:
|
||||
run_stage(self, dry, force, run_cache)
|
||||
else:
|
||||
args = (
|
||||
("outputs", "locked ") if self.locked else ("data sources", "")
|
||||
)
|
||||
logger.info("Verifying %s in %s%s", *args, self)
|
||||
if not dry:
|
||||
stage_cache = self.repo.stage_cache
|
||||
stage_cached = (
|
||||
not force
|
||||
and not self.is_callback
|
||||
and not self.always_changed
|
||||
and self._already_cached()
|
||||
)
|
||||
use_build_cache = False
|
||||
if not stage_cached:
|
||||
self._save_deps()
|
||||
use_build_cache = (
|
||||
not force
|
||||
and not ignore_build_cache
|
||||
and stage_cache.is_cached(self)
|
||||
)
|
||||
|
||||
if use_build_cache:
|
||||
# restore stage from build cache
|
||||
self.repo.stage_cache.restore(self)
|
||||
stage_cached = self._outs_cached()
|
||||
|
||||
if stage_cached:
|
||||
logger.info("Stage is cached, skipping.")
|
||||
self.checkout()
|
||||
else:
|
||||
logger.info("Running command:\n\t{}".format(self.cmd))
|
||||
self._run()
|
||||
check_missing_outputs(self)
|
||||
|
||||
if not dry:
|
||||
self.save()
|
||||
if not no_commit:
|
||||
self.commit()
|
||||
|
||||
def check_missing_outputs(self):
|
||||
paths = [str(out) for out in self.outs if not out.exists]
|
||||
if paths:
|
||||
raise MissingDataSource(paths)
|
||||
|
||||
def _filter_outs(self, path_info):
|
||||
def _func(o):
|
||||
return path_info.isin_or_eq(o.path_info)
|
||||
@@ -701,31 +428,36 @@ class Stage(params.StageParams):
|
||||
return filter(_func, self.outs) if path_info else self.outs
|
||||
|
||||
@rwlocked(write=["outs"])
|
||||
def checkout(
|
||||
self,
|
||||
force=False,
|
||||
progress_callback=None,
|
||||
relink=False,
|
||||
filter_info=None,
|
||||
):
|
||||
checkouts = {"failed": [], "added": [], "modified": []}
|
||||
for out in self._filter_outs(filter_info):
|
||||
try:
|
||||
result = out.checkout(
|
||||
force=force,
|
||||
progress_callback=progress_callback,
|
||||
relink=relink,
|
||||
filter_info=filter_info,
|
||||
)
|
||||
added, modified = result or (None, None)
|
||||
if modified:
|
||||
checkouts["modified"].append(out.path_info)
|
||||
elif added:
|
||||
checkouts["added"].append(out.path_info)
|
||||
except CheckoutError as exc:
|
||||
checkouts["failed"].extend(exc.target_infos)
|
||||
def checkout(self, **kwargs):
|
||||
stats = defaultdict(list)
|
||||
for out in self._filter_outs(kwargs.get("filter_info")):
|
||||
key, outs = self._checkout(out, **kwargs)
|
||||
if key:
|
||||
stats[key].extend(outs)
|
||||
return stats
|
||||
|
||||
return checkouts
|
||||
@staticmethod
|
||||
def _checkout(out, **kwargs):
|
||||
try:
|
||||
result = out.checkout(**kwargs)
|
||||
added, modified = result or (None, None)
|
||||
if not (added or modified):
|
||||
return None, []
|
||||
return "modified" if modified else "added", [out.path_info]
|
||||
except CheckoutError as exc:
|
||||
return "failed", exc.target_infos
|
||||
|
||||
@rwlocked(read=["deps", "outs"])
|
||||
def status(self, check_updates=False):
|
||||
ret = []
|
||||
show_import = self.is_repo_import and check_updates
|
||||
|
||||
if not self.locked or show_import:
|
||||
self._status_deps(ret)
|
||||
self._status_outs(ret)
|
||||
self._status_always_changed(ret)
|
||||
self._status_stage(ret)
|
||||
return {self.addressing: ret} if ret else {}
|
||||
|
||||
@staticmethod
|
||||
def _status(entries):
|
||||
@@ -736,40 +468,31 @@ class Stage(params.StageParams):
|
||||
|
||||
return ret
|
||||
|
||||
def stage_status(self):
|
||||
return ["changed checksum"] if self.stage_changed() else []
|
||||
|
||||
@rwlocked(read=["deps", "outs"])
|
||||
def status(self, check_updates=False):
|
||||
ret = []
|
||||
|
||||
show_import = self.is_repo_import and check_updates
|
||||
|
||||
if not self.locked or show_import:
|
||||
deps_status = self._status(self.deps)
|
||||
if deps_status:
|
||||
ret.append({"changed deps": deps_status})
|
||||
def _status_deps(self, ret):
|
||||
deps_status = self._status(self.deps)
|
||||
if deps_status:
|
||||
ret.append({"changed deps": deps_status})
|
||||
|
||||
def _status_outs(self, ret):
|
||||
outs_status = self._status(self.outs)
|
||||
if outs_status:
|
||||
ret.append({"changed outs": outs_status})
|
||||
|
||||
ret.extend(self.stage_status())
|
||||
def _status_always_changed(self, ret):
|
||||
if self.is_callback or self.always_changed:
|
||||
ret.append("always changed")
|
||||
|
||||
if ret:
|
||||
return {self.addressing: ret}
|
||||
def _status_stage(self, ret):
|
||||
if self.changed_stage():
|
||||
ret.append("changed checksum")
|
||||
|
||||
return {}
|
||||
def already_cached(self):
|
||||
return self.deps_cached() and self.outs_cached()
|
||||
|
||||
def _already_cached(self):
|
||||
return self._deps_cached() and self._outs_cached()
|
||||
|
||||
def _deps_cached(self):
|
||||
def deps_cached(self):
|
||||
return all(not dep.changed() for dep in self.deps)
|
||||
|
||||
def _outs_cached(self):
|
||||
def outs_cached(self):
|
||||
return all(
|
||||
not out.changed_cache() if out.use_cache else not out.changed()
|
||||
for out in self.outs
|
||||
@@ -792,12 +515,10 @@ class Stage(params.StageParams):
|
||||
|
||||
|
||||
class PipelineStage(Stage):
|
||||
def __init__(self, *args, name=None, meta=None, **kwargs):
|
||||
def __init__(self, *args, name=None, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.name = name
|
||||
self.cmd_changed = False
|
||||
# This is how the Stage will discover any discrepancies
|
||||
self.meta = meta or {}
|
||||
|
||||
def __eq__(self, other):
|
||||
return super().__eq__(other) and self.name == other.name
|
||||
@@ -805,19 +526,13 @@ class PipelineStage(Stage):
|
||||
def __hash__(self):
|
||||
return hash((self.path_in_repo, self.name))
|
||||
|
||||
def __repr__(self):
|
||||
return "Stage: '{path}:{name}'".format(
|
||||
path=self.relpath if self.path else "No path", name=self.name
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
return "stage: '{path}:{name}'".format(
|
||||
path=self.relpath if self.path else "No path", name=self.name
|
||||
)
|
||||
|
||||
@property
|
||||
def addressing(self):
|
||||
return super().addressing + ":" + self.name
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
|
||||
if self.path and self.relpath == PIPELINE_FILE:
|
||||
return self.name
|
||||
return f"{super().addressing}:{self.name}"
|
||||
|
||||
def reload(self):
|
||||
return self.dvcfile.stages[self.name]
|
||||
@@ -826,10 +541,14 @@ class PipelineStage(Stage):
|
||||
def is_cached(self):
|
||||
return self.name in self.dvcfile.stages and super().is_cached
|
||||
|
||||
def stage_status(self):
|
||||
return ["changed command"] if self.cmd_changed else []
|
||||
def _status_stage(self, ret):
|
||||
if self.cmd_changed:
|
||||
ret.append("changed command")
|
||||
|
||||
def stage_changed(self, warn=False):
|
||||
if self.cmd_changed and warn:
|
||||
logger.warning("'cmd' of {} has changed.".format(self))
|
||||
def changed_stage(self):
|
||||
if self.cmd_changed:
|
||||
logger.debug(self._changed_stage_entry())
|
||||
return self.cmd_changed
|
||||
|
||||
def _changed_stage_entry(self):
|
||||
return f"'cmd' of {self} has changed."
|
||||
|
||||
+87
-3
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
import yaml
|
||||
from funcy import first
|
||||
@@ -51,7 +52,7 @@ class StageCache:
|
||||
path = self._get_cache_path(key, value)
|
||||
|
||||
try:
|
||||
with open(path, "r") as fobj:
|
||||
with open(path) as fobj:
|
||||
return COMPILED_LOCK_FILE_STAGE_SCHEMA(yaml.safe_load(fobj))
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
@@ -76,6 +77,60 @@ class StageCache:
|
||||
|
||||
return None
|
||||
|
||||
def _create_stage(self, cache, wdir=None):
|
||||
from dvc.stage import create_stage, PipelineStage
|
||||
|
||||
params = []
|
||||
for param in cache.get("params", []):
|
||||
if isinstance(param, str):
|
||||
params.append(param)
|
||||
continue
|
||||
|
||||
assert isinstance(param, dict)
|
||||
assert len(param) == 1
|
||||
path = list(param.keys())[0]
|
||||
params_list = param[path]
|
||||
assert isinstance(params_list, list)
|
||||
params.append(f"{path}:" + ",".join(params_list))
|
||||
|
||||
stage = create_stage(
|
||||
PipelineStage,
|
||||
repo=self.repo,
|
||||
path="dvc.yaml",
|
||||
cmd=cache["cmd"],
|
||||
wdir=wdir,
|
||||
params=params,
|
||||
deps=[dep["path"] for dep in cache.get("deps", [])],
|
||||
outs=[out["path"] for out in cache["outs"]],
|
||||
)
|
||||
StageLoader.fill_from_lock(stage, cache)
|
||||
return stage
|
||||
|
||||
@contextmanager
|
||||
def _cache_type_copy(self):
|
||||
cache_types = self.repo.cache.local.cache_types
|
||||
self.repo.cache.local.cache_types = ["copy"]
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.repo.cache.local.cache_types = cache_types
|
||||
|
||||
def _uncached_outs(self, stage, cache):
|
||||
# NOTE: using temporary stage to avoid accidentally modifying original
|
||||
# stage and to workaround `commit/checkout` not working for uncached
|
||||
# outputs.
|
||||
cached_stage = self._create_stage(cache, wdir=stage.wdir)
|
||||
|
||||
outs_no_cache = [
|
||||
out.def_path for out in stage.outs if not out.use_cache
|
||||
]
|
||||
|
||||
# NOTE: using copy link to make it look like a git-tracked file
|
||||
with self._cache_type_copy():
|
||||
for out in cached_stage.outs:
|
||||
if out.def_path in outs_no_cache:
|
||||
yield out
|
||||
|
||||
def save(self, stage):
|
||||
cache_key = _get_stage_hash(stage)
|
||||
if not cache_key:
|
||||
@@ -84,7 +139,13 @@ class StageCache:
|
||||
cache = to_single_stage_lockfile(stage)
|
||||
cache_value = _get_cache_hash(cache)
|
||||
|
||||
if self._load_cache(cache_key, cache_value):
|
||||
existing_cache = self._load_cache(cache_key, cache_value)
|
||||
cache = existing_cache or cache
|
||||
|
||||
for out in self._uncached_outs(stage, cache):
|
||||
out.commit()
|
||||
|
||||
if existing_cache:
|
||||
return
|
||||
|
||||
# sanity check
|
||||
@@ -104,11 +165,16 @@ class StageCache:
|
||||
return
|
||||
StageLoader.fill_from_lock(stage, cache)
|
||||
|
||||
for out in self._uncached_outs(stage, cache):
|
||||
out.checkout()
|
||||
|
||||
@staticmethod
|
||||
def _transfer(func, from_remote, to_remote):
|
||||
ret = []
|
||||
|
||||
runs = from_remote.path_info / "runs"
|
||||
if not from_remote.exists(runs):
|
||||
return
|
||||
return []
|
||||
|
||||
for src in from_remote.walk_files(runs):
|
||||
rel = src.relative_to(from_remote.path_info)
|
||||
@@ -118,9 +184,27 @@ class StageCache:
|
||||
if to_remote.exists(key) and first(to_remote.walk_files(key)):
|
||||
continue
|
||||
func(src, dst)
|
||||
ret.append((src.parent.name, src.name))
|
||||
|
||||
return ret
|
||||
|
||||
def push(self, remote):
|
||||
remote = self.repo.cloud.get_remote(remote)
|
||||
return self._transfer(remote.upload, self.repo.cache.local, remote)
|
||||
|
||||
def pull(self, remote):
|
||||
remote = self.repo.cloud.get_remote(remote)
|
||||
return self._transfer(remote.download, remote, self.repo.cache.local)
|
||||
|
||||
def get_used_cache(self, used_run_cache, *args, **kwargs):
|
||||
from dvc.cache import NamedCache
|
||||
|
||||
cache = NamedCache()
|
||||
|
||||
for key, value in used_run_cache:
|
||||
entry = self._load_cache(key, value)
|
||||
if not entry:
|
||||
continue
|
||||
stage = self._create_stage(entry)
|
||||
cache.update(stage.get_used_cache(*args, **kwargs))
|
||||
return cache
|
||||
|
||||
+15
-25
@@ -2,35 +2,26 @@ from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class StageCmdFailedError(DvcException):
|
||||
def __init__(self, stage, status=None):
|
||||
msg = "failed to run: {}".format(stage.cmd)
|
||||
def __init__(self, cmd, status=None):
|
||||
msg = f"failed to run: {cmd}"
|
||||
if status is not None:
|
||||
msg += ", exited with {}".format(status)
|
||||
msg += f", exited with {status}"
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class StageFileFormatError(DvcException):
|
||||
def __init__(self, fname, e):
|
||||
msg = "DVC-file '{}' format error: {}".format(fname, str(e))
|
||||
super().__init__(msg)
|
||||
pass
|
||||
|
||||
|
||||
class StageFileDoesNotExistError(DvcException):
|
||||
def __init__(self, fname):
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX, is_dvc_file
|
||||
|
||||
msg = "'{}' does not exist.".format(fname)
|
||||
|
||||
sname = fname + DVC_FILE_SUFFIX
|
||||
if is_dvc_file(sname):
|
||||
msg += " Do you mean '{}'?".format(sname)
|
||||
|
||||
super().__init__(msg)
|
||||
self.file = fname
|
||||
super().__init__(f"'{self.file}' does not exist.")
|
||||
|
||||
|
||||
class StageFileAlreadyExistsError(DvcException):
|
||||
def __init__(self, relpath):
|
||||
msg = "not overwriting '{}'".format(relpath)
|
||||
msg = f"not overwriting '{relpath}'"
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
@@ -38,11 +29,11 @@ class StageFileIsNotDvcFileError(DvcException):
|
||||
def __init__(self, fname):
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX, is_dvc_file
|
||||
|
||||
msg = "'{}' is not a DVC-file".format(fname)
|
||||
msg = f"'{fname}' is not a DVC-file"
|
||||
|
||||
sname = fname + DVC_FILE_SUFFIX
|
||||
if is_dvc_file(sname):
|
||||
msg += " Do you mean '{}'?".format(sname)
|
||||
msg += f" Do you mean '{sname}'?"
|
||||
|
||||
super().__init__(msg)
|
||||
|
||||
@@ -89,8 +80,10 @@ class MissingDataSource(DvcException):
|
||||
|
||||
class StageNotFound(KeyError, DvcException):
|
||||
def __init__(self, file, name):
|
||||
self.file = file.relpath
|
||||
self.name = name
|
||||
super().__init__(
|
||||
"Stage '{}' not found inside '{}' file".format(name, file.relpath)
|
||||
f"Stage '{self.name}' not found inside '{self.file}' file"
|
||||
)
|
||||
|
||||
|
||||
@@ -98,7 +91,7 @@ class StageNameUnspecified(DvcException):
|
||||
def __init__(self, file):
|
||||
super().__init__(
|
||||
"Stage name not provided."
|
||||
"Please specify the name as: `{0}:stage_name`".format(file.relpath)
|
||||
"Please specify the name as: `{}:stage_name`".format(file.relpath)
|
||||
)
|
||||
|
||||
|
||||
@@ -112,8 +105,5 @@ class DuplicateStageName(DvcException):
|
||||
|
||||
|
||||
class InvalidStageName(DvcException):
|
||||
def __init__(self,):
|
||||
super().__init__(
|
||||
"Stage name cannot contain invalid characters: "
|
||||
"'\\', '/', '@' and ':'."
|
||||
)
|
||||
def __init__(self):
|
||||
super().__init__("Stage name cannot contain punctuation characters.")
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user