mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
63972c3d12 | ||
|
|
7027c631c5 | ||
|
|
e8f6cd7f5b | ||
|
|
10f2bb93fa | ||
|
|
2fecc66cb9 | ||
|
|
b47998463f | ||
|
|
d53b740f35 | ||
|
|
9198fa7459 | ||
|
|
6954efc6e6 | ||
|
|
d77af8196e | ||
|
|
c5920b00ab | ||
|
|
ce8036e535 | ||
|
|
b7ad2dc841 | ||
|
|
5ab948f8b6 | ||
|
|
f9fbe97509 | ||
|
|
2fba2e030a | ||
|
|
3e951162b8 | ||
|
|
d74358eecd | ||
|
|
58c9ff0de2 | ||
|
|
f0e991790e | ||
|
|
b7cd785eba | ||
|
|
21a3efe2c7 | ||
|
|
865d355554 | ||
|
|
84712c4acc | ||
|
|
0f16e15453 | ||
|
|
61f30ed5ba | ||
|
|
cde15e3d3e | ||
|
|
7672e324d2 | ||
|
|
130b76c7a5 | ||
|
|
040458a726 | ||
|
|
75e795c491 | ||
|
|
4dac2092c9 | ||
|
|
19eaced304 | ||
|
|
184028f176 | ||
|
|
5f63d538d2 | ||
|
|
324ff4be59 | ||
|
|
3f846a1794 | ||
|
|
b52c925d5f |
+2
-1
@@ -101,7 +101,8 @@ Snap (Snapcraft/Linux)
|
||||
snap install dvc --classic
|
||||
|
||||
This corresponds to the latest tagged release.
|
||||
Add ``--edge`` for the latest ``master`` version.
|
||||
Add ``--beta`` for the latest tagged release candidate,
|
||||
or ``--edge`` for the latest ``master`` version.
|
||||
|
||||
Choco (Chocolatey/Windows)
|
||||
--------------------------
|
||||
|
||||
+2
-6
@@ -116,14 +116,10 @@ def get_parent_parser():
|
||||
|
||||
log_level_group = parent_parser.add_mutually_exclusive_group()
|
||||
log_level_group.add_argument(
|
||||
"-q", "--quiet", action="store_true", default=False, help="Be quiet."
|
||||
"-q", "--quiet", action="count", default=0, help="Be quiet."
|
||||
)
|
||||
log_level_group.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Be verbose.",
|
||||
"-v", "--verbose", action="count", default=0, help="Be verbose."
|
||||
)
|
||||
|
||||
return parent_parser
|
||||
|
||||
@@ -31,7 +31,7 @@ class CmdList(CmdBaseNoRepo):
|
||||
self.args.path,
|
||||
rev=self.args.rev,
|
||||
recursive=self.args.recursive,
|
||||
outs_only=self.args.outs_only,
|
||||
dvc_only=self.args.dvc_only,
|
||||
)
|
||||
if entries:
|
||||
entries = _prettify(entries, sys.stdout.isatty())
|
||||
@@ -62,7 +62,7 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Recursively list files.",
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"--outs-only", action="store_true", help="Show only DVC outputs."
|
||||
"--dvc-only", action="store_true", help="Show only DVC outputs."
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"--rev",
|
||||
|
||||
+32
-11
@@ -87,7 +87,7 @@ class CmdMetricsRemove(CmdBase):
|
||||
return 0
|
||||
|
||||
|
||||
def _show_diff(diff, markdown=False):
|
||||
def _show_diff(diff, markdown=False, no_path=False, old=False):
|
||||
from collections import OrderedDict
|
||||
|
||||
from dvc.utils.diff import table
|
||||
@@ -96,16 +96,23 @@ def _show_diff(diff, markdown=False):
|
||||
for fname, mdiff in diff.items():
|
||||
sorted_mdiff = OrderedDict(sorted(mdiff.items()))
|
||||
for metric, change in sorted_mdiff.items():
|
||||
rows.append(
|
||||
[
|
||||
fname,
|
||||
metric,
|
||||
change["new"],
|
||||
change.get("diff", "diff not supported"),
|
||||
]
|
||||
)
|
||||
row = [] if no_path else [fname]
|
||||
row.append(metric)
|
||||
if old:
|
||||
row.append(change.get("old"))
|
||||
row.append(change["new"])
|
||||
row.append(change.get("diff", "diff not supported"))
|
||||
rows.append(row)
|
||||
|
||||
return table(["Path", "Metric", "Value", "Change"], rows, markdown)
|
||||
header = [] if no_path else ["Path"]
|
||||
header.append("Metric")
|
||||
if old:
|
||||
header.extend(["Old", "New"])
|
||||
else:
|
||||
header.append("Value")
|
||||
header.append("Change")
|
||||
|
||||
return table(header, rows, markdown)
|
||||
|
||||
|
||||
class CmdMetricsDiff(CmdBase):
|
||||
@@ -124,7 +131,9 @@ class CmdMetricsDiff(CmdBase):
|
||||
|
||||
logger.info(json.dumps(diff))
|
||||
else:
|
||||
table = _show_diff(diff, self.args.show_md)
|
||||
table = _show_diff(
|
||||
diff, self.args.show_md, self.args.no_path, self.args.old
|
||||
)
|
||||
if table:
|
||||
logger.info(table)
|
||||
|
||||
@@ -269,6 +278,18 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Show tabulated output in the Markdown format (GFM).",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--no-path",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Don't show metric path.",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--old",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show old metric value.",
|
||||
)
|
||||
metrics_diff_parser.set_defaults(func=CmdMetricsDiff)
|
||||
|
||||
METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
|
||||
|
||||
+19
-4
@@ -8,16 +8,25 @@ from dvc.exceptions import DvcException
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _show_diff(diff, markdown=False):
|
||||
def _show_diff(diff, markdown=False, no_path=False):
|
||||
from dvc.utils.diff import table
|
||||
|
||||
rows = []
|
||||
for fname, pdiff in diff.items():
|
||||
sorted_pdiff = OrderedDict(sorted(pdiff.items()))
|
||||
for param, change in sorted_pdiff.items():
|
||||
rows.append([fname, param, change["old"], change["new"]])
|
||||
row = [] if no_path else [fname]
|
||||
row.append(param)
|
||||
row.append(change["old"])
|
||||
row.append(change["new"])
|
||||
rows.append(row)
|
||||
|
||||
return table(["Path", "Param", "Old", "New"], rows, markdown)
|
||||
header = [] if no_path else ["Path"]
|
||||
header.append("Param")
|
||||
header.append("Old")
|
||||
header.append("New")
|
||||
|
||||
return table(header, rows, markdown)
|
||||
|
||||
|
||||
class CmdParamsDiff(CmdBase):
|
||||
@@ -34,7 +43,7 @@ class CmdParamsDiff(CmdBase):
|
||||
|
||||
logger.info(json.dumps(diff))
|
||||
else:
|
||||
table = _show_diff(diff, self.args.show_md)
|
||||
table = _show_diff(diff, self.args.show_md, self.args.no_path)
|
||||
if table:
|
||||
logger.info(table)
|
||||
|
||||
@@ -100,4 +109,10 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Show tabulated output in the Markdown format (GFM).",
|
||||
)
|
||||
params_diff_parser.add_argument(
|
||||
"--no-path",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Don't show params path.",
|
||||
)
|
||||
params_diff_parser.set_defaults(func=CmdParamsDiff)
|
||||
|
||||
@@ -10,11 +10,10 @@ logger = logging.getLogger(__name__)
|
||||
class CmdPipelineShow(CmdBase):
|
||||
def _show(self, target, commands, outs, locked):
|
||||
import networkx
|
||||
from dvc import dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
stage = dvcfile.Dvcfile(self.repo, path).stages[name]
|
||||
stage = self.repo.get_stage(path, name)
|
||||
G = self.repo.graph
|
||||
stages = networkx.dfs_postorder_nodes(G, stage)
|
||||
if locked:
|
||||
@@ -58,12 +57,11 @@ class CmdPipelineShow(CmdBase):
|
||||
|
||||
def _build_graph(self, target, commands=False, outs=False):
|
||||
import networkx
|
||||
from dvc import dvcfile
|
||||
from dvc.repo.graph import get_pipeline
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
|
||||
target_stage = self.repo.get_stage(path, name)
|
||||
G = get_pipeline(self.repo.pipelines, target_stage)
|
||||
|
||||
nodes = set()
|
||||
|
||||
+13
-23
@@ -4,6 +4,7 @@ import os
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils import format_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -32,21 +33,12 @@ class CmdPlots(CmdBase):
|
||||
raise NotImplementedError
|
||||
|
||||
def run(self):
|
||||
fields = None
|
||||
jsonpath = None
|
||||
if self.args.select:
|
||||
if self.args.select.startswith("$"):
|
||||
jsonpath = self.args.select
|
||||
else:
|
||||
fields = set(self.args.select.split(","))
|
||||
try:
|
||||
plots = self._func(
|
||||
targets=self.args.targets,
|
||||
template=self.args.template,
|
||||
fields=fields,
|
||||
x_field=self.args.x,
|
||||
y_field=self.args.y,
|
||||
path=jsonpath,
|
||||
csv_header=not self.args.no_csv_header,
|
||||
title=self.args.title,
|
||||
x_title=self.args.xlab,
|
||||
@@ -123,17 +115,16 @@ def add_parser(subparsers, parent_parser):
|
||||
"--template",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help="File to be injected with data.",
|
||||
help=(
|
||||
"Special JSON or HTML schema file to inject with the data. "
|
||||
"See {}".format(
|
||||
format_link("https://man.dvc.org/plots#plot-templates")
|
||||
)
|
||||
),
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-o", "--out", default=None, help="Destination path to save plots to.",
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-s",
|
||||
"--select",
|
||||
default=None,
|
||||
help="Choose which field(s) or JSONPath to include in the plots.",
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-x", default=None, help="Field name for x axis."
|
||||
)
|
||||
@@ -182,7 +173,12 @@ def add_parser(subparsers, parent_parser):
|
||||
"--template",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help="File to be injected with data.",
|
||||
help=(
|
||||
"Special JSON or HTML schema file to inject with the data. "
|
||||
"See {}".format(
|
||||
format_link("https://man.dvc.org/plots#plot-templates")
|
||||
)
|
||||
),
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--targets",
|
||||
@@ -192,12 +188,6 @@ def add_parser(subparsers, parent_parser):
|
||||
plots_diff_parser.add_argument(
|
||||
"-o", "--out", default=None, help="Destination path to save plots to.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-s",
|
||||
"--select",
|
||||
default=None,
|
||||
help="Choose which field(s) or JSONPath to include in the plot.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-x", default=None, help="Field name for x axis."
|
||||
)
|
||||
|
||||
@@ -104,6 +104,35 @@ class CmdRemoteList(CmdRemote):
|
||||
return 0
|
||||
|
||||
|
||||
class CmdRemoteRename(CmdRemote):
|
||||
def _rename_default(self, conf):
|
||||
if conf["core"].get("remote") == self.args.name:
|
||||
conf["core"]["remote"] = self.args.new
|
||||
|
||||
def run(self):
|
||||
all_config = self.config.load_config_to_level(None)
|
||||
if self.args.new in all_config.get("remote", {}):
|
||||
raise ConfigError(
|
||||
"Rename failed. Remote name '{}' already exists.".format(
|
||||
{self.args.new}
|
||||
)
|
||||
)
|
||||
|
||||
with self.config.edit(self.args.level) as conf:
|
||||
self._check_exists(conf)
|
||||
conf["remote"][self.args.new] = conf["remote"][self.args.name]
|
||||
del conf["remote"][self.args.name]
|
||||
self._rename_default(conf)
|
||||
|
||||
for level in reversed(self.config.LEVELS):
|
||||
if level == self.args.level:
|
||||
break
|
||||
with self.config.edit(level) as level_conf:
|
||||
self._rename_default(level_conf)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
from dvc.command.config import parent_config_parser
|
||||
|
||||
@@ -220,3 +249,14 @@ def add_parser(subparsers, parent_parser):
|
||||
"name", help="Name of the remote to remove."
|
||||
)
|
||||
remote_remove_parser.set_defaults(func=CmdRemoteRemove)
|
||||
REMOTE_RENAME_HELP = "Rename a DVC remote"
|
||||
remote_rename_parser = remote_subparsers.add_parser(
|
||||
"rename",
|
||||
parents=[parent_config_parser, parent_parser],
|
||||
description=append_doc_link(REMOTE_RENAME_HELP, "remote/rename"),
|
||||
help=REMOTE_RENAME_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
remote_rename_parser.add_argument("name", help="Remote to be renamed")
|
||||
remote_rename_parser.add_argument("new", help="New name of the remote")
|
||||
remote_rename_parser.set_defaults(func=CmdRemoteRename)
|
||||
|
||||
@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdRemove(CmdBase):
|
||||
def _is_outs_only(self, target):
|
||||
def _is_dvc_only(self, target):
|
||||
if not self.args.purge:
|
||||
return True
|
||||
|
||||
@@ -31,8 +31,8 @@ class CmdRemove(CmdBase):
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
outs_only = self._is_outs_only(target)
|
||||
self.repo.remove(target, outs_only=outs_only)
|
||||
dvc_only = self._is_dvc_only(target)
|
||||
self.repo.remove(target, dvc_only=dvc_only)
|
||||
except DvcException:
|
||||
logger.exception(f"failed to remove '{target}'")
|
||||
return 1
|
||||
|
||||
@@ -5,6 +5,7 @@ import os
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.command.metrics import show_metrics
|
||||
from dvc.command.status import CmdDataStatus
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -69,7 +70,7 @@ def add_parser(subparsers, parent_parser):
|
||||
repro_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="DVC-file to reproduce. 'Dvcfile' by default.",
|
||||
help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"-f",
|
||||
|
||||
@@ -2,6 +2,7 @@ from collections import defaultdict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import dvc.output as output
|
||||
from dvc.dependency.azure import AzureDependency
|
||||
from dvc.dependency.gs import GSDependency
|
||||
from dvc.dependency.hdfs import HDFSDependency
|
||||
from dvc.dependency.http import HTTPDependency
|
||||
@@ -17,6 +18,7 @@ from dvc.scheme import Schemes
|
||||
from .repo import RepoDependency
|
||||
|
||||
DEPS = [
|
||||
AzureDependency,
|
||||
GSDependency,
|
||||
HDFSDependency,
|
||||
HTTPDependency,
|
||||
@@ -30,6 +32,7 @@ DEP_MAP = {
|
||||
Schemes.LOCAL: LocalDependency,
|
||||
Schemes.SSH: SSHDependency,
|
||||
Schemes.S3: S3Dependency,
|
||||
Schemes.AZURE: AzureDependency,
|
||||
Schemes.GS: GSDependency,
|
||||
Schemes.HDFS: HDFSDependency,
|
||||
Schemes.HTTP: HTTPDependency,
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.azure import AzureRemote
|
||||
|
||||
|
||||
class AzureDependency(BaseDependency, BaseOutput):
|
||||
REMOTE = AzureRemote
|
||||
+18
-7
@@ -50,13 +50,24 @@ class RepoDependency(LocalDependency):
|
||||
return external_repo(d["url"], rev=rev)
|
||||
|
||||
def _get_checksum(self, locked=True):
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
with self._make_repo(locked=locked) as repo:
|
||||
try:
|
||||
return repo.find_out_by_relpath(self.def_path).info["md5"]
|
||||
except OutputNotFoundError:
|
||||
path = PathInfo(os.path.join(repo.root_dir, self.def_path))
|
||||
|
||||
# we want stream but not fetch, so DVC out directories are
|
||||
# walked, but dir contents is not fetched
|
||||
tree = RepoTree(repo, stream=True)
|
||||
|
||||
# We are polluting our repo cache with some dir listing here
|
||||
return self.repo.cache.local.get_checksum(path)
|
||||
if tree.isdir(path):
|
||||
return self.repo.cache.local.get_dir_checksum(
|
||||
path, tree=tree
|
||||
)
|
||||
return tree.get_file_checksum(path)
|
||||
|
||||
def status(self):
|
||||
current_checksum = self._get_checksum(locked=True)
|
||||
@@ -76,16 +87,16 @@ class RepoDependency(LocalDependency):
|
||||
def download(self, to):
|
||||
with self._make_repo() as repo:
|
||||
if self.def_repo.get(self.PARAM_REV_LOCK) is None:
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
|
||||
|
||||
if hasattr(repo, "cache"):
|
||||
repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
|
||||
|
||||
repo.pull_to(self.def_path, to.path_info)
|
||||
cache = self.repo.cache.local
|
||||
with repo.use_cache(cache):
|
||||
_, _, cache_infos = repo.fetch_external([self.def_path])
|
||||
cache.checkout(to.path_info, cache_infos[0])
|
||||
|
||||
def update(self, rev=None):
|
||||
if rev:
|
||||
self.def_repo[self.PARAM_REV] = rev
|
||||
|
||||
with self._make_repo(locked=False) as repo:
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
|
||||
|
||||
+18
-4
@@ -175,16 +175,19 @@ class PipelineFile(FileMixin):
|
||||
def _lockfile(self):
|
||||
return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
|
||||
|
||||
def dump(self, stage, update_pipeline=False, **kwargs):
|
||||
def dump(self, stage, update_pipeline=False, no_lock=False):
|
||||
"""Dumps given stage appropriately in the dvcfile."""
|
||||
from dvc.stage import PipelineStage
|
||||
|
||||
assert isinstance(stage, PipelineStage)
|
||||
check_dvc_filename(self.path)
|
||||
self._dump_lockfile(stage)
|
||||
|
||||
if update_pipeline and not stage.is_data_source:
|
||||
self._dump_pipeline_file(stage)
|
||||
|
||||
if not no_lock:
|
||||
self._dump_lockfile(stage)
|
||||
|
||||
def _dump_lockfile(self, stage):
|
||||
self._lockfile.dump(stage)
|
||||
|
||||
@@ -194,6 +197,7 @@ class PipelineFile(FileMixin):
|
||||
with open(self.path) as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
else:
|
||||
logger.info("Creating '%s'", self.relpath)
|
||||
open(self.path, "w+").close()
|
||||
|
||||
data["stages"] = data.get("stages", {})
|
||||
@@ -204,6 +208,9 @@ class PipelineFile(FileMixin):
|
||||
else:
|
||||
data["stages"].update(stage_data)
|
||||
|
||||
logger.info(
|
||||
"Adding stage '%s' to '%s'", stage.name, self.relpath,
|
||||
)
|
||||
dump_stage_file(self.path, data)
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
|
||||
@@ -247,15 +254,22 @@ class Lockfile(FileMixin):
|
||||
def dump(self, stage, **kwargs):
|
||||
stage_data = serialize.to_lockfile(stage)
|
||||
if not self.exists():
|
||||
modified = True
|
||||
logger.info("Generating lock file '%s'", self.relpath)
|
||||
data = stage_data
|
||||
open(self.path, "w+").close()
|
||||
else:
|
||||
with self.repo.tree.open(self.path, "r") as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
modified = data.get(stage.name, {}) != stage_data.get(
|
||||
stage.name, {}
|
||||
)
|
||||
if modified:
|
||||
logger.info("Updating lock file '%s'", self.relpath)
|
||||
data.update(stage_data)
|
||||
|
||||
dump_stage_file(self.path, data)
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
if modified:
|
||||
self.repo.scm.track_file(self.relpath)
|
||||
|
||||
|
||||
class Dvcfile:
|
||||
|
||||
+16
-8
@@ -294,19 +294,15 @@ class PathMissingError(DvcException):
|
||||
"The path '{}' does not exist in the target repository '{}'"
|
||||
" neither as a DVC output nor as a Git-tracked file."
|
||||
)
|
||||
default_msg_output_only = (
|
||||
default_msg_dvc_only = (
|
||||
"The path '{}' does not exist in the target repository '{}'"
|
||||
" as an DVC output."
|
||||
)
|
||||
|
||||
def __init__(self, path, repo, output_only=False):
|
||||
msg = (
|
||||
self.default_msg
|
||||
if not output_only
|
||||
else self.default_msg_output_only
|
||||
)
|
||||
def __init__(self, path, repo, dvc_only=False):
|
||||
msg = self.default_msg if not dvc_only else self.default_msg_dvc_only
|
||||
super().__init__(msg.format(path, repo))
|
||||
self.output_only = output_only
|
||||
self.dvc_only = dvc_only
|
||||
|
||||
|
||||
class RemoteCacheRequiredError(DvcException):
|
||||
@@ -326,3 +322,15 @@ class RemoteCacheRequiredError(DvcException):
|
||||
|
||||
class IsADirectoryError(DvcException):
|
||||
"""Raised when a file operation is requested on a directory."""
|
||||
|
||||
|
||||
class NoOutputOrStageError(DvcException):
|
||||
"""
|
||||
Raised when the target is neither an output nor a stage name in dvc.yaml
|
||||
"""
|
||||
|
||||
def __init__(self, target, file):
|
||||
super().__init__(
|
||||
f"'{target}' "
|
||||
f"does not exist as an output or a stage name in '{file}'"
|
||||
)
|
||||
|
||||
+115
-81
@@ -4,12 +4,12 @@ import tempfile
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from distutils.dir_util import copy_tree
|
||||
from typing import Iterable
|
||||
|
||||
from funcy import cached_property, retry, suppress, wrap_with
|
||||
from funcy import cached_property, retry, wrap_with
|
||||
|
||||
from dvc.config import NoRemoteError, NotDvcRepoError
|
||||
from dvc.exceptions import (
|
||||
CheckoutError,
|
||||
FileMissingError,
|
||||
NoOutputInExternalRepoError,
|
||||
NoRemoteInExternalRepoError,
|
||||
@@ -18,9 +18,10 @@ from dvc.exceptions import (
|
||||
)
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.tree import RepoTree
|
||||
from dvc.scm.git import Git
|
||||
from dvc.utils import tmp_fname
|
||||
from dvc.utils.fs import fs_copy, move, remove
|
||||
from dvc.scm.tree import is_working_tree
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,10 +30,12 @@ logger = logging.getLogger(__name__)
|
||||
def external_repo(url, rev=None, for_write=False):
|
||||
logger.debug("Creating external repo %s@%s", url, rev)
|
||||
path = _cached_clone(url, rev, for_write=for_write)
|
||||
if not rev:
|
||||
rev = "HEAD"
|
||||
try:
|
||||
repo = ExternalRepo(path, url)
|
||||
repo = ExternalRepo(path, url, rev, for_write=for_write)
|
||||
except NotDvcRepoError:
|
||||
repo = ExternalGitRepo(path, url)
|
||||
repo = ExternalGitRepo(path, url, rev)
|
||||
|
||||
try:
|
||||
yield repo
|
||||
@@ -64,59 +67,99 @@ def clean_repos():
|
||||
_remove(path)
|
||||
|
||||
|
||||
class ExternalRepo(Repo):
|
||||
def __init__(self, root_dir, url):
|
||||
super().__init__(root_dir)
|
||||
class BaseExternalRepo:
|
||||
|
||||
_local_cache = None
|
||||
|
||||
@property
|
||||
def local_cache(self):
|
||||
if hasattr(self, "cache"):
|
||||
return self.cache.local
|
||||
return self._local_cache
|
||||
|
||||
@contextmanager
|
||||
def use_cache(self, cache):
|
||||
"""Use the specified cache in place of default tmpdir cache for
|
||||
download operations.
|
||||
"""
|
||||
if hasattr(self, "cache"):
|
||||
save_cache = self.cache.local
|
||||
self.cache.local = cache
|
||||
self._local_cache = cache
|
||||
|
||||
yield
|
||||
|
||||
if hasattr(self, "cache"):
|
||||
self.cache.local = save_cache
|
||||
self._local_cache = None
|
||||
|
||||
@cached_property
|
||||
def repo_tree(self):
|
||||
return RepoTree(self, fetch=True)
|
||||
|
||||
def get_rev(self):
|
||||
if is_working_tree(self.tree):
|
||||
return self.scm.get_rev()
|
||||
if hasattr(self.tree, "tree"):
|
||||
return self.tree.tree.rev
|
||||
return self.tree.rev
|
||||
|
||||
def fetch_external(self, paths: Iterable, **kwargs):
|
||||
"""Fetch specified external repo paths into cache.
|
||||
|
||||
Returns 3-tuple in the form
|
||||
(downloaded, failed, list(cache_infos))
|
||||
where cache_infos can be used as checkout targets for the
|
||||
fetched paths.
|
||||
"""
|
||||
download_results = []
|
||||
failed = 0
|
||||
|
||||
paths = [PathInfo(self.root_dir) / path for path in paths]
|
||||
|
||||
def download_update(result):
|
||||
download_results.append(result)
|
||||
|
||||
save_infos = []
|
||||
for path in paths:
|
||||
if not self.repo_tree.exists(path):
|
||||
raise PathMissingError(path, self.url)
|
||||
save_info = self.local_cache.save(
|
||||
path,
|
||||
None,
|
||||
tree=self.repo_tree,
|
||||
download_callback=download_update,
|
||||
)
|
||||
save_infos.append(save_info)
|
||||
|
||||
return sum(download_results), failed, save_infos
|
||||
|
||||
def get_external(self, path, dest):
|
||||
"""Convenience wrapper for fetch_external and checkout."""
|
||||
if self.local_cache:
|
||||
# fetch DVC and git files to tmpdir cache, then checkout
|
||||
_, _, save_infos = self.fetch_external([path])
|
||||
self.local_cache.checkout(PathInfo(dest), save_infos[0])
|
||||
else:
|
||||
# git-only erepo with no cache, just copy files directly
|
||||
# to dest
|
||||
path = PathInfo(self.root_dir) / path
|
||||
if not self.repo_tree.exists(path):
|
||||
raise PathMissingError(path, self.url)
|
||||
self.repo_tree.copytree(path, dest)
|
||||
|
||||
|
||||
class ExternalRepo(Repo, BaseExternalRepo):
|
||||
def __init__(self, root_dir, url, rev, for_write=False):
|
||||
if for_write:
|
||||
super().__init__(root_dir)
|
||||
else:
|
||||
root_dir = os.path.realpath(root_dir)
|
||||
super().__init__(root_dir, scm=Git(root_dir), rev=rev)
|
||||
self.url = url
|
||||
self._set_cache_dir()
|
||||
self._fix_upstream()
|
||||
|
||||
def pull_to(self, path, to_info):
|
||||
"""
|
||||
Pull the corresponding file or directory specified by `path` and
|
||||
checkout it into `to_info`.
|
||||
|
||||
It works with files tracked by Git and DVC, and also local files
|
||||
outside the repository.
|
||||
"""
|
||||
out = None
|
||||
path_info = PathInfo(self.root_dir) / path
|
||||
|
||||
with suppress(OutputNotFoundError):
|
||||
(out,) = self.find_outs_by_path(path_info, strict=False)
|
||||
|
||||
try:
|
||||
if out and out.use_cache:
|
||||
self._pull_cached(out, path_info, to_info)
|
||||
return
|
||||
|
||||
# Check if it is handled by Git (it can't have an absolute path)
|
||||
if os.path.isabs(path):
|
||||
raise FileNotFoundError
|
||||
|
||||
fs_copy(path_info, to_info)
|
||||
except FileNotFoundError:
|
||||
raise PathMissingError(path, self.url)
|
||||
|
||||
def _pull_cached(self, out, path_info, dest):
|
||||
with self.state:
|
||||
tmp = PathInfo(tmp_fname(dest))
|
||||
src = tmp / path_info.relative_to(out.path_info)
|
||||
|
||||
out.path_info = tmp
|
||||
|
||||
# Only pull unless all needed cache is present
|
||||
if out.changed_cache(filter_info=src):
|
||||
self.cloud.pull(out.get_used_cache(filter_info=src))
|
||||
|
||||
try:
|
||||
out.checkout(filter_info=src)
|
||||
except CheckoutError:
|
||||
raise FileNotFoundError
|
||||
|
||||
move(src, dest)
|
||||
remove(tmp)
|
||||
|
||||
@wrap_with(threading.Lock())
|
||||
def _set_cache_dir(self):
|
||||
try:
|
||||
@@ -125,6 +168,7 @@ class ExternalRepo(Repo):
|
||||
cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
|
||||
|
||||
self.cache.local.cache_dir = cache_dir
|
||||
self._local_cache = self.cache.local
|
||||
|
||||
def _fix_upstream(self):
|
||||
if not os.path.isdir(self.url):
|
||||
@@ -165,10 +209,11 @@ class ExternalRepo(Repo):
|
||||
self.config["core"]["remote"] = "auto-generated-upstream"
|
||||
|
||||
|
||||
class ExternalGitRepo:
|
||||
def __init__(self, root_dir, url):
|
||||
self.root_dir = root_dir
|
||||
class ExternalGitRepo(BaseExternalRepo):
|
||||
def __init__(self, root_dir, url, rev):
|
||||
self.root_dir = os.path.realpath(root_dir)
|
||||
self.url = url
|
||||
self.tree = self.scm.get_tree(rev)
|
||||
|
||||
@cached_property
|
||||
def scm(self):
|
||||
@@ -181,23 +226,15 @@ class ExternalGitRepo:
|
||||
def find_out_by_relpath(self, path):
|
||||
raise OutputNotFoundError(path, self)
|
||||
|
||||
def pull_to(self, path, to_info):
|
||||
try:
|
||||
# Git handled files can't have absolute path
|
||||
if os.path.isabs(path):
|
||||
raise FileNotFoundError
|
||||
|
||||
fs_copy(os.path.join(self.root_dir, path), to_info)
|
||||
except FileNotFoundError:
|
||||
raise PathMissingError(path, self.url)
|
||||
|
||||
@contextmanager
|
||||
def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
|
||||
"""Opens a specified resource as a file object."""
|
||||
tree = RepoTree(self)
|
||||
try:
|
||||
abs_path = os.path.join(self.root_dir, path)
|
||||
with open(abs_path, mode, encoding=encoding) as fd:
|
||||
yield fd
|
||||
with tree.open(
|
||||
path, mode=mode, encoding=encoding, **kwargs
|
||||
) as fobj:
|
||||
yield fobj
|
||||
except FileNotFoundError:
|
||||
raise PathMissingError(path, self.url)
|
||||
|
||||
@@ -209,14 +246,12 @@ def _cached_clone(url, rev, for_write=False):
|
||||
revision checked out. If for_write is set prevents reusing this dir via
|
||||
cache.
|
||||
"""
|
||||
if not for_write and Git.is_sha(rev) and (url, rev) in CLONES:
|
||||
return CLONES[url, rev]
|
||||
|
||||
# even if we have already cloned this repo, we may need to
|
||||
# fetch/fast-forward to get specified rev
|
||||
clone_path = _clone_default_branch(url, rev)
|
||||
rev_sha = Git(clone_path).resolve_rev(rev or "HEAD")
|
||||
|
||||
if not for_write and (url, rev_sha) in CLONES:
|
||||
return CLONES[url, rev_sha]
|
||||
if not for_write and (url) in CLONES:
|
||||
return CLONES[url]
|
||||
|
||||
# Copy to a new dir to keep the clone clean
|
||||
repo_path = tempfile.mkdtemp("dvc-erepo")
|
||||
@@ -224,11 +259,10 @@ def _cached_clone(url, rev, for_write=False):
|
||||
copy_tree(clone_path, repo_path)
|
||||
|
||||
# Check out the specified revision
|
||||
if rev is not None:
|
||||
if for_write:
|
||||
_git_checkout(repo_path, rev)
|
||||
|
||||
if not for_write:
|
||||
CLONES[url, rev_sha] = repo_path
|
||||
else:
|
||||
CLONES[url] = repo_path
|
||||
return repo_path
|
||||
|
||||
|
||||
|
||||
+85
-13
@@ -5,6 +5,7 @@ from funcy import cached_property
|
||||
from pathspec import PathSpec
|
||||
from pathspec.patterns import GitWildMatchPattern
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.scm.tree import BaseTree
|
||||
from dvc.utils import relpath
|
||||
|
||||
@@ -35,12 +36,19 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
return dirs, files
|
||||
|
||||
def matches(self, dirname, basename):
|
||||
abs_path = os.path.join(dirname, basename)
|
||||
rel_path = relpath(abs_path, self.dirname)
|
||||
|
||||
if os.pardir + os.sep in rel_path:
|
||||
# NOTE: `relpath` is too slow, so we have to assume that both
|
||||
# `dirname` and `self.dirname` are relative or absolute together.
|
||||
prefix = self.dirname + os.sep
|
||||
if dirname == self.dirname:
|
||||
path = basename
|
||||
elif dirname.startswith(prefix):
|
||||
rel = dirname[len(prefix) :]
|
||||
# NOTE: `os.path.join` is ~x5.5 slower
|
||||
path = f"{rel}{os.sep}{basename}"
|
||||
else:
|
||||
return False
|
||||
return self.ignore_spec.match_file(rel_path)
|
||||
|
||||
return self.ignore_spec.match_file(path)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.ignore_file_path)
|
||||
@@ -76,7 +84,7 @@ class DvcIgnoreRepo(DvcIgnore):
|
||||
def is_dvc_repo(directory):
|
||||
from dvc.repo import Repo
|
||||
|
||||
return os.path.isdir(os.path.join(directory, Repo.DVC_DIR))
|
||||
return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
|
||||
|
||||
dirs = [d for d in dirs if not is_dvc_repo(d)]
|
||||
|
||||
@@ -119,25 +127,89 @@ class CleanTree(BaseTree):
|
||||
return self.tree.tree_root
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8"):
|
||||
return self.tree.open(path, mode, encoding)
|
||||
if self.isfile(path):
|
||||
return self.tree.open(path, mode, encoding)
|
||||
raise FileNotFoundError
|
||||
|
||||
def exists(self, path):
|
||||
return self.tree.exists(path)
|
||||
if self.tree.exists(path) and self._parents_exist(path):
|
||||
if self.tree.isdir(path):
|
||||
return self._valid_dirname(path)
|
||||
return self._valid_filename(path)
|
||||
return False
|
||||
|
||||
def isdir(self, path):
|
||||
return self.tree.isdir(path)
|
||||
return (
|
||||
self.tree.isdir(path)
|
||||
and self._parents_exist(path)
|
||||
and self._valid_dirname(path)
|
||||
)
|
||||
|
||||
def _valid_dirname(self, path):
|
||||
dirname, basename = os.path.split(os.path.normpath(path))
|
||||
dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
|
||||
if dirs:
|
||||
return True
|
||||
return False
|
||||
|
||||
def isfile(self, path):
|
||||
return self.tree.isfile(path)
|
||||
return (
|
||||
self.tree.isfile(path)
|
||||
and self._parents_exist(path)
|
||||
and self._valid_filename(path)
|
||||
)
|
||||
|
||||
def _valid_filename(self, path):
|
||||
dirname, basename = os.path.split(os.path.normpath(path))
|
||||
_, files = self.dvcignore(os.path.abspath(dirname), [], [basename])
|
||||
if files:
|
||||
return True
|
||||
return False
|
||||
|
||||
def isexec(self, path):
|
||||
return self.tree.isexec(path)
|
||||
return self.exists(path) and self.tree.isexec(path)
|
||||
|
||||
def _parents_exist(self, path):
|
||||
from dvc.repo import Repo
|
||||
|
||||
path = PathInfo(path)
|
||||
|
||||
# if parent is tree_root or inside a .dvc dir we can skip this check
|
||||
if path.parent == self.tree_root or Repo.DVC_DIR in path.parts:
|
||||
return True
|
||||
|
||||
# if path is outside of tree, assume this is a local remote/local cache
|
||||
# link/move operation where we do not need to filter ignores
|
||||
path = relpath(path, self.tree_root)
|
||||
if path.startswith("..") or (
|
||||
os.name == "nt"
|
||||
and not os.path.commonprefix(
|
||||
[os.path.abspath(path), self.tree_root]
|
||||
)
|
||||
):
|
||||
return True
|
||||
|
||||
# check if parent directories are in our ignores, starting from
|
||||
# tree_root
|
||||
for parent_dir in reversed(PathInfo(path).parents):
|
||||
dirname, basename = os.path.split(parent_dir)
|
||||
if basename == ".":
|
||||
# parent_dir == tree_root
|
||||
continue
|
||||
dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
|
||||
if not dirs:
|
||||
return False
|
||||
return True
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
for root, dirs, files in self.tree.walk(top, topdown):
|
||||
dirs[:], files[:] = self.dvcignore(root, dirs, files)
|
||||
dirs[:], files[:] = self.dvcignore(
|
||||
os.path.abspath(root), dirs, files
|
||||
)
|
||||
|
||||
yield root, dirs, files
|
||||
|
||||
def stat(self, path):
|
||||
return self.tree.stat(path)
|
||||
if self.exists(path):
|
||||
return self.tree.stat(path)
|
||||
raise FileNotFoundError
|
||||
|
||||
+6
-2
@@ -7,13 +7,17 @@
|
||||
TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
|
||||
|
||||
|
||||
def istextfile(fname, blocksize=512):
|
||||
def istextfile(fname, blocksize=512, tree=None):
|
||||
""" Uses heuristics to guess whether the given file is text or binary,
|
||||
by reading a single block of bytes from the file.
|
||||
If more than 30% of the chars in the block are non-text, or there
|
||||
are NUL ('\x00') bytes in the block, assume this is a binary file.
|
||||
"""
|
||||
with open(fname, "rb") as fobj:
|
||||
if tree:
|
||||
open_func = tree.open
|
||||
else:
|
||||
open_func = open
|
||||
with open_func(fname, "rb") as fobj:
|
||||
block = fobj.read(blocksize)
|
||||
|
||||
if not block:
|
||||
|
||||
+1
-1
@@ -47,7 +47,7 @@ class Lock:
|
||||
disable=not self._friendly,
|
||||
desc=(
|
||||
"If DVC froze, see `hardlink_lock` in {}".format(
|
||||
format_link("man.dvc.org/config#core")
|
||||
format_link("https://man.dvc.org/config#core")
|
||||
)
|
||||
),
|
||||
):
|
||||
|
||||
+50
-10
@@ -18,20 +18,45 @@ FOOTER = (
|
||||
)
|
||||
|
||||
|
||||
def addLoggingLevel(levelName, levelNum, methodName=None):
|
||||
"""
|
||||
Adds a new logging level to the `logging` module and the
|
||||
currently configured logging class.
|
||||
|
||||
Based on https://stackoverflow.com/questions/2183233
|
||||
"""
|
||||
if methodName is None:
|
||||
methodName = levelName.lower()
|
||||
|
||||
assert not hasattr(logging, levelName)
|
||||
assert not hasattr(logging, methodName)
|
||||
assert not hasattr(logging.getLoggerClass(), methodName)
|
||||
|
||||
def logForLevel(self, message, *args, **kwargs):
|
||||
if self.isEnabledFor(levelNum):
|
||||
self._log(levelNum, message, args, **kwargs)
|
||||
|
||||
def logToRoot(message, *args, **kwargs):
|
||||
logging.log(levelNum, message, *args, **kwargs)
|
||||
|
||||
logging.addLevelName(levelNum, levelName)
|
||||
setattr(logging, levelName, levelNum)
|
||||
setattr(logging.getLoggerClass(), methodName, logForLevel)
|
||||
setattr(logging, methodName, logToRoot)
|
||||
|
||||
|
||||
class LoggingException(Exception):
|
||||
def __init__(self, record):
|
||||
msg = "failed to log {}".format(str(record))
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class ExcludeErrorsFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.levelno < logging.WARNING
|
||||
def excludeFilter(level):
|
||||
class ExcludeLevelFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.levelno < level
|
||||
|
||||
|
||||
class ExcludeInfoFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.levelno < logging.INFO
|
||||
return ExcludeLevelFilter
|
||||
|
||||
|
||||
class ColorFormatter(logging.Formatter):
|
||||
@@ -47,6 +72,7 @@ class ColorFormatter(logging.Formatter):
|
||||
"""
|
||||
|
||||
color_code = {
|
||||
"TRACE": colorama.Fore.GREEN,
|
||||
"DEBUG": colorama.Fore.BLUE,
|
||||
"WARNING": colorama.Fore.YELLOW,
|
||||
"ERROR": colorama.Fore.RED,
|
||||
@@ -116,7 +142,11 @@ class LoggerHandler(logging.StreamHandler):
|
||||
|
||||
|
||||
def _is_verbose():
|
||||
return logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG
|
||||
return (
|
||||
logging.NOTSET
|
||||
< logging.getLogger("dvc").getEffectiveLevel()
|
||||
<= logging.DEBUG
|
||||
)
|
||||
|
||||
|
||||
def _iter_causes(exc):
|
||||
@@ -152,12 +182,14 @@ def disable_other_loggers():
|
||||
def setup(level=logging.INFO):
|
||||
colorama.init()
|
||||
|
||||
addLoggingLevel("TRACE", logging.DEBUG - 5)
|
||||
logging.config.dictConfig(
|
||||
{
|
||||
"version": 1,
|
||||
"filters": {
|
||||
"exclude_errors": {"()": ExcludeErrorsFilter},
|
||||
"exclude_info": {"()": ExcludeInfoFilter},
|
||||
"exclude_errors": {"()": excludeFilter(logging.WARNING)},
|
||||
"exclude_info": {"()": excludeFilter(logging.INFO)},
|
||||
"exclude_debug": {"()": excludeFilter(logging.DEBUG)},
|
||||
},
|
||||
"formatters": {"color": {"()": ColorFormatter}},
|
||||
"handlers": {
|
||||
@@ -175,6 +207,13 @@ def setup(level=logging.INFO):
|
||||
"stream": "ext://sys.stdout",
|
||||
"filters": ["exclude_info"],
|
||||
},
|
||||
"console_trace": {
|
||||
"class": "dvc.logger.LoggerHandler",
|
||||
"level": "TRACE",
|
||||
"formatter": "color",
|
||||
"stream": "ext://sys.stdout",
|
||||
"filters": ["exclude_debug"],
|
||||
},
|
||||
"console_errors": {
|
||||
"class": "dvc.logger.LoggerHandler",
|
||||
"level": "WARNING",
|
||||
@@ -188,6 +227,7 @@ def setup(level=logging.INFO):
|
||||
"handlers": [
|
||||
"console_info",
|
||||
"console_debug",
|
||||
"console_trace",
|
||||
"console_errors",
|
||||
],
|
||||
},
|
||||
|
||||
+11
-6
@@ -18,7 +18,6 @@ from dvc.utils import format_link
|
||||
|
||||
"".encode("idna")
|
||||
|
||||
|
||||
logger = logging.getLogger("dvc")
|
||||
|
||||
|
||||
@@ -38,11 +37,17 @@ def main(argv=None):
|
||||
try:
|
||||
args = parse_args(argv)
|
||||
|
||||
if args.quiet:
|
||||
logger.setLevel(logging.CRITICAL)
|
||||
|
||||
elif args.verbose:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
verbosity = args.verbose - args.quiet
|
||||
if verbosity:
|
||||
logger.setLevel(
|
||||
{
|
||||
-2: logging.CRITICAL,
|
||||
-1: logging.ERROR,
|
||||
1: logging.DEBUG,
|
||||
2: logging.TRACE,
|
||||
}[max(-2, min(verbosity, 2))]
|
||||
)
|
||||
logger.trace(args)
|
||||
|
||||
cmd = args.func(args)
|
||||
ret = cmd.run()
|
||||
|
||||
+29
-19
@@ -31,7 +31,11 @@ class OutputIsNotFileOrDirError(DvcException):
|
||||
|
||||
class OutputAlreadyTrackedError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = f"output '{path}' is already tracked by SCM (e.g. Git)"
|
||||
msg = f""" output '{path}' is already tracked by SCM (e.g. Git).
|
||||
You can remove it from Git, then add to DVC.
|
||||
To stop tracking from Git:
|
||||
git rm -r --cached '{path}'
|
||||
git commit -m "stop tracking {path}" """
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
@@ -241,17 +245,15 @@ class BaseOutput:
|
||||
if self.metric or self.plot:
|
||||
self.verify_metric()
|
||||
if not self.IS_DEPENDENCY:
|
||||
logger.info(
|
||||
"Output '{}' doesn't use cache. Skipping saving.".format(
|
||||
self
|
||||
)
|
||||
logger.debug(
|
||||
"Output '%s' doesn't use cache. Skipping saving.", self
|
||||
)
|
||||
return
|
||||
|
||||
assert not self.IS_DEPENDENCY
|
||||
|
||||
if not self.changed():
|
||||
logger.info(f"Output '{self}' didn't change. Skipping saving.")
|
||||
logger.debug("Output '%s' didn't change. Skipping saving.", self)
|
||||
return
|
||||
|
||||
self.info = self.save_info()
|
||||
@@ -267,7 +269,8 @@ class BaseOutput:
|
||||
if self.IS_DEPENDENCY:
|
||||
return ret
|
||||
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
if not self.use_cache:
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
|
||||
if isinstance(self.metric, dict):
|
||||
if (
|
||||
@@ -276,12 +279,14 @@ class BaseOutput:
|
||||
):
|
||||
del self.metric[self.PARAM_METRIC_XPATH]
|
||||
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
if self.metric:
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
|
||||
if self.plot:
|
||||
ret[self.PARAM_PLOT] = self.plot
|
||||
|
||||
ret[self.PARAM_PERSIST] = self.persist
|
||||
if self.persist:
|
||||
ret[self.PARAM_PERSIST] = self.persist
|
||||
|
||||
return ret
|
||||
|
||||
@@ -347,6 +352,17 @@ class BaseOutput:
|
||||
if self.exists:
|
||||
self.remote.unprotect(self.path_info)
|
||||
|
||||
def get_dir_cache(self, **kwargs):
|
||||
if not self.is_dir_checksum:
|
||||
raise DvcException("cannot get dir cache for file checksum")
|
||||
if self.cache.changed_cache_file(self.checksum):
|
||||
self.repo.cloud.pull(
|
||||
NamedCache.make("local", self.checksum, str(self)),
|
||||
show_checksums=False,
|
||||
**kwargs,
|
||||
)
|
||||
return self.dir_cache
|
||||
|
||||
def collect_used_dir_cache(
|
||||
self, remote=None, force=False, jobs=None, filter_info=None
|
||||
):
|
||||
@@ -371,16 +387,10 @@ class BaseOutput:
|
||||
|
||||
cache = NamedCache()
|
||||
|
||||
if self.cache.changed_cache_file(self.checksum):
|
||||
try:
|
||||
self.repo.cloud.pull(
|
||||
NamedCache.make("local", self.checksum, str(self)),
|
||||
jobs=jobs,
|
||||
remote=remote,
|
||||
show_checksums=False,
|
||||
)
|
||||
except DvcException:
|
||||
logger.debug(f"failed to pull cache for '{self}'")
|
||||
try:
|
||||
self.get_dir_cache(jobs=jobs, remote=remote)
|
||||
except DvcException:
|
||||
logger.debug(f"failed to pull cache for '{self}'")
|
||||
|
||||
if self.cache.changed_cache_file(self.checksum):
|
||||
msg = (
|
||||
|
||||
+15
-31
@@ -1,10 +1,8 @@
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
@@ -19,12 +17,6 @@ logger = logging.getLogger(__name__)
|
||||
class AzureRemote(BaseRemote):
|
||||
scheme = Schemes.AZURE
|
||||
path_cls = CloudURLInfo
|
||||
REGEX = (
|
||||
r"azure://((?P<path>[^=;]*)?|("
|
||||
# backward compatibility
|
||||
r"(ContainerName=(?P<container_name>[^;]+);?)?"
|
||||
r"(?P<connection_string>.+)?)?)$"
|
||||
)
|
||||
REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
@@ -34,31 +26,14 @@ class AzureRemote(BaseRemote):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url", "azure://")
|
||||
self.path_info = self.path_cls(url)
|
||||
|
||||
match = re.match(self.REGEX, url) # backward compatibility
|
||||
path = match.group("path")
|
||||
bucket = (
|
||||
urlparse(url if path else "").netloc
|
||||
or match.group("container_name") # backward compatibility
|
||||
or os.getenv("AZURE_STORAGE_CONTAINER_NAME")
|
||||
)
|
||||
if not self.path_info.bucket:
|
||||
container = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
|
||||
self.path_info = self.path_cls(f"azure://{container}")
|
||||
|
||||
self.connection_string = (
|
||||
config.get("connection_string")
|
||||
or match.group("connection_string") # backward compatibility
|
||||
or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
|
||||
)
|
||||
|
||||
if not bucket:
|
||||
raise ValueError("azure storage container name missing")
|
||||
|
||||
if not self.connection_string:
|
||||
raise ValueError("azure storage connection string missing")
|
||||
|
||||
self.path_info = (
|
||||
self.path_cls(url)
|
||||
if path
|
||||
else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
|
||||
self.connection_string = config.get("connection_string") or os.getenv(
|
||||
"AZURE_STORAGE_CONNECTION_STRING"
|
||||
)
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@@ -81,6 +56,15 @@ class AzureRemote(BaseRemote):
|
||||
blob_service.create_container(self.path_info.bucket)
|
||||
return blob_service
|
||||
|
||||
def get_etag(self, path_info):
|
||||
etag = self.blob_service.get_blob_properties(
|
||||
path_info.bucket, path_info.path
|
||||
).properties.etag
|
||||
return etag.strip('"')
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
return self.get_etag(path_info)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
+120
-71
@@ -26,6 +26,7 @@ from dvc.path_info import PathInfo, URLInfo, WindowsPathInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.index import RemoteIndex, RemoteIndexNoop
|
||||
from dvc.remote.slow_link_detection import slow_link_guard
|
||||
from dvc.scm.tree import is_working_tree
|
||||
from dvc.state import StateNoop
|
||||
from dvc.utils import tmp_fname
|
||||
from dvc.utils.fs import makedirs, move
|
||||
@@ -211,23 +212,33 @@ class BaseRemote:
|
||||
checksums = dict(zip(file_infos, tasks))
|
||||
return checksums
|
||||
|
||||
def _collect_dir(self, path_info):
|
||||
def _collect_dir(self, path_info, tree=None, save_tree=False, **kwargs):
|
||||
file_infos = set()
|
||||
|
||||
for fname in self.walk_files(path_info):
|
||||
if tree:
|
||||
walk_files = tree.walk_files
|
||||
else:
|
||||
walk_files = self.walk_files
|
||||
|
||||
for fname in walk_files(path_info, **kwargs):
|
||||
if DvcIgnore.DVCIGNORE_FILE == fname.name:
|
||||
raise DvcIgnoreInCollectedDirError(fname.parent)
|
||||
|
||||
file_infos.add(fname)
|
||||
|
||||
checksums = {fi: self.state.get(fi) for fi in file_infos}
|
||||
not_in_state = {
|
||||
fi for fi, checksum in checksums.items() if checksum is None
|
||||
}
|
||||
if tree:
|
||||
checksums = {fi: tree.get_file_checksum(fi) for fi in file_infos}
|
||||
if save_tree:
|
||||
for fi, checksum in checksums.items():
|
||||
self._save_file(fi, checksum, tree=tree, **kwargs)
|
||||
else:
|
||||
checksums = {fi: self.state.get(fi) for fi in file_infos}
|
||||
not_in_state = {
|
||||
fi for fi, checksum in checksums.items() if checksum is None
|
||||
}
|
||||
|
||||
new_checksums = self._calculate_checksums(not_in_state)
|
||||
|
||||
checksums.update(new_checksums)
|
||||
new_checksums = self._calculate_checksums(not_in_state)
|
||||
checksums.update(new_checksums)
|
||||
|
||||
result = [
|
||||
{
|
||||
@@ -248,18 +259,25 @@ class BaseRemote:
|
||||
# Sorting the list by path to ensure reproducibility
|
||||
return sorted(result, key=itemgetter(self.PARAM_RELPATH))
|
||||
|
||||
def get_dir_checksum(self, path_info):
|
||||
def get_dir_checksum(self, path_info, tree=None):
|
||||
if not self.cache:
|
||||
raise RemoteCacheRequiredError(path_info)
|
||||
|
||||
dir_info = self._collect_dir(path_info)
|
||||
dir_info = self._collect_dir(path_info, tree=None)
|
||||
if tree:
|
||||
# don't save state entry for path_info if it is a tree path
|
||||
path_info = None
|
||||
return self._save_dir_info(dir_info, path_info)
|
||||
|
||||
def _save_dir_info(self, dir_info, path_info=None):
|
||||
checksum, tmp_info = self._get_dir_info_checksum(dir_info)
|
||||
new_info = self.cache.checksum_to_path_info(checksum)
|
||||
if self.cache.changed_cache_file(checksum):
|
||||
self.cache.makedirs(new_info.parent)
|
||||
self.cache.move(tmp_info, new_info, mode=self.CACHE_MODE)
|
||||
|
||||
self.state.save(path_info, checksum)
|
||||
if path_info:
|
||||
self.state.save(path_info, checksum)
|
||||
self.state.save(new_info, checksum)
|
||||
|
||||
return checksum
|
||||
@@ -319,6 +337,8 @@ class BaseRemote:
|
||||
|
||||
@classmethod
|
||||
def is_dir_checksum(cls, checksum):
|
||||
if not checksum:
|
||||
return False
|
||||
return checksum.endswith(cls.CHECKSUM_DIR_SUFFIX)
|
||||
|
||||
def get_checksum(self, path_info):
|
||||
@@ -452,28 +472,45 @@ class BaseRemote:
|
||||
"Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
|
||||
)
|
||||
|
||||
def _save_file(self, path_info, checksum, save_link=True):
|
||||
def _save_file(
|
||||
self, path_info, checksum, save_link=True, tree=None, **kwargs
|
||||
):
|
||||
assert checksum
|
||||
|
||||
cache_info = self.checksum_to_path_info(checksum)
|
||||
if self.changed_cache(checksum):
|
||||
self.move(path_info, cache_info, mode=self.CACHE_MODE)
|
||||
self.link(cache_info, path_info)
|
||||
elif self.iscopy(path_info) and self._cache_is_copy(path_info):
|
||||
# Default relink procedure involves unneeded copy
|
||||
self.unprotect(path_info)
|
||||
if tree:
|
||||
if self.changed_cache(checksum):
|
||||
with tree.open(path_info, mode="rb") as fobj:
|
||||
# if tree has fetch enabled, DVC out will be fetched on
|
||||
# open and we do not need to read/copy any data
|
||||
if not (
|
||||
tree.isdvc(path_info, strict=False) and tree.fetch
|
||||
):
|
||||
self.copy_fobj(fobj, cache_info)
|
||||
callback = kwargs.get("download_callback")
|
||||
if callback:
|
||||
callback(1)
|
||||
else:
|
||||
self.remove(path_info)
|
||||
self.link(cache_info, path_info)
|
||||
if self.changed_cache(checksum):
|
||||
self.move(path_info, cache_info, mode=self.CACHE_MODE)
|
||||
self.link(cache_info, path_info)
|
||||
elif self.iscopy(path_info) and self._cache_is_copy(path_info):
|
||||
# Default relink procedure involves unneeded copy
|
||||
self.unprotect(path_info)
|
||||
else:
|
||||
self.remove(path_info)
|
||||
self.link(cache_info, path_info)
|
||||
|
||||
if save_link:
|
||||
self.state.save_link(path_info)
|
||||
if save_link:
|
||||
self.state.save_link(path_info)
|
||||
|
||||
# we need to update path and cache, since in case of reflink,
|
||||
# or copy cache type moving original file results in updates on
|
||||
# next executed command, which causes md5 recalculation
|
||||
self.state.save(path_info, checksum)
|
||||
if not tree or is_working_tree(tree):
|
||||
self.state.save(path_info, checksum)
|
||||
self.state.save(cache_info, checksum)
|
||||
return {self.PARAM_CHECKSUM: checksum}
|
||||
|
||||
def _cache_is_copy(self, path_info):
|
||||
"""Checks whether cache uses copies."""
|
||||
@@ -497,22 +534,32 @@ class BaseRemote:
|
||||
self.cache_type_confirmed = True
|
||||
return self.cache_types[0] == "copy"
|
||||
|
||||
def _save_dir(self, path_info, checksum, save_link=True):
|
||||
def _save_dir(
|
||||
self, path_info, checksum, save_link=True, tree=None, **kwargs
|
||||
):
|
||||
if tree:
|
||||
dir_info = self._collect_dir(
|
||||
path_info, tree=tree, save_tree=True, **kwargs
|
||||
)
|
||||
checksum = self._save_dir_info(dir_info)
|
||||
else:
|
||||
dir_info = self.get_dir_cache(checksum)
|
||||
|
||||
for entry in Tqdm(
|
||||
dir_info, desc="Saving " + path_info.name, unit="file"
|
||||
):
|
||||
entry_info = path_info / entry[self.PARAM_RELPATH]
|
||||
entry_checksum = entry[self.PARAM_CHECKSUM]
|
||||
self._save_file(entry_info, entry_checksum, save_link=False)
|
||||
|
||||
if save_link:
|
||||
self.state.save_link(path_info)
|
||||
|
||||
cache_info = self.checksum_to_path_info(checksum)
|
||||
dir_info = self.get_dir_cache(checksum)
|
||||
|
||||
for entry in Tqdm(
|
||||
dir_info, desc="Saving " + path_info.name, unit="file"
|
||||
):
|
||||
entry_info = path_info / entry[self.PARAM_RELPATH]
|
||||
entry_checksum = entry[self.PARAM_CHECKSUM]
|
||||
self._save_file(entry_info, entry_checksum, save_link=False)
|
||||
|
||||
if save_link:
|
||||
self.state.save_link(path_info)
|
||||
|
||||
self.state.save(cache_info, checksum)
|
||||
self.state.save(path_info, checksum)
|
||||
if not tree or is_working_tree(tree):
|
||||
self.state.save(path_info, checksum)
|
||||
return {self.PARAM_CHECKSUM: checksum}
|
||||
|
||||
def is_empty(self, path_info):
|
||||
return False
|
||||
@@ -541,22 +588,42 @@ class BaseRemote:
|
||||
def protect(path_info):
|
||||
pass
|
||||
|
||||
def save(self, path_info, checksum_info, save_link=True):
|
||||
def save(
|
||||
self, path_info, checksum_info, save_link=True, tree=None, **kwargs
|
||||
):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise RemoteActionNotImplemented(
|
||||
f"save {path_info.scheme} -> {self.scheme}", self.scheme,
|
||||
)
|
||||
|
||||
checksum = checksum_info[self.PARAM_CHECKSUM]
|
||||
self._save(path_info, checksum, save_link)
|
||||
if tree:
|
||||
if tree.isdir(path_info):
|
||||
# save checksum will be computed during tree walk
|
||||
checksum = None
|
||||
else:
|
||||
checksum = tree.get_file_checksum(path_info)
|
||||
else:
|
||||
checksum = checksum_info[self.PARAM_CHECKSUM]
|
||||
return self._save(path_info, checksum, save_link, tree, **kwargs)
|
||||
|
||||
def _save(self, path_info, checksum, save_link=True):
|
||||
to_info = self.checksum_to_path_info(checksum)
|
||||
logger.debug("Saving '%s' to '%s'.", path_info, to_info)
|
||||
if self.isdir(path_info):
|
||||
self._save_dir(path_info, checksum, save_link)
|
||||
return
|
||||
self._save_file(path_info, checksum, save_link)
|
||||
def _save(self, path_info, checksum, save_link=True, tree=None, **kwargs):
|
||||
if tree:
|
||||
logger.debug("Saving tree path '%s' to cache.", path_info)
|
||||
else:
|
||||
to_info = self.checksum_to_path_info(checksum)
|
||||
logger.debug("Saving '%s' to '%s'.", path_info, to_info)
|
||||
|
||||
if tree:
|
||||
isdir = tree.isdir
|
||||
save_link = False
|
||||
else:
|
||||
isdir = self.isdir
|
||||
|
||||
if isdir(path_info):
|
||||
return self._save_dir(
|
||||
path_info, checksum, save_link, tree, **kwargs
|
||||
)
|
||||
return self._save_file(path_info, checksum, save_link, tree, **kwargs)
|
||||
|
||||
def _handle_transfer_exception(
|
||||
self, from_info, to_info, exception, operation
|
||||
@@ -695,6 +762,9 @@ class BaseRemote:
|
||||
def copy(self, from_info, to_info):
|
||||
raise RemoteActionNotImplemented("copy", self.scheme)
|
||||
|
||||
def copy_fobj(self, fobj, to_info):
|
||||
raise RemoteActionNotImplemented("copy_fobj", self.scheme)
|
||||
|
||||
def symlink(self, from_info, to_info):
|
||||
raise RemoteActionNotImplemented("symlink", self.scheme)
|
||||
|
||||
@@ -761,7 +831,7 @@ class BaseRemote:
|
||||
|
||||
@index_locked
|
||||
def gc(self, named_cache, jobs=None):
|
||||
used = self.extract_used_local_checksums(named_cache)
|
||||
used = set(named_cache.scheme_keys("local"))
|
||||
|
||||
if self.scheme != "":
|
||||
used.update(named_cache.scheme_keys(self.scheme))
|
||||
@@ -777,6 +847,7 @@ class BaseRemote:
|
||||
continue
|
||||
path_info = self.checksum_to_path_info(checksum)
|
||||
if self.is_dir_checksum(checksum):
|
||||
# backward compatibility
|
||||
self._remove_unpacked_dir(checksum)
|
||||
self.remove(path_info)
|
||||
removed = True
|
||||
@@ -835,11 +906,6 @@ class BaseRemote:
|
||||
if self.changed_cache_file(checksum):
|
||||
return True
|
||||
|
||||
if not (path_info and filter_info) and not self._changed_unpacked_dir(
|
||||
checksum
|
||||
):
|
||||
return False
|
||||
|
||||
for entry in self.get_dir_cache(checksum):
|
||||
entry_checksum = entry[self.PARAM_CHECKSUM]
|
||||
|
||||
@@ -851,9 +917,6 @@ class BaseRemote:
|
||||
if self.changed_cache_file(entry_checksum):
|
||||
return True
|
||||
|
||||
if not (path_info and filter_info):
|
||||
self._update_unpacked_dir(checksum)
|
||||
|
||||
return False
|
||||
|
||||
def changed_cache(self, checksum, path_info=None, filter_info=None):
|
||||
@@ -1292,19 +1355,5 @@ class BaseRemote:
|
||||
def unprotect(path_info):
|
||||
pass
|
||||
|
||||
def _get_unpacked_dir_names(self, checksums):
|
||||
return set()
|
||||
|
||||
def extract_used_local_checksums(self, named_cache):
|
||||
used = set(named_cache.scheme_keys("local"))
|
||||
unpacked = self._get_unpacked_dir_names(used)
|
||||
return used | unpacked
|
||||
|
||||
def _changed_unpacked_dir(self, checksum):
|
||||
return True
|
||||
|
||||
def _update_unpacked_dir(self, checksum):
|
||||
pass
|
||||
|
||||
def _remove_unpacked_dir(self, checksum):
|
||||
pass
|
||||
|
||||
@@ -165,6 +165,8 @@ class HDFSRemote(BaseRemote):
|
||||
dirs = deque([root])
|
||||
|
||||
with self.hdfs(self.path_info) as hdfs:
|
||||
if prefix and not hdfs.exists(root):
|
||||
return
|
||||
while dirs:
|
||||
try:
|
||||
entries = hdfs.ls(dirs.pop(), detail=True)
|
||||
|
||||
+71
-73
@@ -21,10 +21,17 @@ from dvc.remote.base import (
|
||||
)
|
||||
from dvc.remote.index import RemoteIndexNoop
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.scm.tree import is_working_tree
|
||||
from dvc.scm.tree import WorkingTree, is_working_tree
|
||||
from dvc.system import System
|
||||
from dvc.utils import file_md5, relpath, tmp_fname
|
||||
from dvc.utils.fs import copyfile, makedirs, move, remove, walk_files
|
||||
from dvc.utils.fs import (
|
||||
copy_fobj_to_file,
|
||||
copyfile,
|
||||
makedirs,
|
||||
move,
|
||||
remove,
|
||||
walk_files,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -61,6 +68,21 @@ class LocalRemote(BaseRemote):
|
||||
def cache_dir(self, value):
|
||||
self.path_info = PathInfo(value) if value else None
|
||||
|
||||
@cached_property
|
||||
def _work_tree(self):
|
||||
if self.repo:
|
||||
return WorkingTree(self.repo.root_dir)
|
||||
return None
|
||||
|
||||
@property
|
||||
def work_tree(self):
|
||||
# When using repo.brancher, repo.tree may change to/from WorkingTree to
|
||||
# GitTree arbitarily. When repo.tree is GitTree, local cache needs to
|
||||
# use its own WorkingTree instance.
|
||||
if self.repo and not is_working_tree(self.repo.tree):
|
||||
return self._work_tree
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def supported(cls, config):
|
||||
return True
|
||||
@@ -70,12 +92,19 @@ class LocalRemote(BaseRemote):
|
||||
return os.path.abspath(self.cache_dir)
|
||||
|
||||
def checksum_to_path(self, checksum):
|
||||
return os.path.join(self.cache_path, checksum[0:2], checksum[2:])
|
||||
# NOTE: `self.cache_path` is already normalized so we can simply use
|
||||
# `os.sep` instead of `os.path.join`. This results in this helper
|
||||
# being ~5.5 times faster.
|
||||
return (
|
||||
f"{self.cache_path}{os.sep}{checksum[0:2]}{os.sep}{checksum[2:]}"
|
||||
)
|
||||
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
assert self.path_info is not None
|
||||
if prefix:
|
||||
path_info = self.path_info / prefix[:2]
|
||||
if not self.exists(path_info):
|
||||
return
|
||||
else:
|
||||
path_info = self.path_info
|
||||
if progress_callback:
|
||||
@@ -92,8 +121,11 @@ class LocalRemote(BaseRemote):
|
||||
return self.checksum_to_path_info(md5).url
|
||||
|
||||
def exists(self, path_info):
|
||||
assert is_working_tree(self.repo.tree)
|
||||
assert isinstance(path_info, str) or path_info.scheme == "local"
|
||||
if not self.repo:
|
||||
return os.path.exists(path_info)
|
||||
if self.work_tree and self.work_tree.exists(path_info):
|
||||
return True
|
||||
return self.repo.tree.exists(path_info)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
@@ -126,13 +158,19 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def isfile(path_info):
|
||||
return os.path.isfile(path_info)
|
||||
def isfile(self, path_info):
|
||||
if not self.repo:
|
||||
return os.path.isfile(path_info)
|
||||
if self.work_tree and self.work_tree.isfile(path_info):
|
||||
return True
|
||||
return self.repo.tree.isfile(path_info)
|
||||
|
||||
@staticmethod
|
||||
def isdir(path_info):
|
||||
return os.path.isdir(path_info)
|
||||
def isdir(self, path_info):
|
||||
if not self.repo:
|
||||
return os.path.isdir(path_info)
|
||||
if self.work_tree and self.work_tree.isdir(path_info):
|
||||
return True
|
||||
return self.repo.tree.isdir(path_info)
|
||||
|
||||
def iscopy(self, path_info):
|
||||
return not (
|
||||
@@ -144,9 +182,11 @@ class LocalRemote(BaseRemote):
|
||||
return os.path.getsize(path_info)
|
||||
|
||||
def walk_files(self, path_info):
|
||||
assert is_working_tree(self.repo.tree)
|
||||
|
||||
for fname in self.repo.tree.walk_files(path_info):
|
||||
if self.work_tree:
|
||||
tree = self.work_tree
|
||||
else:
|
||||
tree = self.repo.tree
|
||||
for fname in tree.walk_files(path_info):
|
||||
yield PathInfo(fname)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
@@ -187,6 +227,17 @@ class LocalRemote(BaseRemote):
|
||||
self.remove(tmp_info)
|
||||
raise
|
||||
|
||||
def copy_fobj(self, fobj, to_info):
|
||||
self.makedirs(to_info.parent)
|
||||
tmp_info = to_info.parent / tmp_fname(to_info.name)
|
||||
try:
|
||||
copy_fobj_to_file(fobj, tmp_info)
|
||||
os.chmod(tmp_info, self._file_mode)
|
||||
os.rename(tmp_info, to_info)
|
||||
except Exception:
|
||||
self.remove(tmp_info)
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def symlink(from_info, to_info):
|
||||
System.symlink(from_info, to_info)
|
||||
@@ -254,6 +305,8 @@ class LocalRemote(BaseRemote):
|
||||
copyfile(
|
||||
from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
|
||||
)
|
||||
|
||||
self.protect(tmp_file)
|
||||
os.rename(tmp_file, to_info)
|
||||
|
||||
def _download(
|
||||
@@ -647,70 +700,15 @@ class LocalRemote(BaseRemote):
|
||||
if actual != mode:
|
||||
raise
|
||||
|
||||
def _get_unpacked_dir_path_info(self, checksum):
|
||||
info = self.checksum_to_path_info(checksum)
|
||||
return info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
|
||||
|
||||
def _remove_unpacked_dir(self, checksum):
|
||||
path_info = self._get_unpacked_dir_path_info(checksum)
|
||||
info = self.checksum_to_path_info(checksum)
|
||||
path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
|
||||
self.remove(path_info)
|
||||
|
||||
def _path_info_changed(self, path_info):
|
||||
if self.exists(path_info) and self.state.get(path_info):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _update_unpacked_dir(self, checksum):
|
||||
unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
|
||||
|
||||
if not self._path_info_changed(unpacked_dir_info):
|
||||
return
|
||||
|
||||
self.remove(unpacked_dir_info)
|
||||
|
||||
try:
|
||||
dir_info = self.get_dir_cache(checksum)
|
||||
self._create_unpacked_dir(checksum, dir_info, unpacked_dir_info)
|
||||
except DvcException:
|
||||
logger.warning(f"Could not create '{unpacked_dir_info}'")
|
||||
|
||||
self.remove(unpacked_dir_info)
|
||||
|
||||
def _create_unpacked_dir(self, checksum, dir_info, unpacked_dir_info):
|
||||
self.makedirs(unpacked_dir_info)
|
||||
|
||||
for entry in Tqdm(dir_info, desc="Creating unpacked dir", unit="file"):
|
||||
entry_cache_info = self.checksum_to_path_info(
|
||||
entry[self.PARAM_CHECKSUM]
|
||||
)
|
||||
relative_path = entry[self.PARAM_RELPATH]
|
||||
# In shared cache mode some cache files might not be owned by the
|
||||
# user, so we need to use symlinks because, unless
|
||||
# /proc/sys/fs/protected_hardlinks is disabled, the user is not
|
||||
# allowed to create hardlinks to files that he doesn't own.
|
||||
link_types = ["hardlink", "symlink"]
|
||||
self._link(
|
||||
entry_cache_info, unpacked_dir_info / relative_path, link_types
|
||||
)
|
||||
|
||||
self.state.save(unpacked_dir_info, checksum)
|
||||
|
||||
def _changed_unpacked_dir(self, checksum):
|
||||
status_unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
|
||||
|
||||
return not self.state.get(status_unpacked_dir_info)
|
||||
|
||||
def _get_unpacked_dir_names(self, checksums):
|
||||
unpacked = set()
|
||||
for c in checksums:
|
||||
if self.is_dir_checksum(c):
|
||||
unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
|
||||
return unpacked
|
||||
|
||||
def is_protected(self, path_info):
|
||||
if not self.exists(path_info):
|
||||
try:
|
||||
mode = os.stat(path_info).st_mode
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
mode = os.stat(path_info).st_mode
|
||||
|
||||
return stat.S_IMODE(mode) == self.CACHE_MODE
|
||||
|
||||
@@ -265,6 +265,8 @@ class SSHRemote(BaseRemote):
|
||||
else:
|
||||
root = self.path_info.path
|
||||
with self.ssh(self.path_info) as ssh:
|
||||
if prefix and not ssh.exists(root):
|
||||
return
|
||||
# If we simply return an iterator then with above closes instantly
|
||||
if progress_callback:
|
||||
for path in ssh.walk_files(root):
|
||||
|
||||
+131
-47
@@ -1,3 +1,4 @@
|
||||
import logging
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
@@ -5,17 +6,25 @@ from functools import wraps
|
||||
from funcy import cached_property, cat, first
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.dvcfile import PIPELINE_FILE, Dvcfile, is_valid_filename
|
||||
from dvc.exceptions import FileMissingError
|
||||
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
|
||||
from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
|
||||
from dvc.exceptions import (
|
||||
NoOutputOrStageError,
|
||||
NotDvcRepoError,
|
||||
OutputNotFoundError,
|
||||
)
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo.tree import RepoTree
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
|
||||
from ..utils import parse_target
|
||||
from .graph import check_acyclic, get_pipeline, get_pipelines
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def locked(f):
|
||||
@wraps(f)
|
||||
@@ -57,8 +66,8 @@ class Repo:
|
||||
from dvc.repo.get_url import get_url
|
||||
from dvc.repo.update import update
|
||||
|
||||
def __init__(self, root_dir=None):
|
||||
from dvc.state import State
|
||||
def __init__(self, root_dir=None, scm=None, rev=None):
|
||||
from dvc.state import State, StateNoop
|
||||
from dvc.lock import make_lock
|
||||
from dvc.scm import SCM
|
||||
from dvc.cache import Cache
|
||||
@@ -70,17 +79,24 @@ class Repo:
|
||||
from dvc.utils.fs import makedirs
|
||||
from dvc.stage.cache import StageCache
|
||||
|
||||
root_dir = self.find_root(root_dir)
|
||||
if scm:
|
||||
# use GitTree instead of WorkingTree as default repo tree instance
|
||||
tree = scm.get_tree(rev)
|
||||
self.root_dir = self.find_root(root_dir, tree)
|
||||
self.scm = scm
|
||||
self.tree = tree
|
||||
self.state = StateNoop()
|
||||
else:
|
||||
root_dir = self.find_root(root_dir)
|
||||
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
|
||||
|
||||
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
|
||||
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
|
||||
|
||||
self.config = Config(self.dvc_dir)
|
||||
|
||||
no_scm = self.config["core"].get("no_scm", False)
|
||||
self.scm = SCM(self.root_dir, no_scm=no_scm)
|
||||
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
if not scm:
|
||||
no_scm = self.config["core"].get("no_scm", False)
|
||||
self.scm = SCM(self.root_dir, no_scm=no_scm)
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
|
||||
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
|
||||
self.index_dir = os.path.join(self.tmp_dir, "index")
|
||||
@@ -94,9 +110,11 @@ class Repo:
|
||||
friendly=True,
|
||||
)
|
||||
|
||||
# NOTE: storing state and link_state in the repository itself to avoid
|
||||
# any possible state corruption in 'shared cache dir' scenario.
|
||||
self.state = State(self)
|
||||
if not scm:
|
||||
# NOTE: storing state and link_state in the repository itself to
|
||||
# avoid any possible state corruption in 'shared cache dir'
|
||||
# scenario.
|
||||
self.state = State(self)
|
||||
|
||||
self.cache = Cache(self)
|
||||
self.cloud = DataCloud(self)
|
||||
@@ -124,9 +142,14 @@ class Repo:
|
||||
return f"{self.__class__.__name__}: '{self.root_dir}'"
|
||||
|
||||
@classmethod
|
||||
def find_root(cls, root=None):
|
||||
def find_root(cls, root=None, tree=None):
|
||||
root_dir = os.path.realpath(root or os.curdir)
|
||||
|
||||
if tree:
|
||||
if tree.isdir(os.path.join(root_dir, cls.DVC_DIR)):
|
||||
return root_dir
|
||||
raise NotDvcRepoError(f"'{root}' does not contain DVC directory")
|
||||
|
||||
if not os.path.isdir(root_dir):
|
||||
raise NotDvcRepoError(f"directory '{root}' does not exist")
|
||||
|
||||
@@ -167,6 +190,25 @@ class Repo:
|
||||
|
||||
self.scm.ignore_list(flist)
|
||||
|
||||
def get_stage(self, path=None, name=None):
|
||||
if not path:
|
||||
path = PIPELINE_FILE
|
||||
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
|
||||
|
||||
dvcfile = Dvcfile(self, path)
|
||||
return dvcfile.stages[name]
|
||||
|
||||
def get_stages(self, path=None, name=None):
|
||||
if not path:
|
||||
path = PIPELINE_FILE
|
||||
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
|
||||
|
||||
if name:
|
||||
return [self.get_stage(path, name)]
|
||||
|
||||
dvcfile = Dvcfile(self, path)
|
||||
return list(dvcfile.stages.values())
|
||||
|
||||
def check_modified_graph(self, new_stages):
|
||||
"""Generate graph including the new stage to check for errors"""
|
||||
# Building graph might be costly for the ones with many DVC-files,
|
||||
@@ -190,10 +232,9 @@ class Repo:
|
||||
stages = nx.dfs_postorder_nodes(graph)
|
||||
return [stage for stage in stages if path_isin(stage.path, path)]
|
||||
|
||||
def collect(self, target, with_deps=False, recursive=False, graph=None):
|
||||
import networkx as nx
|
||||
from ..dvcfile import Dvcfile
|
||||
|
||||
def collect(
|
||||
self, target=None, with_deps=False, recursive=False, graph=None
|
||||
):
|
||||
if not target:
|
||||
return list(graph) if graph else self.stages
|
||||
|
||||
@@ -203,36 +244,81 @@ class Repo:
|
||||
)
|
||||
|
||||
path, name = parse_target(target)
|
||||
dvcfile = Dvcfile(self, path)
|
||||
stages = list(dvcfile.stages.filter(name).values())
|
||||
stages = self.get_stages(path, name)
|
||||
if not with_deps:
|
||||
return stages
|
||||
|
||||
res = set()
|
||||
for stage in stages:
|
||||
pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
|
||||
res.update(nx.dfs_postorder_nodes(pipeline, stage))
|
||||
res.update(self._collect_pipeline(stage, graph=graph))
|
||||
return res
|
||||
|
||||
def collect_granular(self, target, *args, **kwargs):
|
||||
from ..dvcfile import Dvcfile, is_valid_filename
|
||||
def _collect_pipeline(self, stage, graph=None):
|
||||
import networkx as nx
|
||||
|
||||
pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
|
||||
return nx.dfs_postorder_nodes(pipeline, stage)
|
||||
|
||||
def _collect_from_default_dvcfile(self, target):
|
||||
dvcfile = Dvcfile(self, PIPELINE_FILE)
|
||||
if dvcfile.exists():
|
||||
return dvcfile.stages.get(target)
|
||||
|
||||
def collect_granular(
|
||||
self, target=None, with_deps=False, recursive=False, graph=None
|
||||
):
|
||||
"""
|
||||
Priority is in the order of following in case of ambiguity:
|
||||
- .dvc file or .yaml file
|
||||
- dir if recursive and directory exists
|
||||
- stage_name
|
||||
- output file
|
||||
"""
|
||||
if not target:
|
||||
return [(stage, None) for stage in self.stages]
|
||||
|
||||
file, name = parse_target(target)
|
||||
if is_valid_filename(file) and not kwargs.get("with_deps"):
|
||||
# Optimization: do not collect the graph for a specific .dvc target
|
||||
stages = Dvcfile(self, file).stages.filter(name)
|
||||
return [(stage, None) for stage in stages.values()]
|
||||
stages = []
|
||||
|
||||
try:
|
||||
(out,) = self.find_outs_by_path(file, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(file))
|
||||
return [(out.stage, filter_info)]
|
||||
except OutputNotFoundError:
|
||||
stages = self.collect(target, *args, **kwargs)
|
||||
return [(stage, None) for stage in stages]
|
||||
# Optimization: do not collect the graph for a specific target
|
||||
if not file:
|
||||
# parsing is ambiguous when it does not have a colon
|
||||
# or if it's not a dvcfile, as it can be a stage name
|
||||
# in `dvc.yaml` or, an output in a stage.
|
||||
logger.debug(
|
||||
"Checking if stage '%s' is in '%s'", target, PIPELINE_FILE
|
||||
)
|
||||
if not (recursive and os.path.isdir(target)):
|
||||
stage = self._collect_from_default_dvcfile(target)
|
||||
if stage:
|
||||
stages = (
|
||||
self._collect_pipeline(stage) if with_deps else [stage]
|
||||
)
|
||||
elif not with_deps and is_valid_filename(file):
|
||||
stages = self.get_stages(file, name)
|
||||
|
||||
if not stages:
|
||||
if not (recursive and os.path.isdir(target)):
|
||||
try:
|
||||
(out,) = self.find_outs_by_path(target, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(target))
|
||||
return [(out.stage, filter_info)]
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
|
||||
try:
|
||||
stages = self.collect(target, with_deps, recursive, graph)
|
||||
except StageFileDoesNotExistError as exc:
|
||||
# collect() might try to use `target` as a stage name
|
||||
# and throw error that dvc.yaml does not exist, whereas it
|
||||
# should say that both stage name and file does not exist.
|
||||
if file and is_valid_filename(file):
|
||||
raise
|
||||
raise NoOutputOrStageError(target, exc.file) from exc
|
||||
except StageNotFound as exc:
|
||||
raise NoOutputOrStageError(target, exc.file) from exc
|
||||
|
||||
return [(stage, None) for stage in stages]
|
||||
|
||||
def used_cache(
|
||||
self,
|
||||
@@ -429,16 +515,13 @@ class Repo:
|
||||
return PlotTemplates(self.dvc_dir)
|
||||
|
||||
def _collect_stages(self):
|
||||
from dvc.dvcfile import Dvcfile, is_valid_filename
|
||||
|
||||
stages = []
|
||||
outs = set()
|
||||
|
||||
for root, dirs, files in self.tree.walk(self.root_dir):
|
||||
for file_name in filter(is_valid_filename, files):
|
||||
path = os.path.join(root, file_name)
|
||||
stage_loader = Dvcfile(self, path).stages
|
||||
stages.extend(stage_loader.values())
|
||||
stages.extend(self.get_stages(path))
|
||||
outs.update(
|
||||
out.fspath
|
||||
for stage in stages
|
||||
@@ -488,17 +571,18 @@ class Repo:
|
||||
tree = RepoTree(self, stream=True)
|
||||
path = os.path.join(self.root_dir, path)
|
||||
try:
|
||||
with tree.open(
|
||||
os.path.join(self.root_dir, path),
|
||||
mode=mode,
|
||||
encoding=encoding,
|
||||
remote=remote,
|
||||
) as fobj:
|
||||
yield fobj
|
||||
with self.state:
|
||||
with tree.open(
|
||||
os.path.join(self.root_dir, path),
|
||||
mode=mode,
|
||||
encoding=encoding,
|
||||
remote=remote,
|
||||
) as fobj:
|
||||
yield fobj
|
||||
except FileNotFoundError as exc:
|
||||
raise FileMissingError(path) from exc
|
||||
except IsADirectoryError as exc:
|
||||
raise DvcIsADirectoryError from exc
|
||||
raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
|
||||
|
||||
def close(self):
|
||||
self.scm.close()
|
||||
|
||||
+14
-8
@@ -1,7 +1,11 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit
|
||||
from dvc.exceptions import (
|
||||
CheckoutError,
|
||||
CheckoutErrorSuggestGit,
|
||||
NoOutputOrStageError,
|
||||
)
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.utils import relpath
|
||||
|
||||
@@ -18,11 +22,11 @@ def _get_unused_links(repo):
|
||||
return repo.state.get_unused_links(used)
|
||||
|
||||
|
||||
def _fspath_dir(path, root):
|
||||
def _fspath_dir(path):
|
||||
if not os.path.exists(str(path)):
|
||||
return str(path)
|
||||
|
||||
path = relpath(path, root)
|
||||
path = relpath(path)
|
||||
return os.path.join(path, "") if os.path.isdir(path) else path
|
||||
|
||||
|
||||
@@ -56,7 +60,7 @@ def _checkout(
|
||||
targets = [None]
|
||||
unused = _get_unused_links(self)
|
||||
|
||||
stats["deleted"] = [_fspath_dir(u, self.root_dir) for u in unused]
|
||||
stats["deleted"] = [_fspath_dir(u) for u in unused]
|
||||
self.state.remove_links(unused)
|
||||
|
||||
if isinstance(targets, str):
|
||||
@@ -70,7 +74,11 @@ def _checkout(
|
||||
target, with_deps=with_deps, recursive=recursive
|
||||
)
|
||||
)
|
||||
except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
|
||||
except (
|
||||
StageFileDoesNotExistError,
|
||||
StageFileBadNameError,
|
||||
NoOutputOrStageError,
|
||||
) as exc:
|
||||
if not target:
|
||||
raise
|
||||
raise CheckoutErrorSuggestGit(target) from exc
|
||||
@@ -87,9 +95,7 @@ def _checkout(
|
||||
filter_info=filter_info,
|
||||
)
|
||||
for key, items in result.items():
|
||||
stats[key].extend(
|
||||
_fspath_dir(path, self.root_dir) for path in items
|
||||
)
|
||||
stats[key].extend(_fspath_dir(path) for path in items)
|
||||
|
||||
if stats.get("failed"):
|
||||
raise CheckoutError(stats["failed"], stats)
|
||||
|
||||
@@ -46,3 +46,4 @@ def commit(self, target, with_deps=False, recursive=False, force=False):
|
||||
stage.commit()
|
||||
|
||||
Dvcfile(self, stage.path).dump(stage)
|
||||
return stages
|
||||
|
||||
+6
-52
@@ -1,9 +1,7 @@
|
||||
import logging
|
||||
|
||||
from dvc.cache import NamedCache
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.exceptions import DownloadError, OutputNotFoundError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.exceptions import DownloadError
|
||||
from dvc.scm.base import CloneError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -78,35 +76,15 @@ def _fetch(
|
||||
|
||||
|
||||
def _fetch_external(self, repo_url, repo_rev, files, jobs):
|
||||
from dvc.external_repo import external_repo, ExternalRepo
|
||||
from dvc.external_repo import external_repo
|
||||
|
||||
failed, downloaded = 0, 0
|
||||
try:
|
||||
with external_repo(repo_url, repo_rev) as repo:
|
||||
is_dvc_repo = isinstance(repo, ExternalRepo)
|
||||
# gather git-only tracked files if dvc repo
|
||||
git_files = [] if is_dvc_repo else files
|
||||
if is_dvc_repo:
|
||||
repo.cache.local.cache_dir = self.cache.local.cache_dir
|
||||
with repo.state:
|
||||
cache = NamedCache()
|
||||
for name in files:
|
||||
try:
|
||||
out = repo.find_out_by_relpath(name)
|
||||
except OutputNotFoundError:
|
||||
# try to add to cache if they are git-tracked files
|
||||
git_files.append(name)
|
||||
else:
|
||||
cache.update(out.get_used_cache())
|
||||
|
||||
try:
|
||||
downloaded += repo.cloud.pull(cache, jobs=jobs)
|
||||
except DownloadError as exc:
|
||||
failed += exc.amount
|
||||
|
||||
d, f = _git_to_cache(self.cache.local, repo.root_dir, git_files)
|
||||
downloaded += d
|
||||
failed += f
|
||||
with repo.use_cache(self.cache.local):
|
||||
d, f, _ = repo.fetch_external(files, jobs=jobs)
|
||||
downloaded += d
|
||||
failed += f
|
||||
except CloneError:
|
||||
failed += 1
|
||||
logger.exception(
|
||||
@@ -114,27 +92,3 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs):
|
||||
)
|
||||
|
||||
return downloaded, failed
|
||||
|
||||
|
||||
def _git_to_cache(cache, repo_root, files):
|
||||
"""Save files from a git repo directly to the cache."""
|
||||
failed = set()
|
||||
num_downloads = 0
|
||||
repo_root = PathInfo(repo_root)
|
||||
for file in files:
|
||||
info = cache.save_info(repo_root / file)
|
||||
if info.get(cache.PARAM_CHECKSUM) is None:
|
||||
failed.add(file)
|
||||
continue
|
||||
|
||||
if cache.changed_cache(info[cache.PARAM_CHECKSUM]):
|
||||
logger.debug("fetched '%s' from '%s' repo", file, repo_root)
|
||||
num_downloads += 1
|
||||
cache.save(repo_root / file, info, save_link=False)
|
||||
|
||||
if failed:
|
||||
logger.exception(
|
||||
"failed to fetch data for {}".format(", ".join(failed))
|
||||
)
|
||||
|
||||
return num_downloads, len(failed)
|
||||
|
||||
+1
-3
@@ -4,7 +4,6 @@ import os
|
||||
import shortuuid
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.utils import resolve_output
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
@@ -51,7 +50,6 @@ def get(url, path, out=None, rev=None):
|
||||
# Also, we can't use theoretical "move" link type here, because
|
||||
# the same cache file might be used a few times in a directory.
|
||||
repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]
|
||||
|
||||
repo.pull_to(path, PathInfo(out))
|
||||
repo.get_external(path, out)
|
||||
finally:
|
||||
remove(tmp_dir)
|
||||
|
||||
+2
-4
@@ -3,13 +3,11 @@ from . import locked
|
||||
|
||||
@locked
|
||||
def lock(self, target, unlock=False):
|
||||
from .. import dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
dvcfile = dvcfile.Dvcfile(self, path)
|
||||
stage = dvcfile.stages[name]
|
||||
stage = self.get_stage(path, name)
|
||||
stage.locked = False if unlock else True
|
||||
dvcfile.dump(stage, update_pipeline=True)
|
||||
stage.dvcfile.dump(stage, update_pipeline=True)
|
||||
|
||||
return stage
|
||||
|
||||
+37
-40
@@ -4,7 +4,7 @@ from dvc.path_info import PathInfo
|
||||
|
||||
@staticmethod
|
||||
def ls(
|
||||
url, path=None, rev=None, recursive=None, outs_only=False,
|
||||
url, path=None, rev=None, recursive=None, dvc_only=False,
|
||||
):
|
||||
"""Methods for getting files and outputs for the repo.
|
||||
|
||||
@@ -13,7 +13,7 @@ def ls(
|
||||
path (str, optional): relative path into the repo
|
||||
rev (str, optional): SHA commit, branch or tag name
|
||||
recursive (bool, optional): recursively walk the repo
|
||||
outs_only (bool, optional): show only DVC-artifacts
|
||||
dvc_only (bool, optional): show only DVC-artifacts
|
||||
|
||||
Returns:
|
||||
list of `entry`
|
||||
@@ -28,25 +28,16 @@ def ls(
|
||||
}
|
||||
"""
|
||||
from dvc.external_repo import external_repo
|
||||
from dvc.repo import Repo
|
||||
|
||||
with external_repo(url, rev) as repo:
|
||||
path_info = PathInfo(repo.root_dir)
|
||||
if path:
|
||||
path_info /= path
|
||||
|
||||
ret = {}
|
||||
if isinstance(repo, Repo):
|
||||
ret = _ls(repo, path_info, recursive, True)
|
||||
|
||||
nondvc = {}
|
||||
if not outs_only:
|
||||
nondvc = _ls(repo, path_info, recursive, False)
|
||||
|
||||
ret.update(nondvc)
|
||||
ret = _ls(repo, path_info, recursive, dvc_only)
|
||||
|
||||
if path and not ret:
|
||||
raise PathMissingError(path, repo, output_only=outs_only)
|
||||
raise PathMissingError(path, repo, dvc_only=dvc_only)
|
||||
|
||||
ret_list = []
|
||||
for path, info in ret.items():
|
||||
@@ -56,46 +47,52 @@ def ls(
|
||||
return ret_list
|
||||
|
||||
|
||||
def _ls(repo, path_info, recursive=None, dvc=False):
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.repo.tree import DvcTree
|
||||
from dvc.scm.tree import WorkingTree
|
||||
def _ls(repo, path_info, recursive=None, dvc_only=False):
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
if dvc:
|
||||
tree = DvcTree(repo)
|
||||
else:
|
||||
tree = CleanTree(WorkingTree(repo.root_dir))
|
||||
# use our own RepoTree instance instead of repo.repo_tree since we do not
|
||||
# want fetch/stream enabled for ls
|
||||
tree = RepoTree(repo)
|
||||
|
||||
ret = {}
|
||||
try:
|
||||
for root, dirs, files in tree.walk(path_info.fspath):
|
||||
for root, dirs, files in tree.walk(path_info.fspath, dvcfiles=True):
|
||||
for fname in files:
|
||||
info = PathInfo(root) / fname
|
||||
path = str(info.relative_to(path_info))
|
||||
ret[path] = {
|
||||
"isout": dvc,
|
||||
"isdir": False,
|
||||
"isexec": False if dvc else tree.isexec(info.fspath),
|
||||
}
|
||||
dvc = tree.isdvc(info)
|
||||
if dvc or not dvc_only:
|
||||
path = str(info.relative_to(path_info))
|
||||
ret[path] = {
|
||||
"isout": dvc,
|
||||
"isdir": False,
|
||||
"isexec": False if dvc else tree.isexec(info),
|
||||
}
|
||||
|
||||
if not recursive:
|
||||
for dname in dirs:
|
||||
info = PathInfo(root) / dname
|
||||
path = str(info.relative_to(path_info))
|
||||
ret[path] = {
|
||||
"isout": tree.isdvc(info.fspath) if dvc else False,
|
||||
"isdir": True,
|
||||
"isexec": False if dvc else tree.isexec(info.fspath),
|
||||
}
|
||||
if not dvc_only or (
|
||||
tree.dvctree and tree.dvctree.exists(info)
|
||||
):
|
||||
dvc = tree.isdvc(info)
|
||||
path = str(info.relative_to(path_info))
|
||||
ret[path] = {
|
||||
"isout": dvc,
|
||||
"isdir": True,
|
||||
"isexec": False if dvc else tree.isexec(info),
|
||||
}
|
||||
break
|
||||
except NotADirectoryError:
|
||||
return {
|
||||
path_info.name: {
|
||||
"isout": dvc,
|
||||
"isdir": False,
|
||||
"isexec": False if dvc else tree.isexec(path_info.fspath),
|
||||
dvc = tree.isdvc(path_info)
|
||||
if dvc or not dvc_only:
|
||||
return {
|
||||
path_info.name: {
|
||||
"isout": dvc,
|
||||
"isdir": False,
|
||||
"isexec": False if dvc else tree.isexec(path_info),
|
||||
}
|
||||
}
|
||||
}
|
||||
return {}
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
|
||||
|
||||
+5
-6
@@ -7,16 +7,15 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@locked
|
||||
def remove(self, target, outs_only=False):
|
||||
from ..dvcfile import Dvcfile
|
||||
def remove(self, target, dvc_only=False):
|
||||
from ..dvcfile import Dvcfile, is_valid_filename
|
||||
|
||||
path, name = parse_target(target)
|
||||
dvcfile = Dvcfile(self, path)
|
||||
stages = list(dvcfile.stages.filter(name).values())
|
||||
stages = self.get_stages(path, name)
|
||||
for stage in stages:
|
||||
stage.remove_outs(force=True)
|
||||
|
||||
if not outs_only:
|
||||
dvcfile.remove()
|
||||
if path and is_valid_filename(path) and not dvc_only:
|
||||
Dvcfile(self, path).remove()
|
||||
|
||||
return stages
|
||||
|
||||
@@ -61,7 +61,6 @@ def reproduce(
|
||||
all_pipelines=False,
|
||||
**kwargs
|
||||
):
|
||||
from ..dvcfile import Dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
if not target and not all_pipelines:
|
||||
@@ -81,8 +80,7 @@ def reproduce(
|
||||
if all_pipelines:
|
||||
pipelines = active_pipelines
|
||||
else:
|
||||
dvcfile = Dvcfile(self, path)
|
||||
stage = dvcfile.stages[name]
|
||||
stage = self.get_stage(path, name)
|
||||
pipelines = [get_pipeline(active_pipelines, stage)]
|
||||
|
||||
targets = []
|
||||
@@ -164,7 +162,12 @@ def _reproduce_stages(
|
||||
|
||||
force_downstream = kwargs.pop("force_downstream", False)
|
||||
result = []
|
||||
# `ret` is used to add a cosmetic newline.
|
||||
ret = []
|
||||
for stage in pipeline:
|
||||
if ret:
|
||||
logger.info("")
|
||||
|
||||
try:
|
||||
ret = _reproduce_stage(stage, **kwargs)
|
||||
|
||||
|
||||
+5
-3
@@ -45,6 +45,9 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
|
||||
from dvc.stage import PipelineStage, Stage, create_stage
|
||||
from dvc.dvcfile import Dvcfile, PIPELINE_FILE
|
||||
|
||||
if not kwargs.get("cmd"):
|
||||
raise InvalidArgumentError("command is not specified")
|
||||
|
||||
stage_cls = PipelineStage
|
||||
path = PIPELINE_FILE
|
||||
stage_name = kwargs.get("name")
|
||||
@@ -82,13 +85,12 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
|
||||
raise OutputDuplicationError(exc.output, set(exc.stages) - {stage})
|
||||
|
||||
if no_exec:
|
||||
for out in stage.outs:
|
||||
out.ignore()
|
||||
stage.ignore_outs()
|
||||
else:
|
||||
stage.run(
|
||||
no_commit=kwargs.get("no_commit", False),
|
||||
run_cache=kwargs.get("run_cache", True),
|
||||
)
|
||||
|
||||
dvcfile.dump(stage, update_pipeline=True)
|
||||
dvcfile.dump(stage, update_pipeline=True, no_lock=no_exec)
|
||||
return stage
|
||||
|
||||
+175
-51
@@ -1,10 +1,13 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.dvcfile import is_valid_filename
|
||||
from dvc.exceptions import OutputNotFoundError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.remote.base import RemoteActionNotImplemented
|
||||
from dvc.scm.tree import BaseTree, WorkingTree
|
||||
from dvc.scm.tree import BaseTree
|
||||
from dvc.utils import file_md5
|
||||
from dvc.utils.fs import copy_fobj_to_file, makedirs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -40,43 +43,56 @@ class DvcTree(BaseTree):
|
||||
|
||||
return outs
|
||||
|
||||
def _get_granular_checksum(self, path, out, remote=None):
|
||||
if not self.fetch and not self.stream:
|
||||
raise FileNotFoundError
|
||||
dir_cache = out.get_dir_cache(remote=remote)
|
||||
for entry in dir_cache:
|
||||
entry_relpath = entry[out.remote.PARAM_RELPATH]
|
||||
if os.name == "nt":
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
if path == out.path_info / entry_relpath:
|
||||
return entry[out.remote.PARAM_CHECKSUM]
|
||||
raise FileNotFoundError
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8", remote=None):
|
||||
try:
|
||||
outs = self._find_outs(path, strict=False)
|
||||
except OutputNotFoundError as exc:
|
||||
raise FileNotFoundError from exc
|
||||
|
||||
if len(outs) != 1 or outs[0].is_dir_checksum:
|
||||
if len(outs) != 1 or (
|
||||
outs[0].is_dir_checksum and path == outs[0].path_info
|
||||
):
|
||||
raise IsADirectoryError
|
||||
|
||||
out = outs[0]
|
||||
# temporary hack to make cache use WorkingTree and not GitTree, because
|
||||
# cache dir doesn't exist in the latter.
|
||||
saved_tree = self.repo.tree
|
||||
self.repo.tree = WorkingTree(self.repo.root_dir)
|
||||
try:
|
||||
if out.changed_cache():
|
||||
if not self.fetch and not self.stream:
|
||||
raise FileNotFoundError
|
||||
if out.changed_cache(filter_info=path):
|
||||
if not self.fetch and not self.stream:
|
||||
raise FileNotFoundError
|
||||
|
||||
remote_obj = self.repo.cloud.get_remote(remote)
|
||||
if self.stream:
|
||||
try:
|
||||
remote_info = remote_obj.checksum_to_path_info(
|
||||
out.checksum
|
||||
)
|
||||
return remote_obj.open(
|
||||
remote_info, mode=mode, encoding=encoding
|
||||
)
|
||||
except RemoteActionNotImplemented:
|
||||
pass
|
||||
with self.repo.state:
|
||||
cache_info = out.get_used_cache(remote=remote)
|
||||
self.repo.cloud.pull(cache_info, remote=remote)
|
||||
finally:
|
||||
self.repo.tree = saved_tree
|
||||
remote_obj = self.repo.cloud.get_remote(remote)
|
||||
if self.stream:
|
||||
if out.is_dir_checksum:
|
||||
checksum = self._get_granular_checksum(path, out)
|
||||
else:
|
||||
checksum = out.checksum
|
||||
try:
|
||||
remote_info = remote_obj.checksum_to_path_info(checksum)
|
||||
return remote_obj.open(
|
||||
remote_info, mode=mode, encoding=encoding
|
||||
)
|
||||
except RemoteActionNotImplemented:
|
||||
pass
|
||||
cache_info = out.get_used_cache(filter_info=path, remote=remote)
|
||||
self.repo.cloud.pull(cache_info, remote=remote)
|
||||
|
||||
return open(out.cache_path, mode=mode, encoding=encoding)
|
||||
if out.is_dir_checksum:
|
||||
checksum = self._get_granular_checksum(path, out)
|
||||
cache_path = out.cache.checksum_to_path_info(checksum).url
|
||||
else:
|
||||
cache_path = out.cache_path
|
||||
return open(cache_path, mode=mode, encoding=encoding)
|
||||
|
||||
def exists(self, path):
|
||||
try:
|
||||
@@ -91,10 +107,25 @@ class DvcTree(BaseTree):
|
||||
|
||||
path_info = PathInfo(os.path.abspath(path))
|
||||
outs = self._find_outs(path, strict=False, recursive=True)
|
||||
if len(outs) != 1 or outs[0].path_info != path_info:
|
||||
if len(outs) != 1:
|
||||
return True
|
||||
|
||||
return outs[0].is_dir_checksum
|
||||
out = outs[0]
|
||||
if not out.is_dir_checksum:
|
||||
if out.path_info != path_info:
|
||||
return True
|
||||
return False
|
||||
|
||||
if out.path_info == path_info:
|
||||
return True
|
||||
|
||||
# for dir checksum, we need to check if this is a file inside the
|
||||
# directory
|
||||
try:
|
||||
self._get_granular_checksum(path, out)
|
||||
return False
|
||||
except FileNotFoundError:
|
||||
return True
|
||||
|
||||
def isfile(self, path):
|
||||
if not self.exists(path):
|
||||
@@ -127,7 +158,7 @@ class DvcTree(BaseTree):
|
||||
else:
|
||||
assert False
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
def walk(self, top, topdown=True, download_callback=None, **kwargs):
|
||||
from pygtrie import Trie
|
||||
|
||||
assert topdown
|
||||
@@ -147,19 +178,29 @@ class DvcTree(BaseTree):
|
||||
trie[out.path_info.parts] = out
|
||||
|
||||
if out.is_dir_checksum and (self.fetch or self.stream):
|
||||
# will pull dir cache if needed
|
||||
with self.repo.state:
|
||||
cache = out.collect_used_dir_cache()
|
||||
for _, names in cache.scheme_names(out.scheme):
|
||||
for name in names:
|
||||
path_info = out.path_info.parent / name
|
||||
trie[path_info.parts] = None
|
||||
# pull dir cache if needed
|
||||
dir_cache = out.get_dir_cache(**kwargs)
|
||||
|
||||
# pull dir contents if needed
|
||||
if self.fetch:
|
||||
if out.changed_cache(filter_info=top):
|
||||
used_cache = out.get_used_cache(filter_info=top)
|
||||
downloaded = self.repo.cloud.pull(used_cache, **kwargs)
|
||||
if download_callback:
|
||||
download_callback(downloaded)
|
||||
|
||||
for entry in dir_cache:
|
||||
entry_relpath = entry[out.remote.PARAM_RELPATH]
|
||||
if os.name == "nt":
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
path_info = out.path_info / entry_relpath
|
||||
trie[path_info.parts] = None
|
||||
|
||||
yield from self._walk(root, trie, topdown=topdown)
|
||||
|
||||
def isdvc(self, path):
|
||||
def isdvc(self, path, **kwargs):
|
||||
try:
|
||||
return len(self._find_outs(path)) == 1
|
||||
return len(self._find_outs(path, **kwargs)) == 1
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
return False
|
||||
@@ -167,6 +208,15 @@ class DvcTree(BaseTree):
|
||||
def isexec(self, path):
|
||||
return False
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
outs = self._find_outs(path_info, strict=False)
|
||||
if len(outs) != 1:
|
||||
raise OutputNotFoundError
|
||||
out = outs[0]
|
||||
if out.is_dir_checksum:
|
||||
return self._get_granular_checksum(path_info, out)
|
||||
return out.checksum
|
||||
|
||||
|
||||
class RepoTree(BaseTree):
|
||||
"""DVC + git-tracked files tree.
|
||||
@@ -185,7 +235,22 @@ class RepoTree(BaseTree):
|
||||
# git-only erepo's do not need dvctree
|
||||
self.dvctree = None
|
||||
|
||||
@property
|
||||
def fetch(self):
|
||||
if self.dvctree:
|
||||
return self.dvctree.fetch
|
||||
return False
|
||||
|
||||
@property
|
||||
def stream(self):
|
||||
if self.dvctree:
|
||||
return self.dvctree.stream
|
||||
return False
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8", **kwargs):
|
||||
if "b" in mode:
|
||||
encoding = None
|
||||
|
||||
if self.dvctree and self.dvctree.exists(path):
|
||||
try:
|
||||
return self.dvctree.open(
|
||||
@@ -206,8 +271,8 @@ class RepoTree(BaseTree):
|
||||
self.dvctree and self.dvctree.isdir(path)
|
||||
)
|
||||
|
||||
def isdvc(self, path):
|
||||
return self.dvctree is not None and self.dvctree.isdvc(path)
|
||||
def isdvc(self, path, **kwargs):
|
||||
return self.dvctree is not None and self.dvctree.isdvc(path, **kwargs)
|
||||
|
||||
def isfile(self, path):
|
||||
return self.repo.tree.isfile(path) or (
|
||||
@@ -219,6 +284,9 @@ class RepoTree(BaseTree):
|
||||
return self.dvctree.isexec(path)
|
||||
return self.repo.tree.isexec(path)
|
||||
|
||||
def stat(self, path):
|
||||
return self.repo.tree.stat(path)
|
||||
|
||||
def _walk_one(self, walk):
|
||||
try:
|
||||
root, dirs, files = next(walk)
|
||||
@@ -228,7 +296,7 @@ class RepoTree(BaseTree):
|
||||
for _ in dirs:
|
||||
yield from self._walk_one(walk)
|
||||
|
||||
def _walk(self, dvc_walk, repo_walk):
|
||||
def _walk(self, dvc_walk, repo_walk, dvcfiles=False):
|
||||
try:
|
||||
_, dvc_dirs, dvc_fnames = next(dvc_walk)
|
||||
repo_root, repo_dirs, repo_fnames = next(repo_walk)
|
||||
@@ -244,9 +312,11 @@ class RepoTree(BaseTree):
|
||||
dirs = shared + dvc_only + repo_only
|
||||
|
||||
# merge file lists
|
||||
files = set(dvc_fnames)
|
||||
for filename in repo_fnames:
|
||||
files.add(filename)
|
||||
files = {
|
||||
fname
|
||||
for fname in dvc_fnames + repo_fnames
|
||||
if dvcfiles or not is_valid_filename(fname)
|
||||
}
|
||||
|
||||
yield repo_root, dirs, list(files)
|
||||
|
||||
@@ -257,14 +327,24 @@ class RepoTree(BaseTree):
|
||||
|
||||
for dirname in dirs:
|
||||
if dirname in shared:
|
||||
yield from self._walk(dvc_walk, repo_walk)
|
||||
yield from self._walk(dvc_walk, repo_walk, dvcfiles=dvcfiles)
|
||||
elif dirname in dvc_set:
|
||||
yield from self._walk_one(dvc_walk)
|
||||
elif dirname in repo_set:
|
||||
yield from self._walk_one(repo_walk)
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
"""Walk and merge both DVC and repo trees."""
|
||||
def walk(self, top, topdown=True, dvcfiles=False, **kwargs):
|
||||
"""Walk and merge both DVC and repo trees.
|
||||
|
||||
Args:
|
||||
top: path to walk from
|
||||
topdown: if True, tree will be walked from top down.
|
||||
dvcfiles: if True, dvcfiles will be included in the files list
|
||||
for walked directories.
|
||||
|
||||
Any kwargs will be passed into methods used for fetching and/or
|
||||
streaming DVC outs from remotes.
|
||||
"""
|
||||
assert topdown
|
||||
|
||||
if not self.exists(top):
|
||||
@@ -276,7 +356,7 @@ class RepoTree(BaseTree):
|
||||
dvc_exists = self.dvctree and self.dvctree.exists(top)
|
||||
repo_exists = self.repo.tree.exists(top)
|
||||
if dvc_exists and not repo_exists:
|
||||
yield from self.dvctree.walk(top, topdown=topdown)
|
||||
yield from self.dvctree.walk(top, topdown=topdown, **kwargs)
|
||||
return
|
||||
if repo_exists and not dvc_exists:
|
||||
yield from self.repo.tree.walk(top, topdown=topdown)
|
||||
@@ -284,6 +364,50 @@ class RepoTree(BaseTree):
|
||||
if not dvc_exists and not repo_exists:
|
||||
raise FileNotFoundError
|
||||
|
||||
dvc_walk = self.dvctree.walk(top, topdown=topdown)
|
||||
dvc_walk = self.dvctree.walk(top, topdown=topdown, **kwargs)
|
||||
repo_walk = self.repo.tree.walk(top, topdown=topdown)
|
||||
yield from self._walk(dvc_walk, repo_walk)
|
||||
yield from self._walk(dvc_walk, repo_walk, dvcfiles=dvcfiles)
|
||||
|
||||
def walk_files(self, top, **kwargs):
|
||||
for root, _, files in self.walk(top, **kwargs):
|
||||
for fname in files:
|
||||
yield PathInfo(root) / fname
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
"""Return file checksum for specified path.
|
||||
|
||||
If path_info is a DVC out, the pre-computed checksum for the file
|
||||
will be used. If path_info is a git file, MD5 will be computed for
|
||||
the git object.
|
||||
"""
|
||||
if not self.exists(path_info):
|
||||
raise FileNotFoundError
|
||||
if self.dvctree and self.dvctree.exists(path_info):
|
||||
try:
|
||||
return self.dvctree.get_file_checksum(path_info)
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
return file_md5(path_info, self)[0]
|
||||
|
||||
def copytree(self, top, dest):
|
||||
top = PathInfo(top)
|
||||
dest = PathInfo(dest)
|
||||
|
||||
if not self.exists(top):
|
||||
raise FileNotFoundError
|
||||
|
||||
if self.isfile(top):
|
||||
makedirs(dest.parent, exist_ok=True)
|
||||
with self.open(top, mode="rb") as fobj:
|
||||
copy_fobj_to_file(fobj, dest)
|
||||
return
|
||||
|
||||
for root, _, files in self.walk(top):
|
||||
root = PathInfo(root)
|
||||
dest_dir = root.relative_to(top)
|
||||
makedirs(dest_dir, exist_ok=True)
|
||||
for fname in files:
|
||||
src = root / fname
|
||||
dest = dest_dir / fname
|
||||
with self.open(src, mode="rb") as fobj:
|
||||
copy_fobj_to_file(fobj, dest)
|
||||
|
||||
+2
-1
@@ -32,7 +32,8 @@ class BaseTree:
|
||||
def walk_files(self, top):
|
||||
for root, _, files in self.walk(top):
|
||||
for file in files:
|
||||
yield os.path.join(root, file)
|
||||
# NOTE: os.path.join is ~5.5 times slower
|
||||
yield f"{root}{os.sep}{file}"
|
||||
|
||||
|
||||
class WorkingTree(BaseTree):
|
||||
|
||||
+13
-5
@@ -69,14 +69,22 @@ def _serialize_params(params: List[ParamsDependency]):
|
||||
for param_dep in sort_by_path(params):
|
||||
dump = param_dep.dumpd()
|
||||
path, params = dump[PARAM_PATH], dump[PARAM_PARAMS]
|
||||
k = list(params.keys())
|
||||
if not k:
|
||||
continue
|
||||
key_vals[path] = OrderedDict([(key, params[key]) for key in sorted(k)])
|
||||
if isinstance(params, dict):
|
||||
k = list(params.keys())
|
||||
if not k:
|
||||
continue
|
||||
key_vals[path] = OrderedDict(
|
||||
[(key, params[key]) for key in sorted(k)]
|
||||
)
|
||||
else:
|
||||
assert isinstance(params, list)
|
||||
k = params
|
||||
key_vals = OrderedDict()
|
||||
# params from default file is always kept at the start of the `params:`
|
||||
if path == DEFAULT_PARAMS_FILE:
|
||||
keys = k + keys
|
||||
key_vals.move_to_end(path, last=False)
|
||||
if key_vals:
|
||||
key_vals.move_to_end(path, last=False)
|
||||
else:
|
||||
# if it's not a default file, change the shape
|
||||
# to: {path: k}
|
||||
|
||||
+29
-32
@@ -77,7 +77,7 @@ def create_stage(cls, repo, path, **kwargs):
|
||||
logger.warning("Build cache is ignored when persisting outputs.")
|
||||
|
||||
if not ignore_run_cache and stage.can_be_skipped:
|
||||
logger.info("Stage is cached, skipping.")
|
||||
logger.info("Stage is cached, skipping")
|
||||
return None
|
||||
|
||||
return stage
|
||||
@@ -144,14 +144,10 @@ class Stage(params.StageParams):
|
||||
self._dvcfile = dvcfile
|
||||
|
||||
def __repr__(self):
|
||||
return "Stage: '{path}'".format(
|
||||
path=self.path_in_repo if self.path else "No path"
|
||||
)
|
||||
return f"Stage: '{self.addressing}'"
|
||||
|
||||
def __str__(self):
|
||||
return "stage: '{path}'".format(
|
||||
path=self.relpath if self.path else "No path"
|
||||
)
|
||||
return f"stage: '{self.addressing}'"
|
||||
|
||||
@property
|
||||
def addressing(self):
|
||||
@@ -159,7 +155,7 @@ class Stage(params.StageParams):
|
||||
Useful for alternative presentations where we don't need
|
||||
`Stage:` prefix.
|
||||
"""
|
||||
return self.relpath
|
||||
return self.relpath if self.path else "No path"
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.path_in_repo)
|
||||
@@ -209,10 +205,11 @@ class Stage(params.StageParams):
|
||||
return False
|
||||
|
||||
if self.is_callback:
|
||||
logger.warning(
|
||||
'{stage} is a "callback" stage '
|
||||
logger.debug(
|
||||
'%s is a "callback" stage '
|
||||
"(has a command and no dependencies) and thus always "
|
||||
"considered as changed.".format(stage=self)
|
||||
"considered as changed.",
|
||||
self,
|
||||
)
|
||||
return True
|
||||
|
||||
@@ -248,9 +245,9 @@ class Stage(params.StageParams):
|
||||
|
||||
return False
|
||||
|
||||
def changed_stage(self, warn=False):
|
||||
def changed_stage(self):
|
||||
changed = self.md5 != self.compute_md5()
|
||||
if changed and warn:
|
||||
if changed:
|
||||
logger.debug(self._changed_stage_entry())
|
||||
return changed
|
||||
|
||||
@@ -259,14 +256,12 @@ class Stage(params.StageParams):
|
||||
is_changed = (
|
||||
# Short-circuit order: stage md5 is fast,
|
||||
# deps are expected to change
|
||||
self.changed_stage(warn=True)
|
||||
self.changed_stage()
|
||||
or self.changed_deps()
|
||||
or self.changed_outs()
|
||||
)
|
||||
if is_changed:
|
||||
logger.info("%s changed.", self)
|
||||
else:
|
||||
logger.info("%s didn't change.", self)
|
||||
logger.debug("%s changed.", self)
|
||||
return is_changed
|
||||
|
||||
@rwlocked(write=["outs"])
|
||||
@@ -294,8 +289,8 @@ class Stage(params.StageParams):
|
||||
|
||||
@rwlocked(read=["deps"], write=["outs"])
|
||||
def reproduce(self, interactive=False, **kwargs):
|
||||
|
||||
if not (kwargs.get("force", False) or self.changed()):
|
||||
logger.info("Stage '%s' didn't change, skipping", self.addressing)
|
||||
return None
|
||||
|
||||
msg = (
|
||||
@@ -356,7 +351,11 @@ class Stage(params.StageParams):
|
||||
return get_dump(self)
|
||||
|
||||
def compute_md5(self):
|
||||
m = compute_md5(self)
|
||||
# `dvc add`ed files don't need stage md5
|
||||
if self.is_data_source and not (self.is_import or self.is_repo_import):
|
||||
m = None
|
||||
else:
|
||||
m = compute_md5(self)
|
||||
logger.debug(f"Computed {self} md5: '{m}'")
|
||||
return m
|
||||
|
||||
@@ -375,6 +374,10 @@ class Stage(params.StageParams):
|
||||
for out in self.outs:
|
||||
out.save()
|
||||
|
||||
def ignore_outs(self):
|
||||
for out in self.outs:
|
||||
out.ignore()
|
||||
|
||||
@staticmethod
|
||||
def _changed_entries(entries):
|
||||
return [str(entry) for entry in entries if entry.changed_checksum()]
|
||||
@@ -523,19 +526,13 @@ class PipelineStage(Stage):
|
||||
def __hash__(self):
|
||||
return hash((self.path_in_repo, self.name))
|
||||
|
||||
def __repr__(self):
|
||||
return "Stage: '{path}:{name}'".format(
|
||||
path=self.relpath if self.path else "No path", name=self.name
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
return "stage: '{path}:{name}'".format(
|
||||
path=self.relpath if self.path else "No path", name=self.name
|
||||
)
|
||||
|
||||
@property
|
||||
def addressing(self):
|
||||
return super().addressing + ":" + self.name
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
|
||||
if self.path and self.relpath == PIPELINE_FILE:
|
||||
return self.name
|
||||
return f"{super().addressing}:{self.name}"
|
||||
|
||||
def reload(self):
|
||||
return self.dvcfile.stages[self.name]
|
||||
@@ -548,8 +545,8 @@ class PipelineStage(Stage):
|
||||
if self.cmd_changed:
|
||||
ret.append("changed command")
|
||||
|
||||
def changed_stage(self, warn=False):
|
||||
if self.cmd_changed and warn:
|
||||
def changed_stage(self):
|
||||
if self.cmd_changed:
|
||||
logger.debug(self._changed_stage_entry())
|
||||
return self.cmd_changed
|
||||
|
||||
|
||||
+18
-3
@@ -77,15 +77,30 @@ class StageCache:
|
||||
|
||||
return None
|
||||
|
||||
def _create_stage(self, cache):
|
||||
def _create_stage(self, cache, wdir=None):
|
||||
from dvc.stage import create_stage, PipelineStage
|
||||
|
||||
params = []
|
||||
for param in cache.get("params", []):
|
||||
if isinstance(param, str):
|
||||
params.append(param)
|
||||
continue
|
||||
|
||||
assert isinstance(param, dict)
|
||||
assert len(param) == 1
|
||||
path = list(param.keys())[0]
|
||||
params_list = param[path]
|
||||
assert isinstance(params_list, list)
|
||||
params.append(f"{path}:" + ",".join(params_list))
|
||||
|
||||
stage = create_stage(
|
||||
PipelineStage,
|
||||
repo=self.repo,
|
||||
path="dvc.yaml",
|
||||
cmd=cache["cmd"],
|
||||
deps=[dep["path"] for dep in cache["deps"]],
|
||||
wdir=wdir,
|
||||
params=params,
|
||||
deps=[dep["path"] for dep in cache.get("deps", [])],
|
||||
outs=[out["path"] for out in cache["outs"]],
|
||||
)
|
||||
StageLoader.fill_from_lock(stage, cache)
|
||||
@@ -104,7 +119,7 @@ class StageCache:
|
||||
# NOTE: using temporary stage to avoid accidentally modifying original
|
||||
# stage and to workaround `commit/checkout` not working for uncached
|
||||
# outputs.
|
||||
cached_stage = self._create_stage(cache)
|
||||
cached_stage = self._create_stage(cache, wdir=stage.wdir)
|
||||
|
||||
outs_no_cache = [
|
||||
out.def_path for out in stage.outs if not out.use_cache
|
||||
|
||||
+5
-10
@@ -15,15 +15,8 @@ class StageFileFormatError(DvcException):
|
||||
|
||||
class StageFileDoesNotExistError(DvcException):
|
||||
def __init__(self, fname):
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX, is_dvc_file
|
||||
|
||||
msg = f"'{fname}' does not exist."
|
||||
|
||||
sname = fname + DVC_FILE_SUFFIX
|
||||
if is_dvc_file(sname):
|
||||
msg += f" Do you mean '{sname}'?"
|
||||
|
||||
super().__init__(msg)
|
||||
self.file = fname
|
||||
super().__init__(f"'{self.file}' does not exist.")
|
||||
|
||||
|
||||
class StageFileAlreadyExistsError(DvcException):
|
||||
@@ -87,8 +80,10 @@ class MissingDataSource(DvcException):
|
||||
|
||||
class StageNotFound(KeyError, DvcException):
|
||||
def __init__(self, file, name):
|
||||
self.file = file.relpath
|
||||
self.name = name
|
||||
super().__init__(
|
||||
f"Stage '{name}' not found inside '{file.relpath}' file"
|
||||
f"Stage '{self.name}' not found inside '{self.file}' file"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -23,11 +23,7 @@ def sync_import(stage, dry=False, force=False):
|
||||
if dry:
|
||||
return
|
||||
|
||||
if (
|
||||
not force
|
||||
and not stage.changed_stage(warn=True)
|
||||
and stage.already_cached()
|
||||
):
|
||||
if not force and not stage.changed_stage() and stage.already_cached():
|
||||
stage.outs[0].checkout()
|
||||
else:
|
||||
stage.deps[0].download(stage.outs[0])
|
||||
|
||||
@@ -30,16 +30,6 @@ class StageLoader(Mapping):
|
||||
self.stages_data = stages_data or {}
|
||||
self.lockfile_data = lockfile_data or {}
|
||||
|
||||
def filter(self, item=None):
|
||||
if not item:
|
||||
return self
|
||||
|
||||
if item not in self:
|
||||
raise StageNotFound(self.dvcfile, item)
|
||||
|
||||
data = {item: self.stages_data[item]} if item in self else {}
|
||||
return self.__class__(self.dvcfile, data, self.lockfile_data)
|
||||
|
||||
@staticmethod
|
||||
def fill_from_lock(stage, lock_data):
|
||||
from .params import StageParams
|
||||
@@ -224,9 +214,6 @@ class SingleStageLoader(Mapping):
|
||||
self.stage_data = stage_data or {}
|
||||
self.stage_text = stage_text
|
||||
|
||||
def filter(self, item=None):
|
||||
return self
|
||||
|
||||
def __getitem__(self, item):
|
||||
if item:
|
||||
logger.warning(
|
||||
|
||||
+16
-4
@@ -79,11 +79,14 @@ def cmd_run(stage, *args, **kwargs):
|
||||
|
||||
|
||||
def _is_cached(stage):
|
||||
return (
|
||||
cached = (
|
||||
not stage.is_callback
|
||||
and not stage.always_changed
|
||||
and stage.already_cached()
|
||||
)
|
||||
if cached:
|
||||
logger.info("Stage '%s' is cached", stage.addressing)
|
||||
return cached
|
||||
|
||||
|
||||
def restored_from_cache(stage):
|
||||
@@ -93,7 +96,10 @@ def restored_from_cache(stage):
|
||||
return False
|
||||
# restore stage from build cache
|
||||
stage_cache.restore(stage)
|
||||
return stage.outs_cached()
|
||||
restored = stage.outs_cached()
|
||||
if restored:
|
||||
logger.info("Restored stage '%s' from run-cache", stage.addressing)
|
||||
return restored
|
||||
|
||||
|
||||
def run_stage(stage, dry=False, force=False, run_cache=False):
|
||||
@@ -102,10 +108,16 @@ def run_stage(stage, dry=False, force=False, run_cache=False):
|
||||
run_cache and restored_from_cache(stage)
|
||||
)
|
||||
if stage_cached:
|
||||
logger.info("Stage is cached, skipping.")
|
||||
logger.info("Skipping run, checking out outputs")
|
||||
stage.checkout()
|
||||
return
|
||||
|
||||
logger.info("Running command:\n\t%s", stage.cmd)
|
||||
callback_str = "callback " if stage.is_callback else ""
|
||||
logger.info(
|
||||
"Running %s" "stage '%s' with command:",
|
||||
callback_str,
|
||||
stage.addressing,
|
||||
)
|
||||
logger.info("\t%s", stage.cmd)
|
||||
if not dry:
|
||||
cmd_run(stage)
|
||||
|
||||
+7
-7
@@ -7,6 +7,7 @@ import sqlite3
|
||||
from urllib.parse import urlencode, urlunparse
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.scm.tree import WorkingTree
|
||||
from dvc.utils import current_timestamp, relpath, to_chunks
|
||||
from dvc.utils.fs import get_inode, get_mtime_and_size, remove
|
||||
|
||||
@@ -91,6 +92,7 @@ class State: # pylint: disable=too-many-instance-attributes
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
self.root_dir = repo.root_dir
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
|
||||
state_config = repo.config.get("state", {})
|
||||
self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
|
||||
@@ -125,7 +127,7 @@ class State: # pylint: disable=too-many-instance-attributes
|
||||
self.dump()
|
||||
|
||||
def _execute(self, cmd, parameters=()):
|
||||
logger.debug(cmd)
|
||||
logger.trace(cmd)
|
||||
return self.cursor.execute(cmd, parameters)
|
||||
|
||||
def _fetchall(self):
|
||||
@@ -364,9 +366,7 @@ class State: # pylint: disable=too-many-instance-attributes
|
||||
assert checksum is not None
|
||||
assert os.path.exists(path_info)
|
||||
|
||||
actual_mtime, actual_size = get_mtime_and_size(
|
||||
path_info, self.repo.tree
|
||||
)
|
||||
actual_mtime, actual_size = get_mtime_and_size(path_info, self.tree)
|
||||
actual_inode = get_inode(path_info)
|
||||
|
||||
existing_record = self.get_state_record_for_inode(actual_inode)
|
||||
@@ -397,7 +397,7 @@ class State: # pylint: disable=too-many-instance-attributes
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
|
||||
actual_mtime, actual_size = get_mtime_and_size(path, self.repo.tree)
|
||||
actual_mtime, actual_size = get_mtime_and_size(path, self.tree)
|
||||
actual_inode = get_inode(path)
|
||||
|
||||
existing_record = self.get_state_record_for_inode(actual_inode)
|
||||
@@ -423,7 +423,7 @@ class State: # pylint: disable=too-many-instance-attributes
|
||||
if not os.path.exists(path_info):
|
||||
return
|
||||
|
||||
mtime, _ = get_mtime_and_size(path_info, self.repo.tree)
|
||||
mtime, _ = get_mtime_and_size(path_info, self.tree)
|
||||
inode = get_inode(path_info)
|
||||
relative_path = relpath(path_info, self.root_dir)
|
||||
|
||||
@@ -450,7 +450,7 @@ class State: # pylint: disable=too-many-instance-attributes
|
||||
continue
|
||||
|
||||
actual_inode = get_inode(path)
|
||||
actual_mtime, _ = get_mtime_and_size(path, self.repo.tree)
|
||||
actual_mtime, _ = get_mtime_and_size(path, self.tree)
|
||||
|
||||
if (inode, mtime) == (actual_inode, actual_mtime):
|
||||
logger.debug("Removing '%s' as unused link.", path)
|
||||
|
||||
+13
-11
@@ -52,18 +52,14 @@ def file_md5(fname, tree=None):
|
||||
exists_func = tree.exists
|
||||
stat_func = tree.stat
|
||||
open_func = tree.open
|
||||
# assume we don't need to run dos2unix when comparing git blobs
|
||||
binary = True
|
||||
else:
|
||||
exists_func = os.path.exists
|
||||
stat_func = os.stat
|
||||
open_func = open
|
||||
binary = False
|
||||
|
||||
if exists_func(fname):
|
||||
hash_md5 = hashlib.md5()
|
||||
if not binary:
|
||||
binary = not istextfile(fname)
|
||||
binary = not istextfile(fname, tree=tree)
|
||||
size = stat_func(fname).st_size
|
||||
no_progress_bar = True
|
||||
if size >= LARGE_FILE_SIZE:
|
||||
@@ -399,7 +395,7 @@ def format_link(link):
|
||||
|
||||
|
||||
def parse_target(target, default=None):
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK, is_valid_filename
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
if not target:
|
||||
@@ -413,13 +409,19 @@ def parse_target(target, default=None):
|
||||
match.group("path"),
|
||||
match.group("name"),
|
||||
)
|
||||
if path:
|
||||
if os.path.basename(path) == PIPELINE_LOCK:
|
||||
raise DvcException(
|
||||
"Did you mean: `{}`?".format(
|
||||
target.replace(".lock", ".yaml", 1)
|
||||
)
|
||||
)
|
||||
if not name:
|
||||
ret = (target, None)
|
||||
return ret if is_valid_filename(target) else ret[::-1]
|
||||
|
||||
if not path:
|
||||
path = default or PIPELINE_FILE
|
||||
logger.debug("Assuming file to be '%s'", path)
|
||||
|
||||
if os.path.basename(path) == PIPELINE_LOCK:
|
||||
raise DvcException(
|
||||
"Did you mean: `{}`?".format(target.replace(".lock", ".yaml", 1))
|
||||
)
|
||||
|
||||
return path, name
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
_BASE_VERSION = "1.0.0a2"
|
||||
_BASE_VERSION = "1.0.0a6"
|
||||
|
||||
|
||||
def _generate_version(base_version):
|
||||
|
||||
@@ -15,7 +15,7 @@ fi
|
||||
|
||||
echo >env.sh
|
||||
|
||||
if [[ "$TRAVIS_BUILD_STAGE_NAME" == "Test" ]]; then
|
||||
if [[ "$TRAVIS_BUILD_STAGE_NAME" == "test" ]]; then
|
||||
if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then
|
||||
# NOTE: ssh keys for ssh test to be able to ssh to the localhost
|
||||
ssh-keygen -t rsa -N "" -f mykey
|
||||
@@ -52,7 +52,11 @@ elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
|
||||
fi
|
||||
|
||||
if [[ -n "$TRAVIS_TAG" ]]; then
|
||||
echo "export SNAP_CHANNEL=stable" >>env.sh
|
||||
if [[ $(echo "$TRAVIS_TAG" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$') ]]; then
|
||||
echo "export SNAP_CHANNEL=stable" >>env.sh
|
||||
else
|
||||
echo "export SNAP_CHANNEL=beta" >>env.sh
|
||||
fi
|
||||
else
|
||||
echo "export SNAP_CHANNEL=edge" >>env.sh
|
||||
fi
|
||||
|
||||
@@ -3,13 +3,6 @@
|
||||
set -e
|
||||
set -x
|
||||
|
||||
# install docker
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
|
||||
sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y docker-ce
|
||||
|
||||
# run azurite
|
||||
sudo docker run -d --restart always -e executable=blob -p 10000:10000 \
|
||||
--tmpfs /opt/azurite/folder \
|
||||
|
||||
@@ -31,14 +31,14 @@ _dvc_import='-o --out --rev'
|
||||
_dvc_import_url='-f --file'
|
||||
_dvc_init='--no-scm -f --force'
|
||||
_dvc_install=''
|
||||
_dvc_list='-R --recursive --outs-only --rev'
|
||||
_dvc_list='-R --recursive --dvc-only --rev'
|
||||
_dvc_list_COMPGEN=_dvc_compgen_files
|
||||
_dvc_lock=''
|
||||
_dvc_lock_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_metrics='add diff modify remove show'
|
||||
_dvc_metrics_add='-t --type -x --xpath'
|
||||
_dvc_metrics_add_COMPGEN=_dvc_compgen_files
|
||||
_dvc_metrics_diff='--targets -t --type -x --xpath -R --show-json --show-md'
|
||||
_dvc_metrics_diff='--targets -t --type -x --xpath -R --show-json --show-md --no-path --old'
|
||||
_dvc_metrics_modify='-t --type -x --xpath'
|
||||
_dvc_metrics_modify_COMPGEN=_dvc_compgen_files
|
||||
_dvc_metrics_remove=''
|
||||
@@ -48,24 +48,25 @@ _dvc_metrics_show_COMPGEN=_dvc_compgen_files
|
||||
_dvc_move=''
|
||||
_dvc_move_COMPGEN=_dvc_compgen_files
|
||||
_dvc_params='diff'
|
||||
_dvc_params_diff='--all --show-json --show-md'
|
||||
_dvc_params_diff='--all --show-json --show-md --no-path'
|
||||
_dvc_pipeline='list show'
|
||||
_dvc_pipeline_list=''
|
||||
_dvc_pipeline_show='-c --commands -o --outs --ascii --dot --tree -l --locked'
|
||||
_dvc_pipeline_show_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_plots='show diff'
|
||||
_dvc_plots_show='-t --template -f --file -s --select -x -y --stdout --no-csv-header --no-html --title --xlab --ylab'
|
||||
_dvc_plots_diff='-t --template -d --datafile -f --file -s --select -x -y --stdout --no-csv-header --no-html --title --xlab --ylab'
|
||||
_dvc_plots_show='-t --template -o --out -x -y --show-json --no-csv-header --title --xlab --ylab'
|
||||
_dvc_plots_diff='-t --template --targets -o --out -x -y --show-json --no-csv-header --title --xlab --ylab'
|
||||
_dvc_pull='-j --jobs -r --remote -a --all-branches -T --all-tags -f --force -d --with-deps -R --recursive'
|
||||
_dvc_pull_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_push='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
|
||||
_dvc_push_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_remote='add default list modify remove'
|
||||
_dvc_remote='add default list modify remove rename'
|
||||
_dvc_remote_add='--global --system --local -d --default -f --force'
|
||||
_dvc_remote_default='--global --system --local -u --unset'
|
||||
_dvc_remote_list='--global --system --local'
|
||||
_dvc_remote_modify='--global --system --local -u --unset'
|
||||
_dvc_remote_remove='--global --system --local'
|
||||
_dvc_remote_rename='--global --system --local'
|
||||
_dvc_remove='-o --outs -p --purge -f --force'
|
||||
_dvc_remove_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --no-run-cache --force-downstream --no-commit -R --recursive --downstream'
|
||||
|
||||
@@ -170,7 +170,7 @@ _dvc_lock=(
|
||||
_dvc_list=(
|
||||
"--rev[Git revision (e.g. branch, tag, SHA)]:Revision:"
|
||||
{-R,--recursive}"[Recursively add each file under the directory.]"
|
||||
"--outs-only[Only outputs DVC-outs.]"
|
||||
"--dvc-only[Only outputs DVC-outs.]"
|
||||
"1:URL:"
|
||||
"2:Path:"
|
||||
)
|
||||
@@ -218,7 +218,7 @@ _dvc_plots=(
|
||||
)
|
||||
|
||||
_dvc_remote=(
|
||||
"1:Sub command:(add default remove modify list)"
|
||||
"1:Sub command:(add default remove modify list rename)"
|
||||
)
|
||||
|
||||
_dvc_remove=(
|
||||
|
||||
@@ -92,12 +92,7 @@ s3 = ["boto3>=1.9.201"]
|
||||
azure = ["azure-storage-blob==2.1.0"]
|
||||
oss = ["oss2==2.6.1"]
|
||||
ssh = ["paramiko>=2.5.0"]
|
||||
hdfs = [
|
||||
# pyarrow-0.16.0 import fails on 3.7 (works on 3.6 though)
|
||||
# due to: https://issues.apache.org/jira/browse/ARROW-7852
|
||||
"pyarrow==0.15.1; python_version < '3.8'",
|
||||
"pyarrow==0.16.0; python_version == '3.8'",
|
||||
]
|
||||
hdfs = ["pyarrow>=0.17.0"]
|
||||
# gssapi should not be included in all_remotes, because it doesn't have wheels
|
||||
# for linux and mac, so it will fail to compile if user doesn't have all the
|
||||
# requirements, including kerberos itself. Once all the wheels are available,
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
from tests.func.plots import run_copy_metrics # noqa: F401
|
||||
@@ -3,11 +3,10 @@ import json
|
||||
import yaml
|
||||
|
||||
|
||||
def test_metrics_diff_simple(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_simple(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
def _gen(val):
|
||||
tmp_dir.gen({"m.yaml": str(val)})
|
||||
dvc.run(cmd="", metrics=["m.yaml"], single_stage=True)
|
||||
dvc.scm.add(["m.yaml.dvc"])
|
||||
tmp_dir.gen({"m_temp.yaml": str(val)})
|
||||
run_copy_metrics("m_temp.yaml", "m.yaml", metrics=["m.yaml"])
|
||||
dvc.scm.commit(str(val))
|
||||
|
||||
_gen(1)
|
||||
@@ -19,13 +18,13 @@ def test_metrics_diff_simple(tmp_dir, scm, dvc):
|
||||
assert dvc.metrics.diff(a_rev="HEAD~2") == expected
|
||||
|
||||
|
||||
def test_metrics_diff_yaml(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_yaml(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
def _gen(val):
|
||||
metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
|
||||
tmp_dir.gen({"m.yaml": yaml.dump(metrics)})
|
||||
dvc.run(cmd="", metrics=["m.yaml"], single_stage=True)
|
||||
dvc.scm.add(["m.yaml.dvc"])
|
||||
dvc.scm.commit(str(val))
|
||||
tmp_dir.gen({"m_temp.yaml": yaml.dump(metrics)})
|
||||
run_copy_metrics(
|
||||
"m_temp.yaml", "m.yaml", metrics=["m.yaml"], commit=str(val)
|
||||
)
|
||||
|
||||
_gen(1)
|
||||
_gen(2)
|
||||
@@ -36,13 +35,13 @@ def test_metrics_diff_yaml(tmp_dir, scm, dvc):
|
||||
assert dvc.metrics.diff(a_rev="HEAD~2") == expected
|
||||
|
||||
|
||||
def test_metrics_diff_json(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_json(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
def _gen(val):
|
||||
metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
|
||||
tmp_dir.gen({"m.json": json.dumps(metrics)})
|
||||
dvc.run(cmd="", metrics=["m.json"], single_stage=True)
|
||||
dvc.scm.add(["m.json.dvc"])
|
||||
dvc.scm.commit(str(val))
|
||||
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
|
||||
run_copy_metrics(
|
||||
"m_temp.json", "m.json", metrics=["m.json"], commit=str(val)
|
||||
)
|
||||
|
||||
_gen(1)
|
||||
_gen(2)
|
||||
@@ -52,13 +51,13 @@ def test_metrics_diff_json(tmp_dir, scm, dvc):
|
||||
assert dvc.metrics.diff(a_rev="HEAD~2") == expected
|
||||
|
||||
|
||||
def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
def _gen(val):
|
||||
metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
|
||||
tmp_dir.gen({"m.json": json.dumps(metrics)})
|
||||
dvc.run(cmd="", metrics=["m.json"], single_stage=True)
|
||||
dvc.scm.add(["m.json.dvc"])
|
||||
dvc.scm.commit(str(val))
|
||||
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
|
||||
run_copy_metrics(
|
||||
"m_temp.json", "m.json", metrics=["m.json"], commit=str(val)
|
||||
)
|
||||
|
||||
_gen(1)
|
||||
_gen(2)
|
||||
@@ -67,12 +66,15 @@ def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc):
|
||||
assert dvc.metrics.diff(a_rev="HEAD~2") == {}
|
||||
|
||||
|
||||
def test_metrics_diff_broken_json(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_broken_json(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
|
||||
tmp_dir.gen({"m.json": json.dumps(metrics)})
|
||||
dvc.run(cmd="", metrics_no_cache=["m.json"], single_stage=True)
|
||||
dvc.scm.add(["m.json.dvc", "m.json"])
|
||||
dvc.scm.commit("add metrics")
|
||||
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
|
||||
run_copy_metrics(
|
||||
"m_temp.json",
|
||||
"m.json",
|
||||
metrics_no_cache=["m.json"],
|
||||
commit="add metrics",
|
||||
)
|
||||
|
||||
(tmp_dir / "m.json").write_text(json.dumps(metrics) + "ma\nlformed\n")
|
||||
|
||||
@@ -89,10 +91,10 @@ def test_metrics_diff_no_metrics(tmp_dir, scm, dvc):
|
||||
assert dvc.metrics.diff(a_rev="HEAD~1") == {}
|
||||
|
||||
|
||||
def test_metrics_diff_new_metric(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_new_metric(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
|
||||
tmp_dir.gen({"m.json": json.dumps(metrics)})
|
||||
dvc.run(cmd="", metrics_no_cache=["m.json"], single_stage=True)
|
||||
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
|
||||
run_copy_metrics("m_temp.json", "m.json", metrics_no_cache=["m.json"])
|
||||
|
||||
assert dvc.metrics.diff() == {
|
||||
"m.json": {
|
||||
@@ -102,12 +104,15 @@ def test_metrics_diff_new_metric(tmp_dir, scm, dvc):
|
||||
}
|
||||
|
||||
|
||||
def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
|
||||
tmp_dir.gen({"m.json": json.dumps(metrics)})
|
||||
dvc.run(cmd="", metrics_no_cache=["m.json"], single_stage=True)
|
||||
dvc.scm.add(["m.json.dvc", "m.json"])
|
||||
dvc.scm.commit("add metrics")
|
||||
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
|
||||
run_copy_metrics(
|
||||
"m_temp.json",
|
||||
"m.json",
|
||||
metrics_no_cache=["m.json"],
|
||||
commit="add metrics",
|
||||
)
|
||||
|
||||
(tmp_dir / "m.json").unlink()
|
||||
|
||||
@@ -119,11 +124,14 @@ def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc):
|
||||
}
|
||||
|
||||
|
||||
def test_metrics_diff_with_unchanged(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("metrics.yaml", "foo: 1\nxyz: 10")
|
||||
dvc.run(metrics_no_cache=["metrics.yaml"], single_stage=True)
|
||||
scm.add(["metrics.yaml", "metrics.yaml.dvc"])
|
||||
scm.commit("1")
|
||||
def test_metrics_diff_with_unchanged(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metrics_temp.yaml", "foo: 1\nxyz: 10")
|
||||
run_copy_metrics(
|
||||
"metrics_temp.yaml",
|
||||
"metrics.yaml",
|
||||
metrics_no_cache=["metrics.yaml"],
|
||||
commit="1",
|
||||
)
|
||||
|
||||
tmp_dir.scm_gen("metrics.yaml", "foo: 2\nxyz: 10", commit="2")
|
||||
tmp_dir.scm_gen("metrics.yaml", "foo: 3\nxyz: 10", commit="3")
|
||||
|
||||
@@ -8,36 +8,44 @@ def test_show_empty(dvc):
|
||||
dvc.metrics.show()
|
||||
|
||||
|
||||
def test_show_simple(tmp_dir, dvc):
|
||||
tmp_dir.gen("metrics.yaml", "1.1")
|
||||
dvc.run(metrics=["metrics.yaml"], single_stage=True)
|
||||
def test_show_simple(tmp_dir, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metrics_t.yaml", "1.1")
|
||||
run_copy_metrics(
|
||||
"metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"],
|
||||
)
|
||||
assert dvc.metrics.show() == {"": {"metrics.yaml": 1.1}}
|
||||
|
||||
|
||||
def test_show(tmp_dir, dvc):
|
||||
tmp_dir.gen("metrics.yaml", "foo: 1.1")
|
||||
dvc.run(metrics=["metrics.yaml"], single_stage=True)
|
||||
def test_show(tmp_dir, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metrics_t.yaml", "foo: 1.1")
|
||||
run_copy_metrics(
|
||||
"metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"],
|
||||
)
|
||||
assert dvc.metrics.show() == {"": {"metrics.yaml": {"foo": 1.1}}}
|
||||
|
||||
|
||||
def test_show_multiple(tmp_dir, dvc):
|
||||
tmp_dir.gen("foo", "foo: 1\n")
|
||||
tmp_dir.gen("baz", "baz: 2\n")
|
||||
dvc.run(fname="foo.dvc", metrics=["foo"], single_stage=True)
|
||||
dvc.run(fname="baz.dvc", metrics=["baz"], single_stage=True)
|
||||
def test_show_multiple(tmp_dir, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("foo_temp", "foo: 1\n")
|
||||
tmp_dir.gen("baz_temp", "baz: 2\n")
|
||||
run_copy_metrics("foo_temp", "foo", fname="foo.dvc", metrics=["foo"])
|
||||
run_copy_metrics("baz_temp", "baz", fname="baz.dvc", metrics=["baz"])
|
||||
assert dvc.metrics.show() == {"": {"foo": {"foo": 1}, "baz": {"baz": 2}}}
|
||||
|
||||
|
||||
def test_show_invalid_metric(tmp_dir, dvc):
|
||||
tmp_dir.gen("metrics.yaml", "foo:\n- bar\n- baz\nxyz: string")
|
||||
dvc.run(metrics=["metrics.yaml"], single_stage=True)
|
||||
def test_show_invalid_metric(tmp_dir, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metrics_temp.yaml", "foo:\n- bar\n- baz\nxyz: string")
|
||||
run_copy_metrics(
|
||||
"metrics_temp.yaml", "metrics.yaml", metrics=["metrics.yaml"]
|
||||
)
|
||||
with pytest.raises(NoMetricsError):
|
||||
dvc.metrics.show()
|
||||
|
||||
|
||||
def test_show_branch(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("metrics.yaml", "foo: 1")
|
||||
dvc.run(metrics_no_cache=["metrics.yaml"], single_stage=True)
|
||||
def test_show_branch(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metrics_temp.yaml", "foo: 1")
|
||||
run_copy_metrics(
|
||||
"metrics_temp.yaml", "metrics.yaml", metrics_no_cache=["metrics.yaml"]
|
||||
)
|
||||
scm.add(["metrics.yaml", "metrics.yaml.dvc"])
|
||||
scm.commit("init")
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ def test_diff_no_params(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_no_changes(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("bar")
|
||||
assert dvc.params.diff() == {}
|
||||
@@ -12,7 +12,7 @@ def test_diff_no_changes(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("bar")
|
||||
|
||||
@@ -26,7 +26,7 @@ def test_diff(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_new(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
|
||||
assert dvc.params.diff() == {
|
||||
"params.yaml": {"foo": {"old": None, "new": "bar"}}
|
||||
@@ -35,7 +35,7 @@ def test_diff_new(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_deleted(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("bar")
|
||||
|
||||
@@ -48,7 +48,7 @@ def test_diff_deleted(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_deleted_config(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("bar")
|
||||
|
||||
@@ -61,7 +61,7 @@ def test_diff_deleted_config(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_list(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("foo")
|
||||
|
||||
@@ -76,7 +76,7 @@ def test_diff_list(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_dict(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo:\n bar: baz")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("foo")
|
||||
|
||||
@@ -89,7 +89,7 @@ def test_diff_dict(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_with_unchanged(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar\nxyz: val")
|
||||
dvc.run(params=["foo,xyz"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo,xyz"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("bar")
|
||||
|
||||
|
||||
@@ -10,14 +10,24 @@ def test_show_empty(dvc):
|
||||
|
||||
def test_show(tmp_dir, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
assert dvc.params.show() == {"": {"params.yaml": {"foo": "bar"}}}
|
||||
|
||||
|
||||
def test_show_multiple(tmp_dir, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar\nbaz: qux\n")
|
||||
dvc.run(fname="foo.dvc", params=["foo"], single_stage=True)
|
||||
dvc.run(fname="baz.dvc", params=["baz"], single_stage=True)
|
||||
dvc.run(
|
||||
cmd="echo params.yaml",
|
||||
fname="foo.dvc",
|
||||
params=["foo"],
|
||||
single_stage=True,
|
||||
)
|
||||
dvc.run(
|
||||
cmd="echo params.yaml",
|
||||
fname="baz.dvc",
|
||||
params=["baz"],
|
||||
single_stage=True,
|
||||
)
|
||||
assert dvc.params.show() == {
|
||||
"": {"params.yaml": {"foo": "bar", "baz": "qux"}}
|
||||
}
|
||||
@@ -25,13 +35,13 @@ def test_show_multiple(tmp_dir, dvc):
|
||||
|
||||
def test_show_list(tmp_dir, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz\n")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
assert dvc.params.show() == {"": {"params.yaml": {"foo": ["bar", "baz"]}}}
|
||||
|
||||
|
||||
def test_show_branch(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("init")
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_copy_metrics(tmp_dir, run_copy):
|
||||
def run(file1, file2, commit=None, tag=None, **kwargs):
|
||||
stage = tmp_dir.dvc.run(
|
||||
cmd=f"python copy.py {file1} {file2}",
|
||||
deps=[file1],
|
||||
single_stage=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if hasattr(tmp_dir.dvc, "scm"):
|
||||
files = [stage.path]
|
||||
files += [
|
||||
os.fspath(out.path_info)
|
||||
for out in stage.outs
|
||||
if not out.use_cache
|
||||
]
|
||||
tmp_dir.dvc.scm.add(files)
|
||||
if commit:
|
||||
tmp_dir.dvc.scm.commit(commit)
|
||||
if tag:
|
||||
tmp_dir.dvc.scm.tag(tag)
|
||||
return stage
|
||||
|
||||
return run
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
from . import run_copy_metrics # noqa: F401
|
||||
@@ -1,20 +1,32 @@
|
||||
import json
|
||||
|
||||
from .test_plots import PlotData, _run_with, _write_json
|
||||
from .test_plots import PlotData, _write_json
|
||||
|
||||
|
||||
def test_diff_dirty(tmp_dir, scm, dvc):
|
||||
def test_diff_dirty(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric_1 = [{"y": 2}, {"y": 3}]
|
||||
_write_json(tmp_dir, metric_1, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init")
|
||||
_write_json(tmp_dir, metric_1, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
)
|
||||
|
||||
metric_2 = [{"y": 3}, {"y": 5}]
|
||||
_write_json(tmp_dir, metric_2, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="second")
|
||||
_write_json(tmp_dir, metric_2, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="second",
|
||||
)
|
||||
|
||||
metric_3 = [{"y": 5}, {"y": 6}]
|
||||
_write_json(tmp_dir, metric_3, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"])
|
||||
_write_json(tmp_dir, metric_3, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json", "metric.json", plots_no_cache=["metric.json"]
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.diff(fields={"y"})["metric.json"]
|
||||
|
||||
|
||||
+150
-76
@@ -26,20 +26,6 @@ def _remove_whitespace(value):
|
||||
return value.replace(" ", "").replace("\n", "")
|
||||
|
||||
|
||||
def _run_with(tmp_dir, commit=None, tag=None, **kwargs):
|
||||
stage = tmp_dir.dvc.run(single_stage=True, **kwargs)
|
||||
if hasattr(tmp_dir.dvc, "scm"):
|
||||
files = [stage.path]
|
||||
files += [
|
||||
os.fspath(out.path_info) for out in stage.outs if not out.use_cache
|
||||
]
|
||||
tmp_dir.dvc.scm.add(files)
|
||||
if commit:
|
||||
tmp_dir.dvc.scm.commit(commit)
|
||||
if tag:
|
||||
tmp_dir.dvc.scm.tag(tag)
|
||||
|
||||
|
||||
def _write_csv(metric, filename, header=True):
|
||||
with open(filename, "w", newline="") as csvobj:
|
||||
if header:
|
||||
@@ -59,11 +45,13 @@ def _write_json(tmp_dir, metric, filename):
|
||||
tmp_dir.gen(filename, json.dumps(metric, sort_keys=True))
|
||||
|
||||
|
||||
def test_plot_csv_one_column(tmp_dir, scm, dvc):
|
||||
def test_plot_csv_one_column(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
# no header
|
||||
metric = [{"val": 2}, {"val": 3}]
|
||||
_write_csv(metric, "metric.csv", header=False)
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.csv"])
|
||||
_write_csv(metric, "metric_t.csv", header=False)
|
||||
run_copy_metrics(
|
||||
"metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
csv_header=False,
|
||||
@@ -84,13 +72,15 @@ def test_plot_csv_one_column(tmp_dir, scm, dvc):
|
||||
assert plot_content["encoding"]["y"]["title"] == "y_title"
|
||||
|
||||
|
||||
def test_plot_csv_multiple_columns(tmp_dir, scm, dvc):
|
||||
def test_plot_csv_multiple_columns(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [
|
||||
OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
|
||||
OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
|
||||
]
|
||||
_write_csv(metric, "metric.csv")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.csv"])
|
||||
_write_csv(metric, "metric_t.csv")
|
||||
run_copy_metrics(
|
||||
"metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show()["metric.csv"]
|
||||
|
||||
@@ -115,13 +105,15 @@ def test_plot_csv_multiple_columns(tmp_dir, scm, dvc):
|
||||
assert plot_content["encoding"]["y"]["field"] == "val"
|
||||
|
||||
|
||||
def test_plot_csv_choose_axes(tmp_dir, scm, dvc):
|
||||
def test_plot_csv_choose_axes(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [
|
||||
OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
|
||||
OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
|
||||
]
|
||||
_write_csv(metric, "metric.csv")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.csv"])
|
||||
_write_csv(metric, "metric_t.csv")
|
||||
run_copy_metrics(
|
||||
"metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(x_field="first_val", y_field="second_val")[
|
||||
"metric.csv"
|
||||
@@ -146,10 +138,15 @@ def test_plot_csv_choose_axes(tmp_dir, scm, dvc):
|
||||
assert plot_content["encoding"]["y"]["field"] == "second_val"
|
||||
|
||||
|
||||
def test_plot_json_single_val(tmp_dir, scm, dvc):
|
||||
def test_plot_json_single_val(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [{"val": 2}, {"val": 3}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="first run",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show()["metric.json"]
|
||||
|
||||
@@ -162,13 +159,18 @@ def test_plot_json_single_val(tmp_dir, scm, dvc):
|
||||
assert plot_json["encoding"]["y"]["field"] == "val"
|
||||
|
||||
|
||||
def test_plot_json_multiple_val(tmp_dir, scm, dvc):
|
||||
def test_plot_json_multiple_val(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [
|
||||
{"first_val": 100, "val": 2},
|
||||
{"first_val": 200, "val": 3},
|
||||
]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="first run",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show()["metric.json"]
|
||||
|
||||
@@ -191,13 +193,18 @@ def test_plot_json_multiple_val(tmp_dir, scm, dvc):
|
||||
assert plot_content["encoding"]["y"]["field"] == "val"
|
||||
|
||||
|
||||
def test_plot_confusion(tmp_dir, dvc):
|
||||
def test_plot_confusion(tmp_dir, dvc, run_copy_metrics):
|
||||
confusion_matrix = [
|
||||
{"predicted": "B", "actual": "A"},
|
||||
{"predicted": "A", "actual": "A"},
|
||||
]
|
||||
_write_json(tmp_dir, confusion_matrix, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
|
||||
_write_json(tmp_dir, confusion_matrix, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="first run",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
template="confusion", x_field="predicted", y_field="actual",
|
||||
@@ -212,21 +219,35 @@ def test_plot_confusion(tmp_dir, dvc):
|
||||
assert plot_content["encoding"]["y"]["field"] == "actual"
|
||||
|
||||
|
||||
def test_plot_multiple_revs_default(tmp_dir, scm, dvc):
|
||||
def test_plot_multiple_revs_default(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric_1 = [{"y": 2}, {"y": 3}]
|
||||
_write_json(tmp_dir, metric_1, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
|
||||
_write_json(tmp_dir, metric_1, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
metric_2 = [{"y": 3}, {"y": 5}]
|
||||
_write_json(tmp_dir, metric_2, "metric.json")
|
||||
_run_with(
|
||||
tmp_dir, plots_no_cache=["metric.json"], commit="second", tag="v2"
|
||||
_write_json(tmp_dir, metric_2, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="second",
|
||||
tag="v2",
|
||||
)
|
||||
|
||||
metric_3 = [{"y": 5}, {"y": 6}]
|
||||
_write_json(tmp_dir, metric_3, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="third")
|
||||
|
||||
_write_json(tmp_dir, metric_3, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="third",
|
||||
)
|
||||
plot_string = dvc.plots.show(fields={"y"}, revs=["HEAD", "v2", "v1"],)[
|
||||
"metric.json"
|
||||
]
|
||||
@@ -244,22 +265,31 @@ def test_plot_multiple_revs_default(tmp_dir, scm, dvc):
|
||||
assert plot_content["encoding"]["y"]["field"] == "y"
|
||||
|
||||
|
||||
def test_plot_multiple_revs(tmp_dir, scm, dvc):
|
||||
def test_plot_multiple_revs(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
shutil.copy(tmp_dir / ".dvc" / "plots" / "default.json", "template.json")
|
||||
|
||||
metric_1 = [{"y": 2}, {"y": 3}]
|
||||
_write_json(tmp_dir, metric_1, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
|
||||
|
||||
metric_2 = [{"y": 3}, {"y": 5}]
|
||||
_write_json(tmp_dir, metric_2, "metric.json")
|
||||
_run_with(
|
||||
tmp_dir, plots_no_cache=["metric.json"], commit="second", tag="v2"
|
||||
_write_json(tmp_dir, metric_1, "metric_t.json")
|
||||
stage = run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
metric_2 = [{"y": 3}, {"y": 5}]
|
||||
_write_json(tmp_dir, metric_2, "metric_t.json")
|
||||
assert dvc.reproduce(stage.addressing) == [stage]
|
||||
scm.add(["metric.json", stage.path])
|
||||
scm.commit("second")
|
||||
scm.tag("v2")
|
||||
|
||||
metric_3 = [{"y": 5}, {"y": 6}]
|
||||
_write_json(tmp_dir, metric_3, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="third")
|
||||
_write_json(tmp_dir, metric_3, "metric_t.json")
|
||||
assert dvc.reproduce(stage.addressing) == [stage]
|
||||
scm.add(["metric.json", stage.path])
|
||||
scm.commit("third")
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
template="template.json", revs=["HEAD", "v2", "v1"],
|
||||
@@ -278,14 +308,17 @@ def test_plot_multiple_revs(tmp_dir, scm, dvc):
|
||||
assert plot_content["encoding"]["y"]["field"] == "y"
|
||||
|
||||
|
||||
def test_plot_even_if_metric_missing(tmp_dir, scm, dvc, caplog):
|
||||
def test_plot_even_if_metric_missing(
|
||||
tmp_dir, scm, dvc, caplog, run_copy_metrics
|
||||
):
|
||||
tmp_dir.scm_gen("some_file", "content", commit="there is no metric")
|
||||
scm.tag("v1")
|
||||
|
||||
metric = [{"y": 2}, {"y": 3}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(
|
||||
tmp_dir,
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="there is metric",
|
||||
tag="v2",
|
||||
@@ -335,10 +368,16 @@ def custom_template(tmp_dir, dvc):
|
||||
return custom_template
|
||||
|
||||
|
||||
def test_custom_template(tmp_dir, scm, dvc, custom_template):
|
||||
def test_custom_template(tmp_dir, scm, dvc, custom_template, run_copy_metrics):
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
template=os.fspath(custom_template), x_field="a", y_field="b"
|
||||
@@ -358,15 +397,21 @@ def _replace(path, src, dst):
|
||||
|
||||
|
||||
def test_custom_template_with_specified_data(
|
||||
tmp_dir, scm, dvc, custom_template
|
||||
tmp_dir, scm, dvc, custom_template, run_copy_metrics
|
||||
):
|
||||
_replace(
|
||||
custom_template, "DVC_METRIC_DATA", "DVC_METRIC_DATA,metric.json",
|
||||
)
|
||||
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, outs_no_cache=["metric.json"], commit="init", tag="v1")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
outs_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
template=os.fspath(custom_template), x_field="a", y_field="b",
|
||||
@@ -381,7 +426,9 @@ def test_custom_template_with_specified_data(
|
||||
assert plot_content["encoding"]["y"]["field"] == "b"
|
||||
|
||||
|
||||
def test_plot_override_specified_data_source(tmp_dir, scm, dvc):
|
||||
def test_plot_override_specified_data_source(
|
||||
tmp_dir, scm, dvc, run_copy_metrics
|
||||
):
|
||||
shutil.copy(
|
||||
tmp_dir / ".dvc" / "plots" / "default.json",
|
||||
tmp_dir / "newtemplate.json",
|
||||
@@ -393,8 +440,14 @@ def test_plot_override_specified_data_source(tmp_dir, scm, dvc):
|
||||
)
|
||||
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric2.json")
|
||||
_run_with(tmp_dir, outs_no_cache=["metric2.json"], commit="init", tag="v1")
|
||||
_write_json(tmp_dir, metric, "metric1.json")
|
||||
run_copy_metrics(
|
||||
"metric1.json",
|
||||
"metric2.json",
|
||||
outs_no_cache=["metric2.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
targets=["metric2.json"], template="newtemplate.json", x_field="a"
|
||||
@@ -430,10 +483,18 @@ def test_plot_wrong_metric_type(tmp_dir, scm, dvc):
|
||||
dvc.plots.show(targets=["metric.txt"])
|
||||
|
||||
|
||||
def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template):
|
||||
def test_plot_choose_columns(
|
||||
tmp_dir, scm, dvc, custom_template, run_copy_metrics
|
||||
):
|
||||
metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
template=os.fspath(custom_template),
|
||||
@@ -451,10 +512,16 @@ def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template):
|
||||
assert plot_content["encoding"]["y"]["field"] == "c"
|
||||
|
||||
|
||||
def test_plot_default_choose_column(tmp_dir, scm, dvc):
|
||||
def test_plot_default_choose_column(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(fields={"b"})["metric.json"]
|
||||
|
||||
@@ -467,12 +534,14 @@ def test_plot_default_choose_column(tmp_dir, scm, dvc):
|
||||
assert plot_content["encoding"]["y"]["field"] == "b"
|
||||
|
||||
|
||||
def test_plot_yaml(tmp_dir, scm, dvc):
|
||||
def test_plot_yaml(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [{"val": 2}, {"val": 3}]
|
||||
with open("metric.yaml", "w") as fobj:
|
||||
with open("metric_t.yaml", "w") as fobj:
|
||||
yaml.dump(metric, fobj)
|
||||
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.yaml"])
|
||||
run_copy_metrics(
|
||||
"metric_t.yaml", "metric.yaml", plots_no_cache=["metric.yaml"]
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show()["metric.yaml"]
|
||||
|
||||
@@ -483,10 +552,15 @@ def test_plot_yaml(tmp_dir, scm, dvc):
|
||||
]
|
||||
|
||||
|
||||
def test_raise_on_wrong_field(tmp_dir, scm, dvc):
|
||||
def test_raise_on_wrong_field(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [{"val": 2}, {"val": 3}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="first run",
|
||||
)
|
||||
|
||||
with pytest.raises(NoFieldInDataError):
|
||||
dvc.plots.show("metric.json", x_field="no_val")
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
import os
|
||||
|
||||
import mock
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.remote.local import LocalRemote
|
||||
from tests.utils import trees_equal
|
||||
|
||||
|
||||
def test_dont_fail_on_unpacked_create_fail(tmp_dir, dvc):
|
||||
(stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
|
||||
|
||||
with mock.patch.object(
|
||||
LocalRemote, "_create_unpacked_dir", side_effect=DvcException("msg")
|
||||
) as unpacked_create_spy, dvc.state:
|
||||
assert not dvc.cache.local.changed_cache(stage.outs[0].checksum)
|
||||
assert unpacked_create_spy.call_count == 1
|
||||
|
||||
|
||||
def test_remove_unpacked_on_create_fail(tmp_dir, dvc):
|
||||
(stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
|
||||
unpacked_dir = stage.outs[0].cache_path + LocalRemote.UNPACKED_DIR_SUFFIX
|
||||
|
||||
# artificial unpacked dir for test purpose
|
||||
os.makedirs(unpacked_dir)
|
||||
assert os.path.exists(unpacked_dir)
|
||||
|
||||
with mock.patch.object(
|
||||
LocalRemote, "_create_unpacked_dir", side_effect=DvcException("msg")
|
||||
), dvc.state:
|
||||
assert not dvc.cache.local.changed_cache(stage.outs[0].checksum)
|
||||
|
||||
assert not os.path.exists(unpacked_dir)
|
||||
|
||||
|
||||
def test_create_unpacked_on_status(tmp_dir, dvc):
|
||||
(stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
|
||||
unpacked_dir = stage.outs[0].cache_path + LocalRemote.UNPACKED_DIR_SUFFIX
|
||||
assert not os.path.exists(unpacked_dir)
|
||||
|
||||
with dvc.state:
|
||||
assert not dvc.cache.local.changed_cache(stage.outs[0].checksum)
|
||||
assert os.path.exists(unpacked_dir)
|
||||
trees_equal("dir", unpacked_dir)
|
||||
|
||||
|
||||
def test_dir_cache_changed_on_single_cache_file_modification(tmp_dir, dvc):
|
||||
(stage,) = tmp_dir.dvc_gen(
|
||||
{"dir": {"file1": "file1 content", "file2": "file2 content"}}
|
||||
)
|
||||
unpacked_dir = stage.outs[0].cache_path + LocalRemote.UNPACKED_DIR_SUFFIX
|
||||
assert not os.path.exists(unpacked_dir)
|
||||
file_md5 = stage.outs[0].dir_cache[0]["md5"]
|
||||
|
||||
with dvc.state:
|
||||
assert not dvc.cache.local.changed_cache(stage.outs[0].checksum)
|
||||
assert os.path.exists(unpacked_dir)
|
||||
|
||||
cache_file_path = dvc.cache.local.get(file_md5)
|
||||
os.chmod(cache_file_path, 0o644)
|
||||
with open(cache_file_path, "a") as fobj:
|
||||
fobj.write("modification")
|
||||
|
||||
with dvc.state:
|
||||
assert dvc.cache.local.changed_cache(stage.outs[0].checksum)
|
||||
+14
-4
@@ -44,7 +44,11 @@ def test_add(tmp_dir, dvc):
|
||||
assert len(stage.deps) == 0
|
||||
assert stage.cmd is None
|
||||
assert stage.outs[0].info["md5"] == md5
|
||||
assert stage.md5 == "ee343f2482f53efffc109be83cc976ac"
|
||||
assert stage.md5 is None
|
||||
|
||||
assert load_stage_file("foo.dvc") == {
|
||||
"outs": [{"md5": "acbd18db4cc2f85cedef654fccc4a4d8", "path": "foo"}],
|
||||
}
|
||||
|
||||
|
||||
def test_add_unicode(tmp_dir, dvc):
|
||||
@@ -124,10 +128,16 @@ class TestAddDirectoryWithForwardSlash(TestDvc):
|
||||
|
||||
|
||||
def test_add_tracked_file(tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen("tracked_file", "...", commit="add tracked file")
|
||||
path = "tracked_file"
|
||||
tmp_dir.scm_gen(path, "...", commit="add tracked file")
|
||||
msg = f""" output '{path}' is already tracked by SCM \\(e.g. Git\\).
|
||||
You can remove it from Git, then add to DVC.
|
||||
To stop tracking from Git:
|
||||
git rm -r --cached '{path}'
|
||||
git commit -m "stop tracking {path}" """
|
||||
|
||||
with pytest.raises(OutputAlreadyTrackedError):
|
||||
dvc.add("tracked_file")
|
||||
with pytest.raises(OutputAlreadyTrackedError, match=msg):
|
||||
dvc.add(path)
|
||||
|
||||
|
||||
class TestAddDirWithExistingCache(TestDvc):
|
||||
|
||||
@@ -93,6 +93,19 @@ def test_open_external(remote_url, erepo_dir, setup_remote):
|
||||
assert api.read("version", repo=repo_url, rev="branch") == "branchver"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_open_granular(remote_url, tmp_dir, dvc):
|
||||
run_dvc("remote", "add", "-d", "upstream", remote_url)
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo-text"}})
|
||||
run_dvc("push")
|
||||
|
||||
# Remove cache to force download
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
with api.open("dir/foo") as fd:
|
||||
assert fd.read() == "foo-text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_missing(remote_url, tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
|
||||
+60
-11
@@ -7,22 +7,20 @@ import shutil
|
||||
import pytest
|
||||
from mock import patch
|
||||
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX, Dvcfile
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX, PIPELINE_FILE, Dvcfile
|
||||
from dvc.exceptions import (
|
||||
CheckoutError,
|
||||
CheckoutErrorSuggestGit,
|
||||
ConfirmRemoveError,
|
||||
DvcException,
|
||||
NoOutputOrStageError,
|
||||
)
|
||||
from dvc.main import main
|
||||
from dvc.remote import S3Remote
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.repo import Repo as DvcRepo
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import (
|
||||
StageFileBadNameError,
|
||||
StageFileDoesNotExistError,
|
||||
)
|
||||
from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
from dvc.system import System
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import walk_files
|
||||
@@ -403,17 +401,26 @@ class TestCheckoutSuggestGit(TestRepro):
|
||||
def test(self):
|
||||
|
||||
try:
|
||||
self.dvc.checkout(targets=["gitbranch"])
|
||||
self.dvc.checkout(targets="gitbranch")
|
||||
except DvcException as exc:
|
||||
self.assertIsInstance(exc, CheckoutErrorSuggestGit)
|
||||
self.assertIsInstance(exc.__cause__, StageFileDoesNotExistError)
|
||||
self.assertIsInstance(exc.__cause__, NoOutputOrStageError)
|
||||
self.assertIsInstance(
|
||||
exc.__cause__.__cause__, StageFileDoesNotExistError
|
||||
)
|
||||
|
||||
try:
|
||||
self.dvc.checkout(targets=self.FOO)
|
||||
except DvcException as exc:
|
||||
self.assertIsInstance(exc, CheckoutErrorSuggestGit)
|
||||
self.assertIsInstance(exc.__cause__, NoOutputOrStageError)
|
||||
self.assertIsNone(exc.__cause__.__cause__)
|
||||
|
||||
try:
|
||||
self.dvc.checkout(targets=[self.FOO])
|
||||
self.dvc.checkout(targets="looks-like-dvcfile.dvc")
|
||||
except DvcException as exc:
|
||||
self.assertIsInstance(exc, CheckoutErrorSuggestGit)
|
||||
self.assertIsInstance(exc.__cause__, StageFileBadNameError)
|
||||
self.assertIsInstance(exc.__cause__, StageFileDoesNotExistError)
|
||||
self.assertIsNone(exc.__cause__.__cause__)
|
||||
|
||||
|
||||
@@ -770,9 +777,51 @@ def test_checkout_for_external_outputs(tmp_dir, dvc):
|
||||
assert stats == {**empty_checkout, "modified": [str(file_path)]}
|
||||
|
||||
|
||||
def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
def test_checkouts_with_different_addressing(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen({"foo": "foo", "lorem": "lorem"})
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
run_copy("lorem", "ipsum", name="copy-lorem-ipsum")
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / "ipsum").unlink()
|
||||
assert set(dvc.checkout(PIPELINE_FILE)["added"]) == {"bar", "ipsum"}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / "ipsum").unlink()
|
||||
assert set(dvc.checkout(":")["added"]) == {"bar", "ipsum"}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.checkout("copy-foo-bar")["added"] == ["bar"]
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.checkout("dvc.yaml:copy-foo-bar")["added"] == ["bar"]
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.checkout(":copy-foo-bar")["added"] == ["bar"]
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / "data").mkdir()
|
||||
with (tmp_dir / "data").chdir():
|
||||
assert dvc.checkout(relpath(tmp_dir / "dvc.yaml") + ":copy-foo-bar")[
|
||||
"added"
|
||||
] == [relpath(tmp_dir / "bar")]
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.checkout("bar")["added"] == ["bar"]
|
||||
|
||||
|
||||
def test_checkouts_on_same_stage_name_and_output_name(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
run_copy("foo", "copy-foo-bar", name="make_collision")
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / "copy-foo-bar").unlink()
|
||||
assert dvc.checkout("copy-foo-bar")["added"] == ["bar"]
|
||||
assert dvc.checkout("./copy-foo-bar")["added"] == ["copy-foo-bar"]
|
||||
|
||||
|
||||
def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
tmp_dir.gen("lorem", "lorem")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import pytest
|
||||
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
from dvc.stage.exceptions import StageCommitError
|
||||
from dvc.utils.stage import dump_stage_file, load_stage_file
|
||||
|
||||
@@ -70,6 +71,7 @@ def test_commit_changed_md5(tmp_dir, dvc):
|
||||
dvc.commit(stage.path)
|
||||
|
||||
dvc.commit(stage.path, force=True)
|
||||
assert "md5" not in load_stage_file(stage.path)
|
||||
|
||||
|
||||
def test_commit_no_exec(tmp_dir, dvc):
|
||||
@@ -80,3 +82,16 @@ def test_commit_no_exec(tmp_dir, dvc):
|
||||
assert dvc.status(stage.path)
|
||||
dvc.commit(stage.path, force=True)
|
||||
assert dvc.status(stage.path) == {}
|
||||
|
||||
|
||||
def test_commit_pipeline_stage(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
|
||||
assert dvc.status(stage.addressing)
|
||||
assert dvc.commit(stage.addressing, force=True) == [stage]
|
||||
assert not dvc.status(stage.addressing)
|
||||
|
||||
# just to confirm different variants work
|
||||
assert dvc.commit(f":{stage.addressing}") == [stage]
|
||||
assert dvc.commit(f"{PIPELINE_FILE}:{stage.addressing}") == [stage]
|
||||
assert dvc.commit(PIPELINE_FILE) == [stage]
|
||||
|
||||
@@ -912,3 +912,44 @@ def test_pull_stats(tmp_dir, dvc, caplog, setup_remote):
|
||||
with caplog.at_level(level=logging.INFO, logger="dvc"):
|
||||
main(["pull"])
|
||||
assert "Everything is up to date." in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key,expected", [("all_tags", 2), ("all_branches", 3), ("all_commits", 3)]
|
||||
)
|
||||
def test_push_pull_all(tmp_dir, scm, dvc, setup_remote, key, expected):
|
||||
setup_remote(dvc)
|
||||
tmp_dir.dvc_gen({"foo": "foo"}, commit="first")
|
||||
scm.tag("v1")
|
||||
dvc.remove("foo.dvc")
|
||||
tmp_dir.dvc_gen({"bar": "bar"}, commit="second")
|
||||
scm.tag("v2")
|
||||
with tmp_dir.branch("branch", new=True):
|
||||
dvc.remove("bar.dvc")
|
||||
tmp_dir.dvc_gen({"baz": "baz"}, commit="branch")
|
||||
|
||||
assert dvc.push(**{key: True}) == expected
|
||||
|
||||
clean(["foo", "bar", "baz"], dvc)
|
||||
assert dvc.pull(**{key: True})["fetched"] == expected
|
||||
|
||||
|
||||
def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
|
||||
remote_path = setup_remote(dvc)
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
|
||||
|
||||
dvc.push("copy-foo-bar")
|
||||
assert len(recurse_list_dir(remote_path)) == 1
|
||||
# pushing everything so as we can check pull/fetch only downloads
|
||||
# from specified targets
|
||||
dvc.push()
|
||||
clean(["foo", "bar"], dvc)
|
||||
|
||||
dvc.pull("copy-foo-bar")
|
||||
assert (tmp_dir / "bar").exists()
|
||||
assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1
|
||||
clean(["bar"], dvc)
|
||||
|
||||
dvc.fetch("copy-foo-bar")
|
||||
assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
import pytest
|
||||
|
||||
from dvc.dvcfile import PIPELINE_FILE, Dvcfile
|
||||
from dvc.stage.exceptions import (
|
||||
StageFileDoesNotExistError,
|
||||
StageNameUnspecified,
|
||||
)
|
||||
from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
from dvc.stage.loader import StageNotFound
|
||||
|
||||
|
||||
@@ -163,37 +160,3 @@ def test_stage_collection(tmp_dir, dvc):
|
||||
single_stage=True,
|
||||
)
|
||||
assert {s for s in dvc.stages} == {stage1, stage3, stage2}
|
||||
|
||||
|
||||
def test_stage_filter(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
stage2 = run_copy("bar", "bax", name="copy-bar-bax")
|
||||
stage3 = run_copy("bax", "baz", name="copy-bax-baz")
|
||||
stages = Dvcfile(dvc, PIPELINE_FILE).stages
|
||||
assert set(stages.filter(None).values()) == {
|
||||
stage1,
|
||||
stage2,
|
||||
stage3,
|
||||
}
|
||||
assert set(stages.filter("copy-bar-bax").values()) == {stage2}
|
||||
assert stages.filter("copy-bar-bax").get("copy-bar-bax") == stage2
|
||||
with pytest.raises(StageNameUnspecified):
|
||||
stages.get(None)
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
assert stages["unknown"]
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
assert stages.filter("unknown")
|
||||
|
||||
|
||||
def test_stage_filter_in_singlestage_file(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", single_stage=True)
|
||||
dvcfile = Dvcfile(dvc, stage.path)
|
||||
assert set(dvcfile.stages.filter(None).values()) == {stage}
|
||||
assert dvcfile.stages.filter(None).get(None) == stage
|
||||
assert dvcfile.stages.filter("copy-bar-bax").get("copy-bar-bax") == stage
|
||||
assert dvcfile.stages.filter("copy-bar-bax").get(None) == stage
|
||||
assert set(dvcfile.stages.filter("unknown").values()) == {stage}
|
||||
|
||||
@@ -3,6 +3,7 @@ import os
|
||||
from mock import patch
|
||||
|
||||
from dvc.external_repo import external_repo
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.remote import LocalRemote
|
||||
from dvc.scm.git import Git
|
||||
from dvc.utils import relpath
|
||||
@@ -89,7 +90,10 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
|
||||
|
||||
dest = tmp_dir / "file"
|
||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||
repo.pull_to(os.path.join("subdir", "file"), dest)
|
||||
_, _, save_infos = repo.fetch_external(
|
||||
[os.path.join("subdir", "file")]
|
||||
)
|
||||
repo.cache.local.checkout(PathInfo(dest), save_infos[0])
|
||||
|
||||
assert dest.is_file()
|
||||
assert dest.read_text() == "contents"
|
||||
|
||||
@@ -68,7 +68,7 @@ class TestGCBranchesTags(TestDvcGit):
|
||||
self.dvc.scm.tag("v1.0")
|
||||
|
||||
self.dvc.scm.checkout("test", create_new=True)
|
||||
self.dvc.remove(stages[0].relpath, outs_only=True)
|
||||
self.dvc.remove(stages[0].relpath, dvc_only=True)
|
||||
with open(fname, "w+") as fobj:
|
||||
fobj.write("test")
|
||||
stages = self.dvc.add(fname)
|
||||
@@ -77,7 +77,7 @@ class TestGCBranchesTags(TestDvcGit):
|
||||
self.dvc.scm.commit("test")
|
||||
|
||||
self.dvc.scm.checkout("master")
|
||||
self.dvc.remove(stages[0].relpath, outs_only=True)
|
||||
self.dvc.remove(stages[0].relpath, dvc_only=True)
|
||||
with open(fname, "w+") as fobj:
|
||||
fobj.write("trash")
|
||||
stages = self.dvc.add(fname)
|
||||
@@ -85,7 +85,7 @@ class TestGCBranchesTags(TestDvcGit):
|
||||
self.dvc.scm.add([".gitignore", stages[0].relpath])
|
||||
self.dvc.scm.commit("trash")
|
||||
|
||||
self.dvc.remove(stages[0].relpath, outs_only=True)
|
||||
self.dvc.remove(stages[0].relpath, dvc_only=True)
|
||||
with open(fname, "w+") as fobj:
|
||||
fobj.write("master")
|
||||
stages = self.dvc.add(fname)
|
||||
@@ -219,6 +219,8 @@ def test_gc_no_unpacked_dir(tmp_dir, dvc):
|
||||
dir_stages[0].outs[0].cache_path + LocalRemote.UNPACKED_DIR_SUFFIX
|
||||
)
|
||||
|
||||
# older (pre 1.0) versions of dvc used to generate this dir
|
||||
shutil.copytree("dir", unpackeddir)
|
||||
assert os.path.exists(unpackeddir)
|
||||
|
||||
dvc.gc(force=True, workspace=True)
|
||||
|
||||
@@ -157,17 +157,13 @@ def test_get_to_dir(tmp_dir, erepo_dir, dname):
|
||||
assert (tmp_dir / dname / "file").read_text() == "contents"
|
||||
|
||||
|
||||
def test_get_from_non_dvc_master(tmp_dir, git_dir, caplog):
|
||||
def test_get_from_non_dvc_master(tmp_dir, git_dir):
|
||||
with git_dir.chdir(), git_dir.branch("branch", new=True):
|
||||
git_dir.init(dvc=True)
|
||||
git_dir.dvc_gen("some_file", "some text", commit="create some file")
|
||||
|
||||
caplog.clear()
|
||||
Repo.get(os.fspath(git_dir), "some_file", out="some_dst", rev="branch")
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="dvc"):
|
||||
Repo.get(os.fspath(git_dir), "some_file", out="some_dst", rev="branch")
|
||||
|
||||
assert caplog.text == ""
|
||||
assert (tmp_dir / "some_dst").read_text() == "some text"
|
||||
|
||||
|
||||
|
||||
+22
-19
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import shutil
|
||||
import textwrap
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -29,17 +30,19 @@ def match_files(files, expected_files):
|
||||
|
||||
|
||||
def create_dvc_pipeline(tmp_dir, dvc):
|
||||
script = os.linesep.join(
|
||||
[
|
||||
"from pathlib import Path",
|
||||
"Path({}).touch()".format(os.path.join("out", "file")),
|
||||
]
|
||||
script = textwrap.dedent(
|
||||
"""\
|
||||
import os, sys
|
||||
f = sys.argv[1]
|
||||
os.makedirs(os.path.dirname(f))
|
||||
open(f, "w+").close()
|
||||
"""
|
||||
)
|
||||
tmp_dir.scm_gen({"script.py": script}, commit="init")
|
||||
tmp_dir.dvc_gen({"dep": "content"}, commit="init dvc")
|
||||
dvc.run(
|
||||
**{
|
||||
"command": "python script.py",
|
||||
"cmd": "python script.py {}".format(os.path.join("out", "file")),
|
||||
"outs": [os.path.join("out", "file")],
|
||||
"deps": ["dep"],
|
||||
"fname": "out.dvc",
|
||||
@@ -123,11 +126,11 @@ def test_ls_repo_recursive(tmp_dir, dvc, scm):
|
||||
)
|
||||
|
||||
|
||||
def test_ls_repo_outs_only_recursive(tmp_dir, dvc, scm):
|
||||
def test_ls_repo_dvc_only_recursive(tmp_dir, dvc, scm):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
files = Repo.ls(os.fspath(tmp_dir), recursive=True, outs_only=True)
|
||||
files = Repo.ls(os.fspath(tmp_dir), recursive=True, dvc_only=True)
|
||||
match_files(
|
||||
files,
|
||||
(
|
||||
@@ -156,13 +159,13 @@ def test_ls_repo_with_path_dir(tmp_dir, dvc, scm):
|
||||
)
|
||||
|
||||
|
||||
def test_ls_repo_with_path_dir_outs_only_empty(tmp_dir, dvc, scm):
|
||||
def test_ls_repo_with_path_dir_dvc_only_empty(tmp_dir, dvc, scm):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
tmp_dir.scm_gen({"folder/.keep": "content"}, commit="add .keep")
|
||||
|
||||
with pytest.raises(PathMissingError):
|
||||
Repo.ls(os.fspath(tmp_dir), path="folder", outs_only=True)
|
||||
Repo.ls(os.fspath(tmp_dir), path="folder", dvc_only=True)
|
||||
|
||||
|
||||
def test_ls_repo_with_path_subdir(tmp_dir, dvc, scm):
|
||||
@@ -182,21 +185,21 @@ def test_ls_repo_with_path_subdir(tmp_dir, dvc, scm):
|
||||
)
|
||||
|
||||
|
||||
def test_ls_repo_with_path_subdir_outs_only(tmp_dir, dvc, scm):
|
||||
def test_ls_repo_with_path_subdir_dvc_only(tmp_dir, dvc, scm):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
path = os.path.join("data", "subcontent")
|
||||
files = Repo.ls(os.fspath(tmp_dir), path, outs_only=True)
|
||||
files = Repo.ls(os.fspath(tmp_dir), path, dvc_only=True)
|
||||
match_files(files, ((("data.xml",), True), (("statistics",), False),))
|
||||
|
||||
|
||||
def test_ls_repo_with_path_subdir_outs_only_recursive(tmp_dir, dvc, scm):
|
||||
def test_ls_repo_with_path_subdir_dvc_only_recursive(tmp_dir, dvc, scm):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
path = os.path.join("data", "subcontent")
|
||||
files = Repo.ls(os.fspath(tmp_dir), path, outs_only=True, recursive=True)
|
||||
files = Repo.ls(os.fspath(tmp_dir), path, dvc_only=True, recursive=True)
|
||||
match_files(
|
||||
files, ((("data.xml",), True), (("statistics", "data.csv"), True),)
|
||||
)
|
||||
@@ -226,10 +229,10 @@ def test_ls_repo_with_missed_path(tmp_dir, dvc, scm):
|
||||
|
||||
with pytest.raises(PathMissingError) as exc_info:
|
||||
Repo.ls(os.fspath(tmp_dir), path="missed_path")
|
||||
assert not exc_info.value.output_only
|
||||
assert not exc_info.value.dvc_only
|
||||
|
||||
|
||||
def test_ls_repo_with_missed_path_outs_only(tmp_dir, dvc, scm):
|
||||
def test_ls_repo_with_missed_path_dvc_only(tmp_dir, dvc, scm):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
@@ -238,9 +241,9 @@ def test_ls_repo_with_missed_path_outs_only(tmp_dir, dvc, scm):
|
||||
os.fspath(tmp_dir),
|
||||
path="missed_path",
|
||||
recursive=True,
|
||||
outs_only=True,
|
||||
dvc_only=True,
|
||||
)
|
||||
assert exc_info.value.output_only
|
||||
assert exc_info.value.dvc_only
|
||||
|
||||
|
||||
def test_ls_repo_with_removed_dvc_dir(tmp_dir, dvc, scm):
|
||||
@@ -440,5 +443,5 @@ def test_ls_shows_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
|
||||
dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK])
|
||||
dvc.scm.commit("add pipeline stage")
|
||||
|
||||
files = Repo.ls(os.curdir, outs_only=True)
|
||||
files = Repo.ls(os.curdir, dvc_only=True)
|
||||
match_files(files, ((("bar",), True),))
|
||||
|
||||
@@ -296,20 +296,20 @@ def test_pipeline_list_show_multistage(tmp_dir, dvc, run_copy, caplog):
|
||||
with caplog.at_level(logging.INFO, "dvc"):
|
||||
command._show("foobar.dvc", False, False, False)
|
||||
output = caplog.text.splitlines()
|
||||
assert "dvc.yaml:copy-foo-bar" in output[0]
|
||||
assert "copy-foo-bar" in output[0]
|
||||
assert "foobar.dvc" in output[1]
|
||||
|
||||
caplog.clear()
|
||||
with caplog.at_level(logging.INFO, "dvc"):
|
||||
command._show("dvc.yaml:copy-foo-bar", False, False, False)
|
||||
assert "dvc.yaml:copy-foo-bar" in caplog.text
|
||||
command._show("copy-foo-bar", False, False, False)
|
||||
assert "copy-foo-bar" in caplog.text
|
||||
assert "foobar.dvc" not in caplog.text
|
||||
|
||||
command = CmdPipelineList([])
|
||||
caplog.clear()
|
||||
with caplog.at_level(logging.INFO, "dvc"):
|
||||
command.run()
|
||||
assert "dvc.yaml:copy-foo-bar" in caplog.text
|
||||
assert "copy-foo-bar" in caplog.text
|
||||
assert "foobar.dvc" in caplog.text
|
||||
assert "1 pipelines in total"
|
||||
|
||||
@@ -320,13 +320,13 @@ def test_pipeline_ascii_multistage(tmp_dir, dvc, run_copy):
|
||||
run_copy("bar", "foobar", single_stage=True)
|
||||
command = CmdPipelineShow([])
|
||||
nodes, edges, _ = command._build_graph("foobar.dvc")
|
||||
assert set(nodes) == {"dvc.yaml:copy-foo-bar", "foobar.dvc"}
|
||||
assert set(nodes) == {"copy-foo-bar", "foobar.dvc"}
|
||||
assert set(edges) == {
|
||||
("foobar.dvc", "dvc.yaml:copy-foo-bar"),
|
||||
("foobar.dvc", "copy-foo-bar"),
|
||||
}
|
||||
|
||||
nodes, *_ = command._build_graph("dvc.yaml:copy-foo-bar")
|
||||
assert set(nodes) == {"dvc.yaml:copy-foo-bar"}
|
||||
nodes, *_ = command._build_graph("copy-foo-bar")
|
||||
assert set(nodes) == {"copy-foo-bar"}
|
||||
|
||||
|
||||
def test_pipeline_multi_outputs_stages(dvc):
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import errno
|
||||
import os
|
||||
import stat
|
||||
|
||||
import configobj
|
||||
import pytest
|
||||
@@ -238,7 +239,11 @@ def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
|
||||
|
||||
dvc.cache.local = None
|
||||
with pytest.raises(RemoteCacheRequiredError):
|
||||
dvc.run(deps=[os.fspath(external_dir)], single_stage=True)
|
||||
dvc.run(
|
||||
cmd="echo hello world",
|
||||
deps=[os.fspath(external_dir)],
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
|
||||
def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, setup_remote):
|
||||
@@ -284,3 +289,86 @@ def test_remote_modify_default(dvc):
|
||||
|
||||
assert repo_config["core"]["remote"] == remote_repo
|
||||
assert local_config["core"]["remote"] == remote_local
|
||||
|
||||
|
||||
def test_remote_rename(dvc):
|
||||
remote_name = "drive"
|
||||
remote_url = "gdrive://test/test"
|
||||
new_name = "new"
|
||||
other_name = "other"
|
||||
# prepare
|
||||
assert main(["remote", "add", remote_name, remote_url]) == 0
|
||||
config = dvc.config.load_one("repo")
|
||||
assert config["remote"][remote_name]["url"] == remote_url
|
||||
assert new_name not in config.get("remote", {})
|
||||
|
||||
# rename failed
|
||||
assert main(["remote", "rename", remote_name]) == 254
|
||||
assert main(["remote", "rename", new_name, other_name]) == 251
|
||||
config = dvc.config.load_one("repo")
|
||||
assert config["remote"][remote_name]["url"] == remote_url
|
||||
assert new_name not in config.get("remote", {})
|
||||
|
||||
# rename success
|
||||
assert main(["remote", "rename", remote_name, new_name]) == 0
|
||||
config = dvc.config.load_one("repo")
|
||||
assert remote_name not in config.get("remote", {})
|
||||
assert config["remote"][new_name]["url"] == remote_url
|
||||
|
||||
|
||||
def test_remote_duplicated(dvc):
|
||||
remote_name = "drive"
|
||||
remote_url = "gdrive://test/test"
|
||||
used_name = "overlap"
|
||||
another_url = "gdrive://test/test1"
|
||||
# prepare
|
||||
assert main(["remote", "add", remote_name, remote_url]) == 0
|
||||
assert main(["remote", "add", "--local", used_name, another_url]) == 0
|
||||
config = dvc.config.load_one("repo")
|
||||
assert config["remote"][remote_name]["url"] == remote_url
|
||||
local_config = dvc.config.load_one("local")
|
||||
assert local_config["remote"][used_name]["url"] == another_url
|
||||
|
||||
# rename duplicated
|
||||
assert main(["remote", "rename", remote_name, used_name]) == 251
|
||||
config = dvc.config.load_one("repo")
|
||||
assert config["remote"][remote_name]["url"] == remote_url
|
||||
local_config = dvc.config.load_one("local")
|
||||
assert local_config["remote"][used_name]["url"] == another_url
|
||||
|
||||
|
||||
def test_remote_default(dvc):
|
||||
remote_name = "drive"
|
||||
remote_url = "gdrive://test/test"
|
||||
new_name = "new"
|
||||
# prepare
|
||||
assert main(["remote", "add", "-d", remote_name, remote_url]) == 0
|
||||
assert main(["remote", "default", "--local", remote_name]) == 0
|
||||
config = dvc.config.load_one("repo")
|
||||
assert config["core"]["remote"] == remote_name
|
||||
assert config["remote"][remote_name]["url"] == remote_url
|
||||
assert new_name not in config.get("remote", {})
|
||||
local_config = dvc.config.load_one("local")
|
||||
assert local_config["core"]["remote"] == remote_name
|
||||
|
||||
# rename success
|
||||
assert main(["remote", "rename", remote_name, new_name]) == 0
|
||||
config = dvc.config.load_one("repo")
|
||||
assert remote_name not in config.get("remote", {})
|
||||
assert config["core"]["remote"] == new_name
|
||||
assert config["remote"][new_name]["url"] == remote_url
|
||||
assert remote_name not in config.get("remote", {})
|
||||
local_config = dvc.config.load_one("local")
|
||||
assert local_config["core"]["remote"] == new_name
|
||||
|
||||
|
||||
def test_protect_local_remote(tmp_dir, dvc, setup_remote):
|
||||
setup_remote(dvc, name="upstream")
|
||||
(stage,) = tmp_dir.dvc_gen("file", "file content")
|
||||
|
||||
dvc.push()
|
||||
remote = dvc.cloud.get_remote("upstream")
|
||||
remote_cache_file = remote.checksum_to_path_info(stage.outs[0].checksum)
|
||||
|
||||
assert os.path.exists(remote_cache_file)
|
||||
assert stat.S_IMODE(os.stat(remote_cache_file).st_mode) == 0o444
|
||||
|
||||
@@ -17,7 +17,7 @@ class TestRemove(TestDvc):
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
(stage_removed,) = self.dvc.remove(stage.path, outs_only=True)
|
||||
(stage_removed,) = self.dvc.remove(stage.path, dvc_only=True)
|
||||
|
||||
self.assertIsInstance(stage_removed, Stage)
|
||||
self.assertEqual(stage.path, stage_removed.path)
|
||||
|
||||
+251
-2
@@ -1,13 +1,24 @@
|
||||
import os
|
||||
from operator import itemgetter
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.cache import Cache
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
from dvc.exceptions import NoOutputOrStageError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.stage.exceptions import (
|
||||
StageFileDoesNotExistError,
|
||||
StageNameUnspecified,
|
||||
StageNotFound,
|
||||
)
|
||||
from dvc.system import System
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
|
||||
def test_destroy(tmp_dir, dvc, run_copy):
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
|
||||
dvc.config["cache"]["type"] = ["symlink"]
|
||||
dvc.cache = Cache(dvc)
|
||||
|
||||
@@ -88,6 +99,12 @@ def test_collect(tmp_dir, scm, dvc, run_copy):
|
||||
assert collect_outs("dvc.yaml:copy-foo-foobar", recursive=True) == {
|
||||
"foobar"
|
||||
}
|
||||
assert collect_outs("copy-foo-foobar") == {"foobar"}
|
||||
assert collect_outs("copy-foo-foobar", with_deps=True) == {
|
||||
"foobar",
|
||||
"foo",
|
||||
}
|
||||
assert collect_outs("copy-foo-foobar", recursive=True) == {"foobar"}
|
||||
|
||||
run_copy("foobar", "baz", name="copy-foobar-baz")
|
||||
assert collect_outs("dvc.yaml") == {"foobar", "baz"}
|
||||
@@ -98,6 +115,26 @@ def test_collect(tmp_dir, scm, dvc, run_copy):
|
||||
}
|
||||
|
||||
|
||||
def test_collect_dir_recursive(tmp_dir, dvc, run_head):
|
||||
tmp_dir.gen({"dir": {"foo": "foo"}})
|
||||
(stage1,) = dvc.add("dir", recursive=True)
|
||||
with (tmp_dir / "dir").chdir():
|
||||
stage2 = run_head("foo", name="copy-foo-bar")
|
||||
stage3 = run_head("foo-1", single_stage=True)
|
||||
assert set(dvc.collect("dir", recursive=True)) == {stage1, stage2, stage3}
|
||||
|
||||
|
||||
def test_collect_with_not_existing_output_or_stage_name(
|
||||
tmp_dir, dvc, run_copy
|
||||
):
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.collect("some_file")
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
with pytest.raises(StageNotFound):
|
||||
dvc.collect("some_file")
|
||||
|
||||
|
||||
def test_stages(tmp_dir, dvc):
|
||||
def stages():
|
||||
return {stage.relpath for stage in Repo(os.fspath(tmp_dir)).stages}
|
||||
@@ -113,3 +150,215 @@ def test_stages(tmp_dir, dvc):
|
||||
tmp_dir.gen(".dvcignore", "dir")
|
||||
|
||||
assert stages() == {"file.dvc"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stages(tmp_dir, run_copy):
|
||||
stage1, stage2 = tmp_dir.dvc_gen({"foo": "foo", "lorem": "lorem"})
|
||||
return {
|
||||
"foo-generate": stage1,
|
||||
"lorem-generate": stage2,
|
||||
"copy-foo-bar": run_copy("foo", "bar", single_stage=True),
|
||||
"copy-bar-foobar": run_copy("bar", "foobar", name="copy-bar-foobar"),
|
||||
"copy-lorem-ipsum": run_copy("lorem", "ipsum", name="lorem-ipsum"),
|
||||
}
|
||||
|
||||
|
||||
def test_collect_granular_with_no_target(tmp_dir, dvc, stages):
|
||||
assert set(map(itemgetter(0), dvc.collect_granular())) == set(
|
||||
stages.values()
|
||||
)
|
||||
assert list(map(itemgetter(1), dvc.collect_granular())) == [None] * len(
|
||||
stages
|
||||
)
|
||||
|
||||
|
||||
def test_collect_granular_with_target(tmp_dir, dvc, stages):
|
||||
assert dvc.collect_granular("bar.dvc") == [(stages["copy-foo-bar"], None)]
|
||||
assert dvc.collect_granular(PIPELINE_FILE) == [
|
||||
(stages["copy-bar-foobar"], None),
|
||||
(stages["copy-lorem-ipsum"], None),
|
||||
]
|
||||
assert dvc.collect_granular(":") == [
|
||||
(stages["copy-bar-foobar"], None),
|
||||
(stages["copy-lorem-ipsum"], None),
|
||||
]
|
||||
assert dvc.collect_granular("copy-bar-foobar") == [
|
||||
(stages["copy-bar-foobar"], None)
|
||||
]
|
||||
assert dvc.collect_granular(":copy-bar-foobar") == [
|
||||
(stages["copy-bar-foobar"], None)
|
||||
]
|
||||
assert dvc.collect_granular("dvc.yaml:copy-bar-foobar") == [
|
||||
(stages["copy-bar-foobar"], None)
|
||||
]
|
||||
|
||||
with (tmp_dir / dvc.DVC_DIR).chdir():
|
||||
assert dvc.collect_granular(
|
||||
relpath(tmp_dir / PIPELINE_FILE) + ":copy-bar-foobar"
|
||||
) == [(stages["copy-bar-foobar"], None)]
|
||||
|
||||
assert dvc.collect_granular("foobar") == [
|
||||
(stages["copy-bar-foobar"], PathInfo(tmp_dir / "foobar"))
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target",
|
||||
[
|
||||
"not_existing.dvc",
|
||||
"not_existing.dvc:stage_name",
|
||||
"not_existing/dvc.yaml",
|
||||
"not_existing/dvc.yaml:stage_name",
|
||||
],
|
||||
)
|
||||
def test_collect_with_not_existing_dvcfile(tmp_dir, dvc, target):
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.collect_granular(target)
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.collect(target)
|
||||
|
||||
|
||||
def test_collect_granular_with_not_existing_output_or_stage_name(tmp_dir, dvc):
|
||||
with pytest.raises(NoOutputOrStageError):
|
||||
dvc.collect_granular("some_file")
|
||||
with pytest.raises(NoOutputOrStageError):
|
||||
dvc.collect_granular("some_file", recursive=True)
|
||||
|
||||
|
||||
def test_collect_granular_with_deps(tmp_dir, dvc, stages):
|
||||
assert set(
|
||||
map(itemgetter(0), dvc.collect_granular("bar.dvc", with_deps=True))
|
||||
) == {stages["copy-foo-bar"], stages["foo-generate"]}
|
||||
assert set(
|
||||
map(
|
||||
itemgetter(0),
|
||||
dvc.collect_granular("copy-bar-foobar", with_deps=True),
|
||||
)
|
||||
) == {
|
||||
stages["copy-bar-foobar"],
|
||||
stages["copy-foo-bar"],
|
||||
stages["foo-generate"],
|
||||
}
|
||||
assert set(
|
||||
map(
|
||||
itemgetter(0), dvc.collect_granular(PIPELINE_FILE, with_deps=True),
|
||||
)
|
||||
) == set(stages.values())
|
||||
|
||||
|
||||
def test_collect_granular_same_output_name_stage_name(tmp_dir, dvc, run_copy):
|
||||
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
(stage2,) = tmp_dir.dvc_gen("copy-foo-bar", "copy-foo-bar")
|
||||
stage3 = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
|
||||
assert dvc.collect_granular("copy-foo-bar") == [(stage3, None)]
|
||||
|
||||
coll = dvc.collect_granular("copy-foo-bar", with_deps=True)
|
||||
assert set(map(itemgetter(0), coll)) == {stage3, stage1}
|
||||
assert list(map(itemgetter(1), coll)) == [None] * 2
|
||||
|
||||
assert dvc.collect_granular("./copy-foo-bar") == [
|
||||
(stage2, PathInfo(tmp_dir / "copy-foo-bar"))
|
||||
]
|
||||
assert dvc.collect_granular("./copy-foo-bar", with_deps=True) == [
|
||||
(stage2, PathInfo(tmp_dir / "copy-foo-bar"))
|
||||
]
|
||||
|
||||
|
||||
def test_collect_granular_priority_on_collision(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen({"dir": {"foo": "foo"}, "foo": "foo"})
|
||||
(stage1,) = dvc.add("dir", recursive=True)
|
||||
stage2 = run_copy("foo", "bar", name="dir")
|
||||
|
||||
assert dvc.collect_granular("dir") == [(stage2, None)]
|
||||
assert dvc.collect_granular("dir", recursive=True) == [(stage1, None)]
|
||||
|
||||
remove(tmp_dir / "dir")
|
||||
|
||||
assert dvc.collect_granular("dir") == [(stage2, None)]
|
||||
assert dvc.collect_granular("dir", recursive=True) == [(stage2, None)]
|
||||
|
||||
|
||||
def test_collect_granular_collision_output_dir_stage_name(
|
||||
tmp_dir, dvc, run_copy
|
||||
):
|
||||
stage1, *_ = tmp_dir.dvc_gen({"dir": {"foo": "foo"}, "foo": "foo"})
|
||||
stage3 = run_copy("foo", "bar", name="dir")
|
||||
|
||||
assert dvc.collect_granular("dir") == [(stage3, None)]
|
||||
assert not dvc.collect_granular("dir", recursive=True)
|
||||
assert dvc.collect_granular("./dir") == [
|
||||
(stage1, PathInfo(tmp_dir / "dir"))
|
||||
]
|
||||
|
||||
|
||||
def test_collect_granular_not_existing_stage_name(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
(stage,) = tmp_dir.dvc_gen("copy-foo-bar", "copy-foo-bar")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
|
||||
assert dvc.collect_granular("copy-foo-bar.dvc:stage_name_not_needed") == [
|
||||
(stage, None)
|
||||
]
|
||||
with pytest.raises(StageNotFound):
|
||||
dvc.collect_granular("dvc.yaml:does-not-exist")
|
||||
|
||||
|
||||
def test_get_stages(tmp_dir, dvc, run_copy):
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.get_stages()
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
|
||||
|
||||
assert set(dvc.get_stages()) == {stage1, stage2}
|
||||
assert set(dvc.get_stages(path=PIPELINE_FILE)) == {stage1, stage2}
|
||||
assert set(dvc.get_stages(name="copy-bar-foobar")) == {stage2}
|
||||
assert set(dvc.get_stages(path=PIPELINE_FILE, name="copy-bar-foobar")) == {
|
||||
stage2
|
||||
}
|
||||
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.get_stages(path=relpath(tmp_dir / ".." / PIPELINE_FILE))
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
dvc.get_stages(path=PIPELINE_FILE, name="copy")
|
||||
|
||||
|
||||
def test_get_stages_old_dvcfile(tmp_dir, dvc):
|
||||
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
assert set(dvc.get_stages("foo.dvc")) == {stage1}
|
||||
assert set(dvc.get_stages("foo.dvc", name="foo-generate")) == {stage1}
|
||||
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.get_stages(path=relpath(tmp_dir / ".." / "foo.dvc"))
|
||||
|
||||
|
||||
def test_get_stage(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
|
||||
with pytest.raises(StageNameUnspecified):
|
||||
dvc.get_stage()
|
||||
|
||||
with pytest.raises(StageNameUnspecified):
|
||||
dvc.get_stage(path=PIPELINE_FILE)
|
||||
|
||||
assert dvc.get_stage(path=PIPELINE_FILE, name="copy-foo-bar") == stage1
|
||||
assert dvc.get_stage(name="copy-foo-bar") == stage1
|
||||
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.get_stage(path="something.yaml", name="name")
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
dvc.get_stage(name="random_name")
|
||||
|
||||
|
||||
def test_get_stage_single_stage_dvcfile(tmp_dir, dvc):
|
||||
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
assert dvc.get_stage("foo.dvc") == stage1
|
||||
assert dvc.get_stage("foo.dvc", name="jpt") == stage1
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.get_stage(path="bar.dvc", name="name")
|
||||
|
||||
+45
-21
@@ -56,7 +56,7 @@ class SingleStageRun:
|
||||
|
||||
@staticmethod
|
||||
def _get_stage_target(stage):
|
||||
return stage.path
|
||||
return stage.addressing
|
||||
|
||||
|
||||
class TestRepro(SingleStageRun, TestDvc):
|
||||
@@ -264,9 +264,11 @@ class TestReproDepDirWithOutputsUnderIt(SingleStageRun, TestDvc):
|
||||
self.assertEqual(len(stages), 1)
|
||||
self.assertTrue(stages[0] is not None)
|
||||
|
||||
deps = [self.DATA, self.DATA_SUB]
|
||||
stage = self.dvc.run(
|
||||
cmd="ls {}".format(" ".join(deps)),
|
||||
fname="dvcfile2.dvc",
|
||||
deps=[self.DATA, self.DATA_SUB],
|
||||
deps=deps,
|
||||
single_stage=True,
|
||||
)
|
||||
self.assertTrue(stage is not None)
|
||||
@@ -405,7 +407,17 @@ class TestReproDryNoExec(TestDvc):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(
|
||||
["run", "--no-exec", "--single-stage", "-f", DVC_FILE] + deps
|
||||
[
|
||||
"run",
|
||||
"--no-exec",
|
||||
"--single-stage",
|
||||
"-f",
|
||||
DVC_FILE,
|
||||
*deps,
|
||||
"ls {}".format(
|
||||
" ".join(dep for i, dep in enumerate(deps) if i % 2)
|
||||
),
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
@@ -971,7 +983,7 @@ class TestReproExternalBase(SingleStageRun, TestDvc):
|
||||
self.dvc.gc(workspace=True)
|
||||
self.assertEqual(self.dvc.status(), {})
|
||||
|
||||
self.dvc.remove(cmd_stage.path, outs_only=True)
|
||||
self.dvc.remove(cmd_stage.path, dvc_only=True)
|
||||
self.assertNotEqual(self.dvc.status([cmd_stage.addressing]), {})
|
||||
|
||||
self.dvc.checkout([cmd_stage.path], force=True)
|
||||
@@ -1070,7 +1082,7 @@ class TestReproExternalHDFS(HDFS, TestReproExternalBase):
|
||||
self.assertEqual(p.returncode, 0)
|
||||
|
||||
|
||||
@flaky(max_runs=3, min_passes=1)
|
||||
@flaky(max_runs=5, min_passes=1)
|
||||
class TestReproExternalSSH(SSH, TestReproExternalBase):
|
||||
_dir = None
|
||||
cache_type = "copy"
|
||||
@@ -1095,23 +1107,33 @@ class TestReproExternalSSH(SSH, TestReproExternalBase):
|
||||
def write(self, bucket, key, body):
|
||||
path = posixpath.join(self._dir, key)
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.load_system_host_keys()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect("127.0.0.1")
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
ssh = None
|
||||
sftp = None
|
||||
try:
|
||||
sftp.stat(path)
|
||||
sftp.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.load_system_host_keys()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect("127.0.0.1")
|
||||
|
||||
stdin, stdout, stderr = ssh.exec_command(f"mkdir -p $(dirname {path})")
|
||||
self.assertEqual(stdout.channel.recv_exit_status(), 0)
|
||||
sftp = ssh.open_sftp()
|
||||
try:
|
||||
sftp.stat(path)
|
||||
sftp.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
with sftp.open(path, "w+") as fobj:
|
||||
fobj.write(body)
|
||||
stdin, stdout, stderr = ssh.exec_command(
|
||||
f"mkdir -p $(dirname {path})"
|
||||
)
|
||||
self.assertEqual(stdout.channel.recv_exit_status(), 0)
|
||||
|
||||
with sftp.open(path, "w+") as fobj:
|
||||
fobj.write(body)
|
||||
finally:
|
||||
if sftp:
|
||||
sftp.close()
|
||||
if ssh:
|
||||
ssh.close()
|
||||
|
||||
|
||||
class TestReproExternalLOCAL(Local, TestReproExternalBase):
|
||||
@@ -1223,7 +1245,7 @@ class TestReproExternalHTTP(TestReproExternalBase):
|
||||
self.assertTrue(os.path.exists(run_output))
|
||||
|
||||
# Pull
|
||||
self.dvc.remove(import_stage.path, outs_only=True)
|
||||
self.dvc.remove(import_stage.path, dvc_only=True)
|
||||
self.assertFalse(os.path.exists(import_output))
|
||||
|
||||
shutil.move(self.local_cache, cache_id)
|
||||
@@ -1455,9 +1477,11 @@ def repro_dir(tmp_dir, dvc, run_copy):
|
||||
stages["dir_file_copy"] = stage
|
||||
|
||||
last_stage = tmp_dir / "dir" / DVC_FILE
|
||||
deps = [os.fspath(origin_copy_2), os.fspath(dir_file_copy)]
|
||||
stage = dvc.run(
|
||||
cmd="echo {}".format(" ".join(deps)),
|
||||
fname=os.fspath(last_stage),
|
||||
deps=[os.fspath(origin_copy_2), os.fspath(dir_file_copy)],
|
||||
deps=deps,
|
||||
single_stage=True,
|
||||
)
|
||||
assert stage is not None
|
||||
|
||||
@@ -32,7 +32,7 @@ class MultiStageRun:
|
||||
|
||||
@staticmethod
|
||||
def _get_stage_target(stage):
|
||||
return stage.path + ":" + stage.name
|
||||
return stage.addressing
|
||||
|
||||
|
||||
class TestReproFailMultiStage(MultiStageRun, test_repro.TestReproFail):
|
||||
@@ -312,15 +312,13 @@ def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy):
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", name="copy-file")
|
||||
target = ":copy-file"
|
||||
target = "copy-file"
|
||||
assert not dvc.reproduce(target)
|
||||
|
||||
stage.cmd = " ".join(stage.cmd.split()) # change cmd spacing by two
|
||||
PipelineFile(dvc, PIPELINE_FILE)._dump_pipeline_file(stage)
|
||||
|
||||
assert dvc.status([target]) == {
|
||||
PIPELINE_FILE + target: ["changed command"]
|
||||
}
|
||||
assert dvc.status([target]) == {target: ["changed command"]}
|
||||
assert dvc.reproduce(target)[0] == stage
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ import os
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from dvc.repo import Repo
|
||||
from dvc.stage.exceptions import DuplicateStageName, InvalidStageName
|
||||
|
||||
|
||||
@@ -19,6 +21,30 @@ def test_run_with_name(tmp_dir, dvc, run_copy):
|
||||
assert os.path.exists(PIPELINE_LOCK)
|
||||
|
||||
|
||||
def test_run_no_exec(tmp_dir, dvc, run_copy):
|
||||
from dvc.stage import PipelineStage
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
assert not os.path.exists(PIPELINE_FILE)
|
||||
stage = run_copy("foo", "bar", name="copy-foo-to-bar", no_exec=True)
|
||||
assert isinstance(stage, PipelineStage)
|
||||
assert stage.name == "copy-foo-to-bar"
|
||||
assert os.path.exists(PIPELINE_FILE)
|
||||
assert not os.path.exists(PIPELINE_LOCK)
|
||||
|
||||
data, _ = stage.dvcfile._load()
|
||||
assert data == {
|
||||
"stages": {
|
||||
"copy-foo-to-bar": {
|
||||
"cmd": "python copy.py foo bar",
|
||||
"deps": ["copy.py", "foo"],
|
||||
"outs": ["bar"],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_run_with_multistage_and_single_stage(tmp_dir, dvc, run_copy):
|
||||
from dvc.stage import PipelineStage, Stage
|
||||
|
||||
@@ -252,3 +278,39 @@ def test_run_params_custom_file(tmp_dir, dvc):
|
||||
assert data["stages"]["read_params"]["params"] == [
|
||||
{"params2.yaml": ["lists"]}
|
||||
]
|
||||
|
||||
|
||||
def test_run_params_no_exec(tmp_dir, dvc):
|
||||
from dvc.dependency import ParamsDependency
|
||||
|
||||
with (tmp_dir / "params2.yaml").open("w+") as f:
|
||||
yaml.dump(supported_params, f)
|
||||
|
||||
stage = dvc.run(
|
||||
name="read_params",
|
||||
params=["params2.yaml:lists"],
|
||||
cmd="cat params2.yaml",
|
||||
no_exec=True,
|
||||
)
|
||||
|
||||
isinstance(stage.deps[0], ParamsDependency)
|
||||
assert stage.deps[0].params == ["lists"]
|
||||
assert not stage.dvcfile._lockfile.exists()
|
||||
|
||||
data, _ = stage.dvcfile._load()
|
||||
assert data["stages"]["read_params"]["params"] == [
|
||||
{"params2.yaml": ["lists"]}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[
|
||||
{"outs": ["foo"], "deps": ["bar"]},
|
||||
{"outs": ["foo"], "deps": ["bar"], "name": "copy-foo-bar"},
|
||||
],
|
||||
)
|
||||
def test_run_without_cmd(kwargs):
|
||||
with pytest.raises(InvalidArgumentError) as exc:
|
||||
Repo().run(**kwargs)
|
||||
assert "command is not specified" == str(exc.value)
|
||||
|
||||
@@ -76,7 +76,7 @@ class TestRun(TestDvc):
|
||||
class TestRunEmpty(TestDvc):
|
||||
def test(self):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="echo hello world",
|
||||
deps=[],
|
||||
outs=[],
|
||||
outs_no_cache=[],
|
||||
@@ -91,7 +91,7 @@ class TestRunMissingDep(TestDvc):
|
||||
|
||||
with self.assertRaises(DependencyDoesNotExistError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
deps=["non-existing-dep"],
|
||||
outs=[],
|
||||
outs_no_cache=[],
|
||||
@@ -118,7 +118,7 @@ class TestRunCircularDependency(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(CircularDependencyError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
deps=[self.FOO],
|
||||
outs=[self.FOO],
|
||||
fname="circular-dependency.dvc",
|
||||
@@ -128,7 +128,7 @@ class TestRunCircularDependency(TestDvc):
|
||||
def test_outs_no_cache(self):
|
||||
with self.assertRaises(CircularDependencyError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
deps=[self.FOO],
|
||||
outs_no_cache=[self.FOO],
|
||||
fname="circular-dependency.dvc",
|
||||
@@ -138,7 +138,7 @@ class TestRunCircularDependency(TestDvc):
|
||||
def test_non_normalized_paths(self):
|
||||
with self.assertRaises(CircularDependencyError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
deps=["./foo"],
|
||||
outs=["foo"],
|
||||
fname="circular-dependency.dvc",
|
||||
@@ -173,7 +173,7 @@ class TestRunDuplicatedArguments(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(ArgumentDuplicationError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
deps=[],
|
||||
outs=[self.FOO, self.FOO],
|
||||
fname="circular-dependency.dvc",
|
||||
@@ -183,7 +183,7 @@ class TestRunDuplicatedArguments(TestDvc):
|
||||
def test_outs_no_cache(self):
|
||||
with self.assertRaises(ArgumentDuplicationError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
outs=[self.FOO],
|
||||
outs_no_cache=[self.FOO],
|
||||
fname="circular-dependency.dvc",
|
||||
@@ -193,7 +193,7 @@ class TestRunDuplicatedArguments(TestDvc):
|
||||
def test_non_normalized_paths(self):
|
||||
with self.assertRaises(ArgumentDuplicationError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
deps=[],
|
||||
outs=["foo", "./foo"],
|
||||
fname="circular-dependency.dvc",
|
||||
@@ -204,22 +204,30 @@ class TestRunDuplicatedArguments(TestDvc):
|
||||
class TestRunStageInsideOutput(TestDvc):
|
||||
def test_cwd(self):
|
||||
self.dvc.run(
|
||||
cmd="", deps=[], outs=[self.DATA_DIR], single_stage=True,
|
||||
cmd=f"mkdir {self.DATA_DIR}",
|
||||
deps=[],
|
||||
outs=[self.DATA_DIR],
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
with self.assertRaises(StagePathAsOutputError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
fname=os.path.join(self.DATA_DIR, "inside-cwd.dvc"),
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
def test_file_name(self):
|
||||
self.dvc.run(cmd="", deps=[], outs=[self.DATA_DIR], single_stage=True)
|
||||
self.dvc.run(
|
||||
cmd=f"mkdir {self.DATA_DIR}",
|
||||
deps=[],
|
||||
outs=[self.DATA_DIR],
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
with self.assertRaises(StagePathAsOutputError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
outs=[self.FOO],
|
||||
fname=os.path.join(self.DATA_DIR, "inside-cwd.dvc"),
|
||||
single_stage=True,
|
||||
@@ -229,30 +237,30 @@ class TestRunStageInsideOutput(TestDvc):
|
||||
class TestRunBadCwd(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(StagePathOutsideError):
|
||||
self.dvc.run(cmd="", wdir=self.mkdtemp(), single_stage=True)
|
||||
self.dvc.run(cmd="command", wdir=self.mkdtemp(), single_stage=True)
|
||||
|
||||
def test_same_prefix(self):
|
||||
with self.assertRaises(StagePathOutsideError):
|
||||
path = f"{self._root_dir}-{uuid.uuid4()}"
|
||||
os.mkdir(path)
|
||||
self.dvc.run(cmd="", wdir=path, single_stage=True)
|
||||
self.dvc.run(cmd="command", wdir=path, single_stage=True)
|
||||
|
||||
|
||||
class TestRunBadWdir(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(StagePathOutsideError):
|
||||
self.dvc.run(cmd="", wdir=self.mkdtemp(), single_stage=True)
|
||||
self.dvc.run(cmd="command", wdir=self.mkdtemp(), single_stage=True)
|
||||
|
||||
def test_same_prefix(self):
|
||||
with self.assertRaises(StagePathOutsideError):
|
||||
path = f"{self._root_dir}-{uuid.uuid4()}"
|
||||
os.mkdir(path)
|
||||
self.dvc.run(cmd="", wdir=path, single_stage=True)
|
||||
self.dvc.run(cmd="command", wdir=path, single_stage=True)
|
||||
|
||||
def test_not_found(self):
|
||||
with self.assertRaises(StagePathNotFoundError):
|
||||
path = os.path.join(self._root_dir, str(uuid.uuid4()))
|
||||
self.dvc.run(cmd="", wdir=path, single_stage=True)
|
||||
self.dvc.run(cmd="command", wdir=path, single_stage=True)
|
||||
|
||||
def test_not_dir(self):
|
||||
with self.assertRaises(StagePathNotDirectoryError):
|
||||
@@ -261,7 +269,7 @@ class TestRunBadWdir(TestDvc):
|
||||
path = os.path.join(path, str(uuid.uuid4()))
|
||||
open(path, "a").close()
|
||||
self.dvc.run(
|
||||
cmd="", wdir=path, single_stage=True,
|
||||
cmd="command", wdir=path, single_stage=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -269,7 +277,7 @@ class TestRunBadName(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(StagePathOutsideError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
fname=os.path.join(self.mkdtemp(), self.FOO + DVC_FILE_SUFFIX),
|
||||
single_stage=True,
|
||||
)
|
||||
@@ -279,7 +287,7 @@ class TestRunBadName(TestDvc):
|
||||
path = f"{self._root_dir}-{uuid.uuid4()}"
|
||||
os.mkdir(path)
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
fname=os.path.join(path, self.FOO + DVC_FILE_SUFFIX),
|
||||
single_stage=True,
|
||||
)
|
||||
@@ -288,7 +296,7 @@ class TestRunBadName(TestDvc):
|
||||
with self.assertRaises(StagePathNotFoundError):
|
||||
path = os.path.join(self._root_dir, str(uuid.uuid4()))
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
fname=os.path.join(path, self.FOO + DVC_FILE_SUFFIX),
|
||||
single_stage=True,
|
||||
)
|
||||
@@ -574,6 +582,7 @@ class TestCmdRunOverwrite(TestDvc):
|
||||
"out.dvc",
|
||||
"-d",
|
||||
self.BAR,
|
||||
f"cat {self.BAR}",
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
@@ -8,7 +8,7 @@ from dvc.main import main
|
||||
from dvc.output.local import LocalOutput
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.repo import Repo
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage import PipelineStage, Stage
|
||||
from dvc.stage.exceptions import StageFileFormatError
|
||||
from dvc.utils.stage import dump_stage_file, load_stage_file
|
||||
from tests.basic_env import TestDvc
|
||||
@@ -182,33 +182,62 @@ def test_meta_is_preserved(tmp_dir, dvc):
|
||||
|
||||
def test_parent_repo_collect_stages(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen({"subdir": {}})
|
||||
tmp_dir.gen({"deep": {"dir": {}}})
|
||||
subrepo_dir = tmp_dir / "subdir"
|
||||
deep_subrepo_dir = tmp_dir / "deep" / "dir"
|
||||
|
||||
with subrepo_dir.chdir():
|
||||
subrepo = Repo.init(subdir=True)
|
||||
subrepo_dir.gen("subrepo_file", "subrepo file content")
|
||||
subrepo.add("subrepo_file")
|
||||
|
||||
with deep_subrepo_dir.chdir():
|
||||
deep_subrepo = Repo.init(subdir=True)
|
||||
deep_subrepo_dir.gen("subrepo_file", "subrepo file content")
|
||||
deep_subrepo.add("subrepo_file")
|
||||
|
||||
stages = dvc.collect(None)
|
||||
subrepo_stages = subrepo.collect(None)
|
||||
deep_subrepo_stages = deep_subrepo.collect(None)
|
||||
|
||||
assert stages == []
|
||||
assert subrepo_stages != []
|
||||
assert deep_subrepo_stages != []
|
||||
|
||||
|
||||
def test_stage_addressing(tmp_dir, dvc, run_copy):
|
||||
def test_stage_strings_representation(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", single_stage=True)
|
||||
assert stage1.addressing == "bar.dvc"
|
||||
assert repr(stage1) == "Stage: 'bar.dvc'"
|
||||
assert str(stage1) == "stage: 'bar.dvc'"
|
||||
|
||||
stage2 = run_copy("bar", "baz", name="copy-bar-baz")
|
||||
assert stage2.addressing == "dvc.yaml:copy-bar-baz"
|
||||
assert stage2.addressing == "copy-bar-baz"
|
||||
assert repr(stage2) == "Stage: 'copy-bar-baz'"
|
||||
assert str(stage2) == "stage: 'copy-bar-baz'"
|
||||
|
||||
folder = tmp_dir / "dir"
|
||||
folder.mkdir()
|
||||
with folder.chdir():
|
||||
assert stage1.addressing == os.path.relpath(stage1.path)
|
||||
assert (
|
||||
stage2.addressing
|
||||
== os.path.relpath(stage2.path) + ":" + stage2.name
|
||||
)
|
||||
rel_path = os.path.relpath(stage1.path)
|
||||
assert stage1.addressing == rel_path
|
||||
assert repr(stage1) == f"Stage: '{rel_path}'"
|
||||
assert str(stage1) == f"stage: '{rel_path}'"
|
||||
|
||||
rel_path = os.path.relpath(stage2.path)
|
||||
assert stage2.addressing == f"{rel_path}:{stage2.name}"
|
||||
assert repr(stage2) == f"Stage: '{rel_path}:{stage2.name}'"
|
||||
assert str(stage2) == f"stage: '{rel_path}:{stage2.name}'"
|
||||
|
||||
|
||||
def test_stage_on_no_path_string_repr(tmp_dir, dvc):
|
||||
s = Stage(dvc)
|
||||
assert s.addressing == "No path"
|
||||
assert repr(s) == "Stage: 'No path'"
|
||||
assert str(s) == "stage: 'No path'"
|
||||
|
||||
p = PipelineStage(dvc, name="stage_name")
|
||||
assert p.addressing == "No path:stage_name"
|
||||
assert repr(p) == "Stage: 'No path:stage_name'"
|
||||
assert str(p) == "stage: 'No path:stage_name'"
|
||||
|
||||
@@ -73,12 +73,12 @@ def test_status_on_pipeline_stages(tmp_dir, dvc, run_copy):
|
||||
|
||||
stage.cmd = " ".join(stage.cmd.split())
|
||||
stage.dvcfile._dump_pipeline_file(stage)
|
||||
assert dvc.status() == {"dvc.yaml:copy-foo-bar": ["changed command"]}
|
||||
assert dvc.status("copy-foo-bar") == {"copy-foo-bar": ["changed command"]}
|
||||
|
||||
# delete outputs
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.status() == {
|
||||
"dvc.yaml:copy-foo-bar": [
|
||||
"copy-foo-bar": [
|
||||
{"changed outs": {"bar": "deleted"}},
|
||||
"changed command",
|
||||
]
|
||||
@@ -86,7 +86,7 @@ def test_status_on_pipeline_stages(tmp_dir, dvc, run_copy):
|
||||
(tmp_dir / "foo").unlink()
|
||||
assert dvc.status() == {
|
||||
"foo.dvc": [{"changed outs": {"foo": "deleted"}}],
|
||||
"dvc.yaml:copy-foo-bar": [
|
||||
"copy-foo-bar": [
|
||||
{"changed deps": {"foo": "deleted"}},
|
||||
{"changed outs": {"bar": "deleted"}},
|
||||
"changed command",
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
import os
|
||||
from os.path import join
|
||||
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.tree import RepoTree
|
||||
from dvc.scm import SCM
|
||||
from dvc.scm.git import GitTree
|
||||
from dvc.scm.tree import WorkingTree
|
||||
from dvc.utils.fs import remove
|
||||
from tests.basic_env import TestDir, TestGit, TestGitSubmodule
|
||||
|
||||
|
||||
@@ -172,3 +177,67 @@ class TestWalkInGit(AssertWalkEqualMixin, TestGit):
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_repotree_walk_fetch(tmp_dir, dvc, scm, setup_remote):
|
||||
setup_remote(dvc)
|
||||
out = tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="init")[0].outs[0]
|
||||
dvc.push()
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
tree = RepoTree(dvc, fetch=True)
|
||||
with dvc.state:
|
||||
for _, _, _ in tree.walk("dir"):
|
||||
pass
|
||||
|
||||
assert os.path.exists(out.cache_path)
|
||||
for entry in out.dir_cache:
|
||||
checksum = entry[out.remote.PARAM_CHECKSUM]
|
||||
assert os.path.exists(dvc.cache.local.checksum_to_path_info(checksum))
|
||||
|
||||
|
||||
def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, setup_remote):
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.gen({"dir": {"subdir": {"foo": "foo"}, "bar": "bar"}})
|
||||
erepo_dir.dvc_add("dir/subdir", commit="subdir")
|
||||
erepo_dir.scm_add("dir", commit="dir")
|
||||
setup_remote(erepo_dir.dvc)
|
||||
erepo_dir.dvc.push()
|
||||
|
||||
# test only cares that either fetch or stream are set so that DVC dirs are
|
||||
# walked.
|
||||
#
|
||||
# for this test, all file objects are being opened() and copied from tree
|
||||
# into dvc.cache, not fetched or streamed from a remote
|
||||
tree = RepoTree(erepo_dir.dvc, stream=True)
|
||||
expected = [
|
||||
tree.get_file_checksum(erepo_dir / path)
|
||||
for path in ("dir/bar", "dir/subdir/foo")
|
||||
]
|
||||
|
||||
with erepo_dir.dvc.state:
|
||||
cache = dvc.cache.local
|
||||
with cache.state:
|
||||
cache.save(PathInfo(erepo_dir / "dir"), None, tree=tree)
|
||||
for checksum in expected:
|
||||
assert os.path.exists(cache.checksum_to_path_info(checksum))
|
||||
|
||||
|
||||
def test_cleantree_subrepo(tmp_dir, dvc, scm, monkeypatch):
|
||||
tmp_dir.gen({"subdir": {}})
|
||||
subrepo_dir = tmp_dir / "subdir"
|
||||
with subrepo_dir.chdir():
|
||||
subrepo = Repo.init(subdir=True)
|
||||
subrepo_dir.gen({"foo": "foo", "dir": {"bar": "bar"}})
|
||||
|
||||
assert isinstance(dvc.tree, CleanTree)
|
||||
assert not dvc.tree.exists(subrepo_dir / "foo")
|
||||
assert not dvc.tree.isfile(subrepo_dir / "foo")
|
||||
assert not dvc.tree.exists(subrepo_dir / "dir")
|
||||
assert not dvc.tree.isdir(subrepo_dir / "dir")
|
||||
|
||||
assert isinstance(subrepo.tree, CleanTree)
|
||||
assert subrepo.tree.exists(subrepo_dir / "foo")
|
||||
assert subrepo.tree.isfile(subrepo_dir / "foo")
|
||||
assert subrepo.tree.exists(subrepo_dir / "dir")
|
||||
assert subrepo.tree.isdir(subrepo_dir / "dir")
|
||||
|
||||
@@ -17,7 +17,7 @@ def test_list(mocker):
|
||||
url = "local_dir"
|
||||
m = _test_cli(mocker, url)
|
||||
m.assert_called_once_with(
|
||||
url, None, recursive=False, rev=None, outs_only=False
|
||||
url, None, recursive=False, rev=None, dvc_only=False
|
||||
)
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ def test_list_recursive(mocker):
|
||||
url = "local_dir"
|
||||
m = _test_cli(mocker, url, "-R")
|
||||
m.assert_called_once_with(
|
||||
url, None, recursive=True, rev=None, outs_only=False
|
||||
url, None, recursive=True, rev=None, dvc_only=False
|
||||
)
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ def test_list_git_ssh_rev(mocker):
|
||||
url = "[email protected]:repo"
|
||||
m = _test_cli(mocker, url, "--rev", "123")
|
||||
m.assert_called_once_with(
|
||||
url, None, recursive=False, rev="123", outs_only=False
|
||||
url, None, recursive=False, rev="123", dvc_only=False
|
||||
)
|
||||
|
||||
|
||||
@@ -42,13 +42,13 @@ def test_list_targets(mocker):
|
||||
target = "subdir"
|
||||
m = _test_cli(mocker, url, target)
|
||||
m.assert_called_once_with(
|
||||
url, target, recursive=False, rev=None, outs_only=False
|
||||
url, target, recursive=False, rev=None, dvc_only=False
|
||||
)
|
||||
|
||||
|
||||
def test_list_outputs_only(mocker):
|
||||
url = "local_dir"
|
||||
m = _test_cli(mocker, url, None, "--outs-only")
|
||||
m = _test_cli(mocker, url, None, "--dvc-only")
|
||||
m.assert_called_once_with(
|
||||
url, None, recursive=False, rev=None, outs_only=True
|
||||
url, None, recursive=False, rev=None, dvc_only=True
|
||||
)
|
||||
|
||||
@@ -17,6 +17,9 @@ def test_metrics_diff(dvc, mocker):
|
||||
"--targets",
|
||||
"target1",
|
||||
"target2",
|
||||
"--show-md",
|
||||
"--no-path",
|
||||
"--old",
|
||||
]
|
||||
)
|
||||
assert cli_args.func == CmdMetricsDiff
|
||||
@@ -174,3 +177,41 @@ def test_metrics_diff_markdown():
|
||||
| metrics.yaml | a.d.e | 4 | 1 |
|
||||
| metrics.yaml | x.b | 6 | diff not supported |"""
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_diff_no_path():
|
||||
assert _show_diff(
|
||||
{
|
||||
"metrics.yaml": {
|
||||
"x.b": {"old": 5, "new": 6, "diff": 1},
|
||||
"a.d.e": {"old": 3, "new": 4, "diff": 1},
|
||||
"a.b.c": {"old": 1, "new": 2, "diff": 1},
|
||||
}
|
||||
},
|
||||
no_path=True,
|
||||
) == textwrap.dedent(
|
||||
"""\
|
||||
Metric Value Change
|
||||
a.b.c 2 1
|
||||
a.d.e 4 1
|
||||
x.b 6 1"""
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_diff_with_old():
|
||||
assert _show_diff(
|
||||
{
|
||||
"metrics.yaml": {
|
||||
"x.b": {"old": 5, "new": 6, "diff": 1},
|
||||
"a.d.e": {"old": 3, "new": 4, "diff": 1},
|
||||
"a.b.c": {"old": 1, "new": 2, "diff": 1},
|
||||
}
|
||||
},
|
||||
old=True,
|
||||
) == textwrap.dedent(
|
||||
"""\
|
||||
Path Metric Old New Change
|
||||
metrics.yaml a.b.c 1 2 1
|
||||
metrics.yaml a.d.e 3 4 1
|
||||
metrics.yaml x.b 5 6 1"""
|
||||
)
|
||||
|
||||
@@ -7,7 +7,16 @@ from dvc.command.params import CmdParamsDiff, _show_diff
|
||||
|
||||
def test_params_diff(dvc, mocker):
|
||||
cli_args = parse_args(
|
||||
["params", "diff", "HEAD~10", "HEAD~1", "--all", "--show-json"]
|
||||
[
|
||||
"params",
|
||||
"diff",
|
||||
"HEAD~10",
|
||||
"HEAD~1",
|
||||
"--all",
|
||||
"--show-json",
|
||||
"--show-md",
|
||||
"--no-path",
|
||||
]
|
||||
)
|
||||
assert cli_args.func == CmdParamsDiff
|
||||
|
||||
@@ -142,3 +151,22 @@ def test_params_diff_markdown():
|
||||
| params.yaml | a.d.e | None | 4 |
|
||||
| params.yaml | x.b | 5 | 6 |"""
|
||||
)
|
||||
|
||||
|
||||
def test_params_diff_no_path():
|
||||
assert _show_diff(
|
||||
{
|
||||
"params.yaml": {
|
||||
"x.b": {"old": 5, "new": 6},
|
||||
"a.d.e": {"old": 3, "new": 4},
|
||||
"a.b.c": {"old": 1, "new": 2},
|
||||
}
|
||||
},
|
||||
no_path=True,
|
||||
) == textwrap.dedent(
|
||||
"""\
|
||||
Param Old New
|
||||
a.b.c 1 2
|
||||
a.d.e 3 4
|
||||
x.b 5 6"""
|
||||
)
|
||||
|
||||
@@ -13,8 +13,6 @@ def test_metrics_diff(dvc, mocker):
|
||||
"template",
|
||||
"--targets",
|
||||
"datafile",
|
||||
"--select",
|
||||
"column1,column2",
|
||||
"--show-json",
|
||||
"-x",
|
||||
"x_field",
|
||||
@@ -43,8 +41,6 @@ def test_metrics_diff(dvc, mocker):
|
||||
targets=["datafile"],
|
||||
template="template",
|
||||
revs=["HEAD", "tag1", "tag2"],
|
||||
fields={"column1", "column2"},
|
||||
path=None,
|
||||
x_field="x_field",
|
||||
y_field="y_field",
|
||||
csv_header=True,
|
||||
@@ -63,8 +59,6 @@ def test_metrics_show(dvc, mocker):
|
||||
"result.extension",
|
||||
"-t",
|
||||
"template",
|
||||
"-s",
|
||||
"$.data",
|
||||
"--show-json",
|
||||
"--no-csv-header",
|
||||
"datafile",
|
||||
@@ -82,8 +76,6 @@ def test_metrics_show(dvc, mocker):
|
||||
cmd.repo,
|
||||
targets=["datafile"],
|
||||
template="template",
|
||||
fields=None,
|
||||
path="$.data",
|
||||
x_field=None,
|
||||
y_field=None,
|
||||
csv_header=False,
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
from dvc.dependency.azure import AzureDependency
|
||||
from tests.unit.dependency.test_local import TestLocalDependency
|
||||
|
||||
|
||||
class TestAzureDependency(TestLocalDependency):
|
||||
def _get_cls(self):
|
||||
return AzureDependency
|
||||
@@ -1,4 +1,8 @@
|
||||
import pytest
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.remote.azure import AzureRemote
|
||||
from tests.remotes import Azure
|
||||
|
||||
container_name = "container-name"
|
||||
connection_string = (
|
||||
@@ -9,13 +13,11 @@ connection_string = (
|
||||
)
|
||||
|
||||
|
||||
def test_init_compat(dvc):
|
||||
url = (
|
||||
"azure://ContainerName={container_name};{connection_string}"
|
||||
).format(
|
||||
container_name=container_name, connection_string=connection_string,
|
||||
)
|
||||
config = {"url": url}
|
||||
def test_init_env_var(monkeypatch, dvc):
|
||||
monkeypatch.setenv("AZURE_STORAGE_CONTAINER_NAME", container_name)
|
||||
monkeypatch.setenv("AZURE_STORAGE_CONNECTION_STRING", connection_string)
|
||||
|
||||
config = {"url": "azure://"}
|
||||
remote = AzureRemote(dvc, config)
|
||||
assert remote.path_info == "azure://" + container_name
|
||||
assert remote.connection_string == connection_string
|
||||
@@ -28,3 +30,19 @@ def test_init(dvc):
|
||||
remote = AzureRemote(dvc, config)
|
||||
assert remote.path_info == url
|
||||
assert remote.connection_string == connection_string
|
||||
|
||||
|
||||
def test_get_file_checksum(tmp_dir):
|
||||
if not Azure.should_test():
|
||||
pytest.skip("no azurite running")
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
|
||||
remote = AzureRemote(None, {})
|
||||
to_info = remote.path_cls(Azure.get_url())
|
||||
remote.upload(PathInfo("foo"), to_info)
|
||||
assert remote.exists(to_info)
|
||||
checksum = remote.get_file_checksum(to_info)
|
||||
assert checksum
|
||||
assert isinstance(checksum, str)
|
||||
assert checksum.strip("'").strip('"') == checksum
|
||||
|
||||
@@ -136,3 +136,11 @@ def test_cache_checksums(dvc):
|
||||
):
|
||||
checksums = list(remote.cache_checksums())
|
||||
assert checksums == ["123456"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"checksum, result",
|
||||
[(None, False), ("", False), ("3456.dir", True), ("3456", False)],
|
||||
)
|
||||
def test_is_dir_checksum(checksum, result):
|
||||
assert BaseRemote.is_dir_checksum(checksum) == result
|
||||
|
||||
@@ -87,6 +87,21 @@ def test_collect_optimization(tmp_dir, dvc, mocker):
|
||||
dvc.collect_granular(stage.path)
|
||||
|
||||
|
||||
def test_collect_optimization_on_stage_name(tmp_dir, dvc, mocker, run_copy):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
# Forget cached stages and graph and error out on collection
|
||||
dvc._reset()
|
||||
mocker.patch(
|
||||
"dvc.repo.Repo.stages",
|
||||
property(raiser(Exception("Should not collect"))),
|
||||
)
|
||||
|
||||
# Should read stage directly instead of collecting the whole graph
|
||||
assert dvc.collect("copy-foo-bar") == [stage]
|
||||
assert dvc.collect_granular("copy-foo-bar") == [(stage, None)]
|
||||
|
||||
|
||||
def test_skip_graph_checks(tmp_dir, dvc, mocker, run_copy):
|
||||
# See https://github.com/iterative/dvc/issues/2671 for more info
|
||||
mock_collect_graph = mocker.patch("dvc.repo.Repo._collect_graph")
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
|
||||
@@ -76,7 +79,21 @@ def test_isdir_mixed(tmp_dir, dvc):
|
||||
assert not tree.isfile("dir")
|
||||
|
||||
|
||||
def test_walk(tmp_dir, dvc):
|
||||
@pytest.mark.parametrize(
|
||||
"dvcfiles,extra_expected",
|
||||
[
|
||||
(False, []),
|
||||
(
|
||||
True,
|
||||
[
|
||||
PathInfo("dir") / "subdir1" / "foo1.dvc",
|
||||
PathInfo("dir") / "subdir1" / "bar1.dvc",
|
||||
PathInfo("dir") / "subdir2" / "foo2.dvc",
|
||||
],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_walk(tmp_dir, dvc, dvcfiles, extra_expected):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
"dir": {
|
||||
@@ -90,30 +107,32 @@ def test_walk(tmp_dir, dvc):
|
||||
tree = RepoTree(dvc)
|
||||
|
||||
expected = [
|
||||
os.path.join("dir", "subdir1"),
|
||||
os.path.join("dir", "subdir2"),
|
||||
os.path.join("dir", "subdir1", "foo1"),
|
||||
os.path.join("dir", "subdir1", "foo1.dvc"),
|
||||
os.path.join("dir", "subdir1", "bar1"),
|
||||
os.path.join("dir", "subdir1", "bar1.dvc"),
|
||||
os.path.join("dir", "subdir2", "foo2"),
|
||||
os.path.join("dir", "subdir2", "foo2.dvc"),
|
||||
os.path.join("dir", "foo"),
|
||||
os.path.join("dir", "bar"),
|
||||
PathInfo("dir") / "subdir1",
|
||||
PathInfo("dir") / "subdir2",
|
||||
PathInfo("dir") / "subdir1" / "foo1",
|
||||
PathInfo("dir") / "subdir1" / "bar1",
|
||||
PathInfo("dir") / "subdir2" / "foo2",
|
||||
PathInfo("dir") / "foo",
|
||||
PathInfo("dir") / "bar",
|
||||
]
|
||||
|
||||
actual = []
|
||||
for root, dirs, files in tree.walk("dir"):
|
||||
for root, dirs, files in tree.walk("dir", dvcfiles=dvcfiles):
|
||||
for entry in dirs + files:
|
||||
actual.append(os.path.join(root, entry))
|
||||
|
||||
expected = [str(path) for path in expected + extra_expected]
|
||||
assert set(actual) == set(expected)
|
||||
assert len(actual) == len(expected)
|
||||
|
||||
|
||||
def test_isdvc(tmp_dir, dvc):
|
||||
tmp_dir.gen({"foo": "foo", "bar": "bar"})
|
||||
tmp_dir.gen({"foo": "foo", "bar": "bar", "dir": {"baz": "baz"}})
|
||||
dvc.add("foo")
|
||||
dvc.add("dir")
|
||||
tree = RepoTree(dvc)
|
||||
assert tree.isdvc("foo")
|
||||
assert not tree.isdvc("bar")
|
||||
assert tree.isdvc("dir")
|
||||
assert not tree.isdvc("dir/baz")
|
||||
assert tree.isdvc("dir/baz", recursive=True, strict=False)
|
||||
|
||||
@@ -3,6 +3,7 @@ import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo.tree import DvcTree
|
||||
|
||||
|
||||
@@ -115,13 +116,13 @@ def test_walk(tmp_dir, dvc):
|
||||
(
|
||||
True,
|
||||
[
|
||||
"dir/subdir1",
|
||||
"dir/subdir2",
|
||||
"dir/subdir1/foo1",
|
||||
"dir/subdir1/bar1",
|
||||
"dir/subdir2/foo2",
|
||||
"dir/foo",
|
||||
"dir/bar",
|
||||
PathInfo("dir") / "subdir1",
|
||||
PathInfo("dir") / "subdir2",
|
||||
PathInfo("dir") / "subdir1" / "foo1",
|
||||
PathInfo("dir") / "subdir1" / "bar1",
|
||||
PathInfo("dir") / "subdir2" / "foo2",
|
||||
PathInfo("dir") / "foo",
|
||||
PathInfo("dir") / "bar",
|
||||
],
|
||||
),
|
||||
],
|
||||
@@ -143,10 +144,11 @@ def test_walk_dir(tmp_dir, dvc, fetch, expected):
|
||||
|
||||
expected = [str(tmp_dir / path) for path in expected]
|
||||
|
||||
actual = []
|
||||
for root, dirs, files in tree.walk("dir"):
|
||||
for entry in dirs + files:
|
||||
actual.append(os.path.join(root, entry))
|
||||
with dvc.state:
|
||||
actual = []
|
||||
for root, dirs, files in tree.walk("dir"):
|
||||
for entry in dirs + files:
|
||||
actual.append(os.path.join(root, entry))
|
||||
|
||||
assert set(actual) == set(expected)
|
||||
assert len(actual) == len(expected)
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import os
|
||||
|
||||
|
||||
def test_stage_cache(tmp_dir, dvc, run_copy, mocker):
|
||||
tmp_dir.gen("dep", "dep")
|
||||
tmp_dir.gen(
|
||||
"script.py",
|
||||
(
|
||||
'open("out", "w+").write("out"); '
|
||||
'open("out_no_cache", "w+").write("out_no_cache")'
|
||||
),
|
||||
)
|
||||
stage = dvc.run(
|
||||
cmd="python script.py",
|
||||
deps=["script.py", "dep"],
|
||||
outs=["out"],
|
||||
outs_no_cache=["out_no_cache"],
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
with dvc.lock, dvc.state:
|
||||
stage.remove(remove_outs=True, force=True)
|
||||
|
||||
assert not (tmp_dir / "out").exists()
|
||||
assert not (tmp_dir / "out_no_cache").exists()
|
||||
assert not (tmp_dir / "out.dvc").exists()
|
||||
|
||||
cache_dir = os.path.join(
|
||||
dvc.stage_cache.cache_dir,
|
||||
"10",
|
||||
"10b45372fdf4ec14d3f779c5b256378d7a12780e4c7f549a44138e492f098bfe",
|
||||
)
|
||||
cache_file = os.path.join(
|
||||
cache_dir,
|
||||
"bb32e04c6da96a7192513390acedbe4cd6123f8fe5b0ba5fffe39716fe87f6f4",
|
||||
)
|
||||
|
||||
assert os.path.isdir(cache_dir)
|
||||
assert os.listdir(cache_dir) == [os.path.basename(cache_file)]
|
||||
assert os.path.isfile(cache_file)
|
||||
|
||||
run_spy = mocker.patch("dvc.stage.run.cmd_run")
|
||||
checkout_spy = mocker.spy(stage, "checkout")
|
||||
with dvc.lock, dvc.state:
|
||||
stage.run()
|
||||
|
||||
assert not run_spy.called
|
||||
assert checkout_spy.call_count == 1
|
||||
|
||||
assert (tmp_dir / "out").exists()
|
||||
assert (tmp_dir / "out_no_cache").exists()
|
||||
assert (tmp_dir / "out").read_text() == "out"
|
||||
assert (tmp_dir / "out_no_cache").read_text() == "out_no_cache"
|
||||
|
||||
|
||||
def test_stage_cache_params(tmp_dir, dvc, run_copy, mocker):
|
||||
tmp_dir.gen("params.yaml", "foo: 1\nbar: 2")
|
||||
tmp_dir.gen("myparams.yaml", "baz: 3\nqux: 4")
|
||||
tmp_dir.gen(
|
||||
"script.py",
|
||||
(
|
||||
'open("out", "w+").write("out"); '
|
||||
'open("out_no_cache", "w+").write("out_no_cache")'
|
||||
),
|
||||
)
|
||||
stage = dvc.run(
|
||||
cmd="python script.py",
|
||||
params=["foo,bar", "myparams.yaml:baz,qux"],
|
||||
outs=["out"],
|
||||
outs_no_cache=["out_no_cache"],
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
with dvc.lock, dvc.state:
|
||||
stage.remove(remove_outs=True, force=True)
|
||||
|
||||
assert not (tmp_dir / "out").exists()
|
||||
assert not (tmp_dir / "out_no_cache").exists()
|
||||
assert not (tmp_dir / "out.dvc").exists()
|
||||
|
||||
cache_dir = os.path.join(
|
||||
dvc.stage_cache.cache_dir,
|
||||
"65",
|
||||
"651d0a5b82e05e48b03acf44954f6a8599760e652a143d517a17d1065eca61a1",
|
||||
)
|
||||
cache_file = os.path.join(
|
||||
cache_dir,
|
||||
"2196a5a4dd24c5759437511fcf9d6aa66b259e1dac58e3f212aefd1797a6f114",
|
||||
)
|
||||
|
||||
assert os.path.isdir(cache_dir)
|
||||
assert os.listdir(cache_dir) == [os.path.basename(cache_file)]
|
||||
assert os.path.isfile(cache_file)
|
||||
|
||||
run_spy = mocker.patch("dvc.stage.run.cmd_run")
|
||||
checkout_spy = mocker.spy(stage, "checkout")
|
||||
with dvc.lock, dvc.state:
|
||||
stage.run()
|
||||
|
||||
assert not run_spy.called
|
||||
assert checkout_spy.call_count == 1
|
||||
|
||||
assert (tmp_dir / "out").exists()
|
||||
assert (tmp_dir / "out_no_cache").exists()
|
||||
assert (tmp_dir / "out").read_text() == "out"
|
||||
assert (tmp_dir / "out_no_cache").read_text() == "out_no_cache"
|
||||
|
||||
|
||||
def test_stage_cache_wdir(tmp_dir, dvc, run_copy, mocker):
|
||||
tmp_dir.gen("dep", "dep")
|
||||
tmp_dir.gen(
|
||||
"script.py",
|
||||
(
|
||||
'open("out", "w+").write("out"); '
|
||||
'open("out_no_cache", "w+").write("out_no_cache")'
|
||||
),
|
||||
)
|
||||
tmp_dir.gen({"wdir": {}})
|
||||
stage = dvc.run(
|
||||
cmd="python ../script.py",
|
||||
deps=["../script.py", "../dep"],
|
||||
outs=["out"],
|
||||
outs_no_cache=["out_no_cache"],
|
||||
single_stage=True,
|
||||
wdir="wdir",
|
||||
)
|
||||
|
||||
with dvc.lock, dvc.state:
|
||||
stage.remove(remove_outs=True, force=True)
|
||||
|
||||
assert not (tmp_dir / "wdir" / "out").exists()
|
||||
assert not (tmp_dir / "wdir" / "out_no_cache").exists()
|
||||
assert not (tmp_dir / "wdir" / "out.dvc").exists()
|
||||
|
||||
cache_dir = os.path.join(
|
||||
dvc.stage_cache.cache_dir,
|
||||
"d2",
|
||||
"d2b5da199f4da73a861027f5f76020a948794011db9704814fdb2a488ca93ec2",
|
||||
)
|
||||
cache_file = os.path.join(
|
||||
cache_dir,
|
||||
"65cc63ade5ab338541726b26185ebaf42331141ec3a670a7d6e8a227505afade",
|
||||
)
|
||||
|
||||
assert os.path.isdir(cache_dir)
|
||||
assert os.listdir(cache_dir) == [os.path.basename(cache_file)]
|
||||
assert os.path.isfile(cache_file)
|
||||
|
||||
run_spy = mocker.patch("dvc.stage.run.cmd_run")
|
||||
checkout_spy = mocker.spy(stage, "checkout")
|
||||
with dvc.lock, dvc.state:
|
||||
stage.run()
|
||||
|
||||
assert not run_spy.called
|
||||
assert checkout_spy.call_count == 1
|
||||
|
||||
assert (tmp_dir / "wdir" / "out").exists()
|
||||
assert (tmp_dir / "wdir" / "out_no_cache").exists()
|
||||
assert (tmp_dir / "wdir" / "out").read_text() == "out"
|
||||
assert (tmp_dir / "wdir" / "out_no_cache").read_text() == "out_no_cache"
|
||||
@@ -6,5 +6,9 @@ from dvc.stage.run import run_stage
|
||||
|
||||
def test_run_stage_dry(caplog):
|
||||
with caplog.at_level(level=logging.INFO, logger="dvc"):
|
||||
run_stage(Stage(None, cmd="mycmd arg1 arg2"), dry=True)
|
||||
assert caplog.messages == ["Running command:\n\tmycmd arg1 arg2"]
|
||||
stage = Stage(None, "stage.dvc", cmd="mycmd arg1 arg2")
|
||||
run_stage(stage, dry=True)
|
||||
assert caplog.messages == [
|
||||
"Running callback stage 'stage.dvc' with command:",
|
||||
"\t" + "mycmd arg1 arg2",
|
||||
]
|
||||
|
||||
@@ -20,14 +20,14 @@ TEST_STAGE_DICT = {
|
||||
|
||||
|
||||
def test_stage_checksum():
|
||||
stage = Stage(None, "path")
|
||||
stage = Stage(None, "path", cmd="mycmd")
|
||||
|
||||
with mock.patch.object(stage, "dumpd", return_value=TEST_STAGE_DICT):
|
||||
assert stage.compute_md5() == "e9521a22111493406ea64a88cda63e0b"
|
||||
|
||||
|
||||
def test_wdir_default_ignored():
|
||||
stage = Stage(None, "path")
|
||||
stage = Stage(None, "path", cmd="mycmd")
|
||||
d = dict(TEST_STAGE_DICT, wdir=".")
|
||||
|
||||
with mock.patch.object(stage, "dumpd", return_value=d):
|
||||
@@ -35,7 +35,7 @@ def test_wdir_default_ignored():
|
||||
|
||||
|
||||
def test_wdir_non_default_is_not_ignored():
|
||||
stage = Stage(None, "path")
|
||||
stage = Stage(None, "path", cmd="mycmd")
|
||||
d = dict(TEST_STAGE_DICT, wdir="..")
|
||||
|
||||
with mock.patch.object(stage, "dumpd", return_value=d):
|
||||
@@ -43,7 +43,7 @@ def test_wdir_non_default_is_not_ignored():
|
||||
|
||||
|
||||
def test_meta_ignored():
|
||||
stage = Stage(None, "path")
|
||||
stage = Stage(None, "path", cmd="mycmd")
|
||||
d = dict(TEST_STAGE_DICT, meta={"author": "Suor"})
|
||||
|
||||
with mock.patch.object(stage, "dumpd", return_value=d):
|
||||
@@ -102,55 +102,3 @@ def test_always_changed(dvc):
|
||||
with dvc.lock:
|
||||
assert stage.changed()
|
||||
assert stage.status()["path"] == ["always changed"]
|
||||
|
||||
|
||||
def test_stage_cache(tmp_dir, dvc, run_copy, mocker):
|
||||
tmp_dir.gen("dep", "dep")
|
||||
tmp_dir.gen(
|
||||
"script.py",
|
||||
(
|
||||
'open("out", "w+").write("out"); '
|
||||
'open("out_no_cache", "w+").write("out_no_cache")'
|
||||
),
|
||||
)
|
||||
stage = dvc.run(
|
||||
cmd="python script.py",
|
||||
deps=["script.py", "dep"],
|
||||
outs=["out"],
|
||||
outs_no_cache=["out_no_cache"],
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
with dvc.lock, dvc.state:
|
||||
stage.remove(remove_outs=True, force=True)
|
||||
|
||||
assert not (tmp_dir / "out").exists()
|
||||
assert not (tmp_dir / "out_no_cache").exists()
|
||||
assert not (tmp_dir / "out.dvc").exists()
|
||||
|
||||
cache_dir = os.path.join(
|
||||
dvc.stage_cache.cache_dir,
|
||||
"10",
|
||||
"10b45372fdf4ec14d3f779c5b256378d7a12780e4c7f549a44138e492f098bfe",
|
||||
)
|
||||
cache_file = os.path.join(
|
||||
cache_dir,
|
||||
"bb32e04c6da96a7192513390acedbe4cd6123f8fe5b0ba5fffe39716fe87f6f4",
|
||||
)
|
||||
|
||||
assert os.path.isdir(cache_dir)
|
||||
assert os.listdir(cache_dir) == [os.path.basename(cache_file)]
|
||||
assert os.path.isfile(cache_file)
|
||||
|
||||
run_spy = mocker.patch("dvc.stage.run.cmd_run")
|
||||
checkout_spy = mocker.spy(stage, "checkout")
|
||||
with dvc.lock, dvc.state:
|
||||
stage.run()
|
||||
|
||||
assert not run_spy.called
|
||||
assert checkout_spy.call_count == 1
|
||||
|
||||
assert (tmp_dir / "out").exists()
|
||||
assert (tmp_dir / "out_no_cache").exists()
|
||||
assert (tmp_dir / "out").read_text() == "out"
|
||||
assert (tmp_dir / "out_no_cache").read_text() == "out_no_cache"
|
||||
|
||||
@@ -69,7 +69,6 @@ def test_ignore_from_file_should_filter_dirs_and_files():
|
||||
),
|
||||
("dont_ignore.txt", ["dont_ignore"], False),
|
||||
("dont_ignore.txt", ["dont*", "!dont_ignore.txt"], False),
|
||||
("../../../something.txt", ["**/something.txt"], False),
|
||||
],
|
||||
)
|
||||
def test_match_ignore_from_file(
|
||||
|
||||
@@ -222,10 +222,11 @@ class TestColorFormatter:
|
||||
|
||||
|
||||
def test_handlers():
|
||||
out, deb, err = logger.handlers
|
||||
out, deb, vrb, err = logger.handlers
|
||||
|
||||
assert out.level == logging.INFO
|
||||
assert deb.level == logging.DEBUG
|
||||
assert vrb.level == logging.TRACE
|
||||
assert err.level == logging.WARNING
|
||||
|
||||
|
||||
@@ -233,6 +234,7 @@ def test_logging_debug_with_datetime(caplog, dt):
|
||||
with caplog.at_level(logging.DEBUG, logger="dvc"):
|
||||
logger.warning("WARNING")
|
||||
logger.debug("DEBUG")
|
||||
logger.trace("TRACE")
|
||||
logger.error("ERROR")
|
||||
|
||||
for record in caplog.records:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user