mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
@@ -31,6 +31,7 @@ import dvc.command.lock as lock
|
||||
import dvc.command.pipeline as pipeline
|
||||
import dvc.command.daemon as daemon
|
||||
import dvc.command.commit as commit
|
||||
import dvc.command.tag as tag
|
||||
from dvc.exceptions import DvcParserError
|
||||
from dvc import VERSION
|
||||
|
||||
@@ -60,6 +61,7 @@ COMMANDS = [
|
||||
pipeline,
|
||||
daemon,
|
||||
commit,
|
||||
tag,
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import yaml
|
||||
|
||||
import dvc.logger as logger
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
|
||||
|
||||
|
||||
class CmdTagAdd(CmdBase):
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
self.repo.tag.add(
|
||||
self.args.tag,
|
||||
target=target,
|
||||
with_deps=self.args.with_deps,
|
||||
recursive=self.args.recursive,
|
||||
)
|
||||
except DvcException:
|
||||
logger.error("failed to add tag")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
class CmdTagRemove(CmdBase):
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
self.repo.tag.remove(
|
||||
self.args.tag,
|
||||
target=target,
|
||||
with_deps=self.args.with_deps,
|
||||
recursive=self.args.recursive,
|
||||
)
|
||||
except DvcException:
|
||||
logger.error("failed to remove tag")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
class CmdTagList(CmdBase):
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
tags = self.repo.tag.list(
|
||||
target,
|
||||
with_deps=self.args.with_deps,
|
||||
recursive=self.args.recursive,
|
||||
)
|
||||
if tags:
|
||||
logger.info(yaml.dump(tags, default_flow_style=False))
|
||||
except DvcException:
|
||||
logger.error("failed list tags")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
TAG_HELP = "A set of commands to manage DVC tags."
|
||||
tag_parser = subparsers.add_parser(
|
||||
"tag",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(TAG_HELP, "tag"),
|
||||
add_help=False,
|
||||
)
|
||||
|
||||
tag_subparsers = tag_parser.add_subparsers(
|
||||
dest="cmd",
|
||||
help="Use DVC tag CMD --help to display command-specific help.",
|
||||
)
|
||||
|
||||
fix_subparsers(tag_subparsers)
|
||||
|
||||
TAG_ADD_HELP = "Add DVC tag."
|
||||
tag_add_parser = tag_subparsers.add_parser(
|
||||
"add",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(TAG_ADD_HELP, "tag-add"),
|
||||
help=TAG_ADD_HELP,
|
||||
)
|
||||
tag_add_parser.add_argument("tag", help="Dvc tag.")
|
||||
tag_add_parser.add_argument(
|
||||
"targets", nargs="*", default=[None], help="Dvc files."
|
||||
)
|
||||
tag_add_parser.add_argument(
|
||||
"-d",
|
||||
"--with-deps",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Add tag for all dependencies of the specified DVC file.",
|
||||
)
|
||||
tag_add_parser.add_argument(
|
||||
"-R",
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Add tag for subdirectories of the specified directory.",
|
||||
)
|
||||
tag_add_parser.set_defaults(func=CmdTagAdd)
|
||||
|
||||
TAG_REMOVE_HELP = "Remove DVC tag."
|
||||
tag_remove_parser = tag_subparsers.add_parser(
|
||||
"remove",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(TAG_REMOVE_HELP, "tag-remove"),
|
||||
help=TAG_REMOVE_HELP,
|
||||
)
|
||||
tag_remove_parser.add_argument("tag", help="Dvc tag.")
|
||||
tag_remove_parser.add_argument(
|
||||
"targets", nargs="*", default=[None], help="Dvc files."
|
||||
)
|
||||
tag_remove_parser.add_argument(
|
||||
"-d",
|
||||
"--with-deps",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Remove tag for all dependencies of the specified DVC file.",
|
||||
)
|
||||
tag_remove_parser.add_argument(
|
||||
"-R",
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Remove tag for subdirectories of the specified directory.",
|
||||
)
|
||||
tag_remove_parser.set_defaults(func=CmdTagRemove)
|
||||
|
||||
TAG_LIST_HELP = "List DVC tags."
|
||||
tag_list_parser = tag_subparsers.add_parser(
|
||||
"list",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(TAG_LIST_HELP, "tag-list"),
|
||||
help=TAG_LIST_HELP,
|
||||
)
|
||||
tag_list_parser.add_argument(
|
||||
"targets", nargs="*", default=[None], help="Dvc files."
|
||||
)
|
||||
tag_list_parser.add_argument(
|
||||
"-d",
|
||||
"--with-deps",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="List tags for all dependencies of the specified DVC file.",
|
||||
)
|
||||
tag_list_parser.add_argument(
|
||||
"-R",
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="List tags for subdirectories of the specified directory.",
|
||||
)
|
||||
tag_list_parser.set_defaults(func=CmdTagList)
|
||||
@@ -248,3 +248,10 @@ class TargetNotDirectoryError(DvcException):
|
||||
super(TargetNotDirectoryError, self).__init__(
|
||||
"Target: {} is not a directory".format(path)
|
||||
)
|
||||
|
||||
|
||||
class CheckoutErrorSuggestGit(DvcException):
|
||||
def __init__(self, target, cause):
|
||||
super(CheckoutErrorSuggestGit, self).__init__(
|
||||
"Did you mean 'git checkout {}'?".format(target), cause=cause
|
||||
)
|
||||
|
||||
+41
-16
@@ -33,26 +33,31 @@ OUTS_MAP = {
|
||||
"local": OutputLOCAL,
|
||||
}
|
||||
|
||||
SCHEMA = {
|
||||
OutputBase.PARAM_PATH: str,
|
||||
# NOTE: currently there are only 3 possible checksum names:
|
||||
#
|
||||
# 1) md5 (LOCAL, SSH, GS);
|
||||
# 2) etag (S3);
|
||||
# 3) checksum (HDFS);
|
||||
#
|
||||
# so when a few types of outputs share the same name, we only need
|
||||
# specify it once.
|
||||
# NOTE: currently there are only 3 possible checksum names:
|
||||
#
|
||||
# 1) md5 (LOCAL, SSH, GS);
|
||||
# 2) etag (S3);
|
||||
# 3) checksum (HDFS);
|
||||
#
|
||||
# so when a few types of outputs share the same name, we only need
|
||||
# specify it once.
|
||||
CHECKSUM_SCHEMA = {
|
||||
schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None),
|
||||
schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None),
|
||||
schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
|
||||
schema.Optional(OutputBase.PARAM_CACHE): bool,
|
||||
schema.Optional(OutputBase.PARAM_METRIC): OutputBase.METRIC_SCHEMA,
|
||||
schema.Optional(OutputBase.PARAM_PERSIST): bool,
|
||||
}
|
||||
|
||||
TAGS_SCHEMA = {schema.Optional(str): CHECKSUM_SCHEMA}
|
||||
|
||||
def _get(stage, p, info, cache, metric, persist):
|
||||
SCHEMA = CHECKSUM_SCHEMA.copy()
|
||||
SCHEMA[OutputBase.PARAM_PATH] = str
|
||||
SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)] = bool
|
||||
SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)] = OutputBase.METRIC_SCHEMA
|
||||
SCHEMA[schema.Optional(OutputBase.PARAM_TAGS)] = TAGS_SCHEMA
|
||||
SCHEMA[schema.Optional(OutputBase.PARAM_PERSIST)] = bool
|
||||
|
||||
|
||||
def _get(stage, p, info, cache, metric, persist=False, tags=None):
|
||||
parsed = urlparse(p)
|
||||
if parsed.scheme == "remote":
|
||||
name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
|
||||
@@ -66,11 +71,21 @@ def _get(stage, p, info, cache, metric, persist):
|
||||
remote=remote,
|
||||
metric=metric,
|
||||
persist=persist,
|
||||
tags=tags,
|
||||
)
|
||||
|
||||
for o in OUTS:
|
||||
if o.supported(p):
|
||||
return o(stage, p, info, cache=cache, remote=None, metric=metric)
|
||||
return o(
|
||||
stage,
|
||||
p,
|
||||
info,
|
||||
cache=cache,
|
||||
remote=None,
|
||||
metric=metric,
|
||||
persist=persist,
|
||||
tags=tags,
|
||||
)
|
||||
return OutputLOCAL(
|
||||
stage,
|
||||
p,
|
||||
@@ -79,6 +94,7 @@ def _get(stage, p, info, cache, metric, persist):
|
||||
remote=None,
|
||||
metric=metric,
|
||||
persist=persist,
|
||||
tags=tags,
|
||||
)
|
||||
|
||||
|
||||
@@ -89,8 +105,17 @@ def loadd_from(stage, d_list):
|
||||
cache = d.pop(OutputBase.PARAM_CACHE, True)
|
||||
metric = d.pop(OutputBase.PARAM_METRIC, False)
|
||||
persist = d.pop(OutputBase.PARAM_PERSIST, False)
|
||||
tags = d.pop(OutputBase.PARAM_TAGS, None)
|
||||
ret.append(
|
||||
_get(stage, p, info=d, cache=cache, metric=metric, persist=persist)
|
||||
_get(
|
||||
stage,
|
||||
p,
|
||||
info=d,
|
||||
cache=cache,
|
||||
metric=metric,
|
||||
persist=persist,
|
||||
tags=tags,
|
||||
)
|
||||
)
|
||||
return ret
|
||||
|
||||
|
||||
+17
-2
@@ -47,6 +47,8 @@ class OutputBase(object):
|
||||
},
|
||||
)
|
||||
|
||||
PARAM_TAGS = "tags"
|
||||
|
||||
DoesNotExistError = OutputDoesNotExistError
|
||||
IsNotFileOrDirError = OutputIsNotFileOrDirError
|
||||
|
||||
@@ -59,6 +61,7 @@ class OutputBase(object):
|
||||
cache=True,
|
||||
metric=False,
|
||||
persist=False,
|
||||
tags=None,
|
||||
):
|
||||
self.stage = stage
|
||||
self.repo = stage.repo
|
||||
@@ -68,6 +71,7 @@ class OutputBase(object):
|
||||
self.use_cache = False if self.IS_DEPENDENCY else cache
|
||||
self.metric = False if self.IS_DEPENDENCY else metric
|
||||
self.persist = persist
|
||||
self.tags = None if self.IS_DEPENDENCY else (tags or {})
|
||||
|
||||
if (
|
||||
self.use_cache
|
||||
@@ -197,6 +201,9 @@ class OutputBase(object):
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
ret[self.PARAM_PERSIST] = self.persist
|
||||
|
||||
if self.tags:
|
||||
ret[self.PARAM_TAGS] = self.tags
|
||||
|
||||
return ret
|
||||
|
||||
def verify_metric(self):
|
||||
@@ -207,12 +214,20 @@ class OutputBase(object):
|
||||
def download(self, to_info, resume=False):
|
||||
self.remote.download([self.path_info], [to_info], resume=resume)
|
||||
|
||||
def checkout(self, force=False, progress_callback=None):
|
||||
def checkout(self, force=False, progress_callback=None, tag=None):
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
if tag:
|
||||
info = self.tags[tag]
|
||||
else:
|
||||
info = self.info
|
||||
|
||||
getattr(self.repo.cache, self.scheme).checkout(
|
||||
output=self, force=force, progress_callback=progress_callback
|
||||
self.path_info,
|
||||
info,
|
||||
force=force,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
|
||||
def remove(self, ignore_remove=False):
|
||||
|
||||
@@ -19,6 +19,7 @@ class OutputHDFS(OutputBase):
|
||||
cache=True,
|
||||
metric=False,
|
||||
persist=False,
|
||||
tags=None,
|
||||
):
|
||||
super(OutputHDFS, self).__init__(
|
||||
stage,
|
||||
@@ -28,6 +29,7 @@ class OutputHDFS(OutputBase):
|
||||
cache=cache,
|
||||
metric=metric,
|
||||
persist=persist,
|
||||
tags=tags,
|
||||
)
|
||||
if remote:
|
||||
path = posixpath.join(remote.url, urlparse(path).path.lstrip("/"))
|
||||
|
||||
+3
-6
@@ -22,6 +22,7 @@ class OutputLOCAL(OutputBase):
|
||||
cache=True,
|
||||
metric=False,
|
||||
persist=False,
|
||||
tags=None,
|
||||
):
|
||||
super(OutputLOCAL, self).__init__(
|
||||
stage,
|
||||
@@ -31,6 +32,7 @@ class OutputLOCAL(OutputBase):
|
||||
cache=cache,
|
||||
metric=metric,
|
||||
persist=persist,
|
||||
tags=tags,
|
||||
)
|
||||
if remote:
|
||||
p = os.path.join(
|
||||
@@ -155,12 +157,7 @@ class OutputLOCAL(OutputBase):
|
||||
|
||||
@property
|
||||
def dir_cache(self):
|
||||
if self.checksum not in self._dir_cache.keys():
|
||||
self._dir_cache[
|
||||
self.checksum
|
||||
] = self.repo.cache.local.load_dir_cache(self.checksum)
|
||||
|
||||
return self._dir_cache[self.checksum]
|
||||
return self.repo.cache.local.load_dir_cache(self.checksum)
|
||||
|
||||
def get_files_number(self):
|
||||
if self.cache is None:
|
||||
|
||||
@@ -19,6 +19,7 @@ class OutputS3(OutputBase):
|
||||
cache=True,
|
||||
metric=False,
|
||||
persist=False,
|
||||
tags=None,
|
||||
):
|
||||
super(OutputS3, self).__init__(
|
||||
stage,
|
||||
@@ -28,6 +29,7 @@ class OutputS3(OutputBase):
|
||||
cache=cache,
|
||||
metric=metric,
|
||||
persist=persist,
|
||||
tags=tags,
|
||||
)
|
||||
bucket = remote.bucket if remote else urlparse(path).netloc
|
||||
path = urlparse(path).path.lstrip("/")
|
||||
|
||||
@@ -20,6 +20,7 @@ class OutputSSH(OutputBase):
|
||||
cache=True,
|
||||
metric=False,
|
||||
persist=False,
|
||||
tags=None,
|
||||
):
|
||||
super(OutputSSH, self).__init__(
|
||||
stage,
|
||||
@@ -29,6 +30,7 @@ class OutputSSH(OutputBase):
|
||||
cache=cache,
|
||||
metric=metric,
|
||||
persist=persist,
|
||||
tags=tags,
|
||||
)
|
||||
parsed = urlparse(path)
|
||||
host = remote.host if remote else parsed.hostname
|
||||
|
||||
+18
-14
@@ -322,10 +322,9 @@ class RemoteBase(object):
|
||||
|
||||
self.remove(path_info)
|
||||
|
||||
def do_checkout(self, output, force=False, progress_callback=None):
|
||||
path_info = output.path_info
|
||||
checksum = output.info.get(self.PARAM_CHECKSUM)
|
||||
|
||||
def do_checkout(
|
||||
self, path_info, checksum, force=False, progress_callback=None
|
||||
):
|
||||
if self.exists(path_info):
|
||||
msg = "data '{}' exists. Removing before checkout."
|
||||
logger.warning(msg.format(str(path_info)))
|
||||
@@ -334,33 +333,38 @@ class RemoteBase(object):
|
||||
from_info = self.checksum_to_path_info(checksum)
|
||||
self.copy(from_info, path_info)
|
||||
|
||||
def checkout(self, output, force=False, progress_callback=None):
|
||||
scheme = output.path_info["scheme"]
|
||||
def checkout(
|
||||
self, path_info, checksum_info, force=False, progress_callback=None
|
||||
):
|
||||
scheme = path_info["scheme"]
|
||||
if scheme not in ["", "local"] and scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
checksum = output.info.get(self.PARAM_CHECKSUM)
|
||||
checksum = checksum_info.get(self.PARAM_CHECKSUM)
|
||||
if not checksum:
|
||||
msg = "No checksum info for '{}'."
|
||||
logger.debug(msg.format(str(output.path_info)))
|
||||
logger.debug(msg.format(str(path_info)))
|
||||
return
|
||||
|
||||
if not self.changed(output.path_info, output.info):
|
||||
if not self.changed(path_info, checksum_info):
|
||||
msg = "Data '{}' didn't change."
|
||||
logger.debug(msg.format(str(output.path_info)))
|
||||
logger.debug(msg.format(str(path_info)))
|
||||
return
|
||||
|
||||
if self.changed_cache(checksum):
|
||||
msg = "Cache '{}' not found. File '{}' won't be created."
|
||||
logger.warning(msg.format(checksum, str(output.path_info)))
|
||||
self.safe_remove(output.path_info, force=force)
|
||||
logger.warning(msg.format(checksum, str(path_info)))
|
||||
self.safe_remove(path_info, force=force)
|
||||
return
|
||||
|
||||
msg = "Checking out '{}' with cache '{}'."
|
||||
logger.debug(msg.format(str(output.path_info), checksum))
|
||||
logger.debug(msg.format(str(path_info), checksum))
|
||||
|
||||
self.do_checkout(
|
||||
output, force=force, progress_callback=progress_callback
|
||||
path_info,
|
||||
checksum,
|
||||
force=force,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
+15
-9
@@ -78,6 +78,8 @@ class RemoteLOCAL(RemoteBase):
|
||||
|
||||
self.path_info = {"scheme": "local"}
|
||||
|
||||
self._dir_info = {}
|
||||
|
||||
@staticmethod
|
||||
def compat_config(config):
|
||||
ret = config.copy()
|
||||
@@ -256,9 +258,12 @@ class RemoteLOCAL(RemoteBase):
|
||||
|
||||
def load_dir_cache(self, md5):
|
||||
path = self.get(md5)
|
||||
|
||||
assert self.is_dir_cache(path)
|
||||
|
||||
dir_info = self._dir_info.get(md5)
|
||||
if dir_info:
|
||||
return dir_info
|
||||
|
||||
try:
|
||||
with open(path, "r") as fd:
|
||||
d = json.load(fd)
|
||||
@@ -275,6 +280,8 @@ class RemoteLOCAL(RemoteBase):
|
||||
for info in d:
|
||||
info["relpath"] = self.to_ospath(info["relpath"])
|
||||
|
||||
self._dir_info[md5] = d
|
||||
|
||||
return d
|
||||
|
||||
def dump_dir_cache(self, md5, dir_info):
|
||||
@@ -299,15 +306,13 @@ class RemoteLOCAL(RemoteBase):
|
||||
def is_dir_cache(cls, cache):
|
||||
return cache.endswith(cls.MD5_DIR_SUFFIX)
|
||||
|
||||
def do_checkout(self, output, force=False, progress_callback=None):
|
||||
path_info = output.path_info
|
||||
checksum = output.info.get(self.PARAM_CHECKSUM)
|
||||
|
||||
def do_checkout(
|
||||
self, path_info, checksum, force=False, progress_callback=None
|
||||
):
|
||||
path = path_info["path"]
|
||||
md5 = checksum
|
||||
cache = self.get(md5)
|
||||
cache = self.get(checksum)
|
||||
|
||||
if not output.is_dir_cache:
|
||||
if not self.is_dir_cache(cache):
|
||||
if os.path.exists(path):
|
||||
self.safe_remove(path_info, force=force)
|
||||
|
||||
@@ -322,11 +327,12 @@ class RemoteLOCAL(RemoteBase):
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
dir_info = output.dir_cache
|
||||
dir_relpath = os.path.relpath(path)
|
||||
|
||||
logger.debug("Linking directory '{}'.".format(dir_relpath))
|
||||
|
||||
dir_info = self.load_dir_cache(checksum)
|
||||
|
||||
for processed, entry in enumerate(dir_info):
|
||||
relpath = entry[self.PARAM_RELPATH]
|
||||
m = entry[self.PARAM_CHECKSUM]
|
||||
|
||||
@@ -45,6 +45,7 @@ class Repo(object):
|
||||
from dvc.updater import Updater
|
||||
from dvc.repo.metrics import Metrics
|
||||
from dvc.scm.tree import WorkingTree
|
||||
from dvc.repo.tag import Tag
|
||||
|
||||
root_dir = self.find_root(root_dir)
|
||||
|
||||
@@ -70,6 +71,7 @@ class Repo(object):
|
||||
self.updater = Updater(self.dvc_dir)
|
||||
|
||||
self.metrics = Metrics(self)
|
||||
self.tag = Tag(self)
|
||||
|
||||
self._ignore()
|
||||
|
||||
@@ -143,10 +145,13 @@ class Repo(object):
|
||||
assert len(pipelines) == 1
|
||||
return pipelines[0]
|
||||
|
||||
def collect(self, target, with_deps=False):
|
||||
def collect(self, target, with_deps=False, recursive=False):
|
||||
import networkx as nx
|
||||
from dvc.stage import Stage
|
||||
|
||||
if not target or recursive:
|
||||
return self.active_stages(target)
|
||||
|
||||
stage = Stage.load(self, target)
|
||||
if not with_deps:
|
||||
return [stage]
|
||||
|
||||
+7
-10
@@ -1,7 +1,7 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import dvc.logger as logger
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.exceptions import CheckoutErrorSuggestGit
|
||||
from dvc.progress import ProgressCallback
|
||||
|
||||
|
||||
@@ -29,15 +29,12 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False):
|
||||
|
||||
all_stages = self.stages()
|
||||
|
||||
if target and not recursive:
|
||||
try:
|
||||
stages = self.collect(target, with_deps=with_deps)
|
||||
except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
|
||||
raise DvcException(
|
||||
str(exc) + " Did you mean 'git checkout {}'?".format(target)
|
||||
)
|
||||
else:
|
||||
stages = self.active_stages(target)
|
||||
try:
|
||||
stages = self.collect(target, with_deps=with_deps, recursive=recursive)
|
||||
except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
|
||||
if not target:
|
||||
raise
|
||||
raise CheckoutErrorSuggestGit(target, exc)
|
||||
|
||||
with self.state:
|
||||
_cleanup_unused_links(self, all_stages)
|
||||
|
||||
+1
-5
@@ -1,9 +1,5 @@
|
||||
def commit(self, target, with_deps=False, recursive=False, force=False):
|
||||
if target and not recursive:
|
||||
stages = self.collect(target, with_deps=with_deps)
|
||||
else:
|
||||
stages = self.active_stages(target)
|
||||
|
||||
stages = self.collect(target, with_deps=with_deps, recursive=recursive)
|
||||
with self.state:
|
||||
for stage in stages:
|
||||
stage.check_can_commit(force=force)
|
||||
|
||||
+1
-4
@@ -6,10 +6,7 @@ import dvc.logger as logger
|
||||
def _local_status(self, target=None, with_deps=False):
|
||||
status = {}
|
||||
|
||||
if target:
|
||||
stages = self.collect(target, with_deps=with_deps)
|
||||
else:
|
||||
stages = self.active_stages()
|
||||
stages = self.collect(target, with_deps=with_deps)
|
||||
|
||||
for stage in stages:
|
||||
if stage.locked:
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
class Tag(object):
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
|
||||
def add(self, *args, **kwargs):
|
||||
from dvc.repo.tag.add import add
|
||||
|
||||
return add(self.repo, *args, **kwargs)
|
||||
|
||||
def list(self, *args, **kwargs):
|
||||
from dvc.repo.tag.list import list
|
||||
|
||||
return list(self.repo, *args, **kwargs)
|
||||
|
||||
def remove(self, *args, **kwargs):
|
||||
from dvc.repo.tag.remove import remove
|
||||
|
||||
return remove(self.repo, *args, **kwargs)
|
||||
@@ -0,0 +1,15 @@
|
||||
import dvc.logger as logger
|
||||
|
||||
|
||||
def add(self, tag, target=None, with_deps=False, recursive=False):
|
||||
stages = self.collect(target, with_deps=with_deps, recursive=recursive)
|
||||
for stage in stages:
|
||||
changed = False
|
||||
for out in stage.outs:
|
||||
if not out.info:
|
||||
logger.warning("missing checksum info for '{}'".format(out))
|
||||
continue
|
||||
out.tags[tag] = out.info.copy()
|
||||
changed = True
|
||||
if changed:
|
||||
stage.dump()
|
||||
@@ -0,0 +1,11 @@
|
||||
def list(self, target=None, with_deps=False, recursive=False):
|
||||
ret = {}
|
||||
stages = self.collect(target, with_deps=with_deps, recursive=recursive)
|
||||
for stage in stages:
|
||||
outs_tags = {}
|
||||
for out in stage.outs:
|
||||
if out.tags:
|
||||
outs_tags[str(out)] = out.tags.copy()
|
||||
if outs_tags:
|
||||
ret.update({stage.relpath: outs_tags})
|
||||
return ret
|
||||
@@ -0,0 +1,15 @@
|
||||
import dvc.logger as logger
|
||||
|
||||
|
||||
def remove(self, tag, target=None, with_deps=False, recursive=False):
|
||||
stages = self.collect(target, with_deps=with_deps, recursive=recursive)
|
||||
for stage in stages:
|
||||
changed = False
|
||||
for out in stage.outs:
|
||||
if tag not in out.tags.keys():
|
||||
logger.warning("tag '{}' not found for '{}'".format(tag, out))
|
||||
continue
|
||||
del out.tags[tag]
|
||||
changed = True
|
||||
if changed:
|
||||
stage.dump()
|
||||
+26
-2
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
|
||||
|
||||
from dvc.utils.compat import str, open
|
||||
|
||||
import re
|
||||
import os
|
||||
import yaml
|
||||
import subprocess
|
||||
@@ -129,6 +130,8 @@ class Stage(object):
|
||||
Optional(PARAM_LOCKED): bool,
|
||||
}
|
||||
|
||||
TAG_REGEX = r"^(?P<path>.*)@(?P<tag>[^\\/@:]*)$"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo,
|
||||
@@ -139,6 +142,7 @@ class Stage(object):
|
||||
outs=None,
|
||||
md5=None,
|
||||
locked=False,
|
||||
tag=None,
|
||||
):
|
||||
if deps is None:
|
||||
deps = []
|
||||
@@ -153,6 +157,7 @@ class Stage(object):
|
||||
self.deps = deps
|
||||
self.md5 = md5
|
||||
self.locked = locked
|
||||
self.tag = tag
|
||||
|
||||
def __repr__(self):
|
||||
return "Stage: '{path}'".format(
|
||||
@@ -536,8 +541,17 @@ class Stage(object):
|
||||
if not repo.tree.isfile(fname):
|
||||
raise StageFileIsNotDvcFileError(fname)
|
||||
|
||||
@classmethod
|
||||
def _get_path_tag(cls, s):
|
||||
regex = re.compile(cls.TAG_REGEX)
|
||||
match = regex.match(s)
|
||||
if not match:
|
||||
return s, None
|
||||
return match.group("path"), match.group("tag")
|
||||
|
||||
@staticmethod
|
||||
def load(repo, fname):
|
||||
fname, tag = Stage._get_path_tag(fname)
|
||||
|
||||
# it raises the proper exceptions by priority:
|
||||
# 1. when the file doesn't exists
|
||||
@@ -563,6 +577,7 @@ class Stage(object):
|
||||
cmd=d.get(Stage.PARAM_CMD),
|
||||
md5=d.get(Stage.PARAM_MD5),
|
||||
locked=d.get(Stage.PARAM_LOCKED, False),
|
||||
tag=tag,
|
||||
)
|
||||
|
||||
stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
|
||||
@@ -624,7 +639,14 @@ class Stage(object):
|
||||
# NOTE: excluding parameters that don't affect the state of the
|
||||
# pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if
|
||||
# it has changed, we might not have that output in our cache.
|
||||
m = dict_md5(d, exclude=[self.PARAM_LOCKED, OutputLOCAL.PARAM_METRIC])
|
||||
m = dict_md5(
|
||||
d,
|
||||
exclude=[
|
||||
self.PARAM_LOCKED,
|
||||
OutputLOCAL.PARAM_METRIC,
|
||||
OutputLOCAL.PARAM_TAGS,
|
||||
],
|
||||
)
|
||||
logger.debug("Computed stage '{}' md5: '{}'".format(self.relpath, m))
|
||||
return m
|
||||
|
||||
@@ -791,7 +813,9 @@ class Stage(object):
|
||||
|
||||
def checkout(self, force=False, progress_callback=None):
|
||||
for out in self.outs:
|
||||
out.checkout(force=force, progress_callback=progress_callback)
|
||||
out.checkout(
|
||||
force=force, tag=self.tag, progress_callback=progress_callback
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _status(entries):
|
||||
|
||||
+8
-21
@@ -15,12 +15,13 @@ from dvc.system import System
|
||||
from dvc.utils import walk_files
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.test_repro import TestRepro
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage import Stage, StageFileBadNameError, StageFileDoesNotExistError
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
from dvc.exceptions import (
|
||||
DvcException,
|
||||
ConfirmRemoveError,
|
||||
TargetNotDirectoryError,
|
||||
CheckoutErrorSuggestGit,
|
||||
)
|
||||
|
||||
from mock import patch
|
||||
@@ -385,30 +386,16 @@ class TestCheckoutSuggestGit(TestRepro):
|
||||
try:
|
||||
self.dvc.checkout(target="gitbranch")
|
||||
except DvcException as exc:
|
||||
msg = str(exc)
|
||||
self.assertEqual(
|
||||
msg,
|
||||
(
|
||||
"'gitbranch' does not exist. "
|
||||
"Did you mean 'git checkout gitbranch'?"
|
||||
),
|
||||
)
|
||||
self.assertIsInstance(exc, CheckoutErrorSuggestGit)
|
||||
self.assertIsInstance(exc.cause, StageFileDoesNotExistError)
|
||||
self.assertIsNone(exc.cause.cause)
|
||||
|
||||
try:
|
||||
self.dvc.checkout(target=self.FOO)
|
||||
except DvcException as exc:
|
||||
msg = str(exc)
|
||||
self.assertEqual(
|
||||
msg,
|
||||
(
|
||||
"bad stage filename '{}'."
|
||||
" Stage files should be named 'Dvcfile'"
|
||||
" or have a '.dvc' suffix (e.g. '{}.dvc')."
|
||||
" Did you mean 'git checkout {}'?".format(
|
||||
self.FOO, self.FOO, self.FOO
|
||||
)
|
||||
),
|
||||
)
|
||||
self.assertIsInstance(exc, CheckoutErrorSuggestGit)
|
||||
self.assertIsInstance(exc.cause, StageFileBadNameError)
|
||||
self.assertIsNone(exc.cause.cause)
|
||||
|
||||
|
||||
class TestCheckoutShouldHaveSelfClearingProgressBar(TestDvc):
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
import os
|
||||
import shutil
|
||||
import filecmp
|
||||
|
||||
from dvc.main import main
|
||||
from dvc.logger import logger
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
from tests.utils import reset_logger_standard_output, reset_logger_error_output
|
||||
from tests.utils.logger import MockLoggerHandlers, ConsoleFontColorsRemover
|
||||
|
||||
|
||||
class TestTag(TestDvc):
|
||||
def test(self):
|
||||
fname = "file"
|
||||
shutil.copyfile(self.FOO, fname)
|
||||
|
||||
stages = self.dvc.add(fname)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
|
||||
ret = main(["tag", "add", "v1", stage.path])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
os.unlink(fname)
|
||||
shutil.copyfile(self.BAR, fname)
|
||||
|
||||
ret = main(["repro", stage.path])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(["tag", "add", "v2", stage.path])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(["tag", "list", stage.path])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(["checkout", stage.path + "@v1"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
self.assertTrue(filecmp.cmp(fname, self.FOO, shallow=False))
|
||||
|
||||
ret = main(["checkout", stage.path + "@v2"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
self.assertTrue(filecmp.cmp(fname, self.BAR, shallow=False))
|
||||
|
||||
|
||||
class TestTagAll(TestDvc):
|
||||
def test(self):
|
||||
ret = main(["add", self.FOO, self.BAR])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
with MockLoggerHandlers(logger), ConsoleFontColorsRemover():
|
||||
reset_logger_standard_output()
|
||||
|
||||
ret = main(["tag", "list"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
self.assertEqual("", logger.handlers[0].stream.getvalue())
|
||||
|
||||
ret = main(["tag", "add", "v1"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
with MockLoggerHandlers(logger), ConsoleFontColorsRemover():
|
||||
reset_logger_standard_output()
|
||||
|
||||
ret = main(["tag", "list"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
self.assertEqual(
|
||||
logger.handlers[0].stream.getvalue(),
|
||||
"bar.dvc:\n"
|
||||
" bar:\n"
|
||||
" v1:\n"
|
||||
" md5: 8978c98bb5a48c2fb5f2c4c905768afa\n"
|
||||
"foo.dvc:\n"
|
||||
" foo:\n"
|
||||
" v1:\n"
|
||||
" md5: acbd18db4cc2f85cedef654fccc4a4d8\n"
|
||||
"\n",
|
||||
)
|
||||
|
||||
ret = main(["tag", "remove", "v1"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
with MockLoggerHandlers(logger), ConsoleFontColorsRemover():
|
||||
reset_logger_standard_output()
|
||||
|
||||
ret = main(["tag", "list"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
self.assertEqual("", logger.handlers[0].stream.getvalue())
|
||||
|
||||
|
||||
class TestTagAddNoChecksumInfo(TestDvc):
|
||||
def test(self):
|
||||
ret = main(["run", "-o", self.FOO, "--no-exec"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
with MockLoggerHandlers(logger), ConsoleFontColorsRemover():
|
||||
reset_logger_error_output()
|
||||
|
||||
ret = main(["tag", "add", "v1", "foo.dvc"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
self.assertEqual(
|
||||
"Warning: missing checksum info for 'foo'\n",
|
||||
logger.handlers[1].stream.getvalue(),
|
||||
)
|
||||
|
||||
|
||||
class TestTagRemoveNoTag(TestDvc):
|
||||
def test(self):
|
||||
ret = main(["add", self.FOO])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
with MockLoggerHandlers(logger), ConsoleFontColorsRemover():
|
||||
reset_logger_error_output()
|
||||
|
||||
ret = main(["tag", "remove", "v1", "foo.dvc"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
self.assertEqual(
|
||||
"Warning: tag 'v1' not found for 'foo'\n",
|
||||
logger.handlers[1].stream.getvalue(),
|
||||
)
|
||||
Reference in New Issue
Block a user