Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:iterative/dvc

Browse Source
This commit is contained in:
Jorge Orpinel
2020-01-30 12:24:22 -06:00
parent 1672cdfb06 d864caf79b
commit aa849ab2d5
20 changed files with 602 additions and 980 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/api.py
-159
View File
@@ -1,48 +1,11 @@
from builtins import open as builtin_open
import importlib
import os
import sys
from contextlib import contextmanager, _GeneratorContextManager as GCM
import threading
from funcy import wrap_with
import ruamel.yaml
from voluptuous import Schema, Required, Invalid
from dvc.repo import Repo
from dvc.exceptions import DvcException, NotDvcRepoError
from dvc.external_repo import external_repo
SUMMON_FILE_SCHEMA = Schema(
{
Required("objects"): [
{
Required("name"): str,
"meta": dict,
Required("summon"): {
Required("type"): str,
"deps": [str],
str: object,
},
}
]
}
)
SUMMON_PYTHON_SCHEMA = Schema(
{
Required("type"): "python",
Required("call"): str,
"args": dict,
"deps": [str],
}
)
class SummonError(DvcException):
pass
class UrlNotDvcRepoError(DvcException):
"""Thrown if given url is not a DVC repository.
@@ -120,128 +83,6 @@ def _make_repo(repo_url=None, rev=None):
yield repo
def summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml", args=None):
"""Instantiate an object described in the `summon_file`."""
with prepare_summon(
name, repo=repo, rev=rev, summon_file=summon_file
) as desc:
try:
summon_dict = SUMMON_PYTHON_SCHEMA(desc.obj["summon"])
except Invalid as exc:
raise SummonError(str(exc)) from exc
_args = {**summon_dict.get("args", {}), **(args or {})}
return _invoke_method(summon_dict["call"], _args, desc.repo.root_dir)
@contextmanager
def prepare_summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml"):
"""Does a couple of things every summon needs as a prerequisite:
clones the repo, parses the summon file and pulls the deps.
Calling code is expected to complete the summon logic following
instructions stated in "summon" dict of the object spec.
Returns a SummonDesc instance, which contains references to a Repo object,
named object specification and resolved paths to deps.
"""
with _make_repo(repo, rev=rev) as _repo:
_require_dvc(_repo)
try:
path = os.path.join(_repo.root_dir, summon_file)
obj = _get_object_spec(name, path)
yield SummonDesc(_repo, obj)
except SummonError as exc:
raise SummonError(
str(exc) + " at '{}' in '{}'".format(summon_file, repo)
) from exc.__cause__
class SummonDesc:
def __init__(self, repo, obj):
self.repo = repo
self.obj = obj
self._pull_deps()
@property
def deps(self):
return [os.path.join(self.repo.root_dir, d) for d in self._deps]
@property
def _deps(self):
return self.obj["summon"].get("deps", [])
def _pull_deps(self):
if not self._deps:
return
outs = [self.repo.find_out_by_relpath(d) for d in self._deps]
with self.repo.state:
for out in outs:
self.repo.cloud.pull(out.get_used_cache())
out.checkout()
def _get_object_spec(name, path):
"""
Given a summonable object's name, search for it on the given file
and return its description.
"""
try:
with builtin_open(path, "r") as fobj:
content = SUMMON_FILE_SCHEMA(ruamel.yaml.safe_load(fobj.read()))
objects = [x for x in content["objects"] if x["name"] == name]
if not objects:
raise SummonError("No object with name '{}'".format(name))
elif len(objects) >= 2:
raise SummonError(
"More than one object with name '{}'".format(name)
)
return objects[0]
except FileNotFoundError as exc:
raise SummonError("Summon file not found") from exc
except ruamel.yaml.YAMLError as exc:
raise SummonError("Failed to parse summon file") from exc
except Invalid as exc:
raise SummonError(str(exc)) from exc
@wrap_with(threading.Lock())
def _invoke_method(call, args, path):
# XXX: Some issues with this approach:
# * Import will pollute sys.modules
# * sys.path manipulation is "theoretically" not needed,
# but tests are failing for an unknown reason.
cwd = os.getcwd()
try:
os.chdir(path)
sys.path.insert(0, path)
method = _import_string(call)
return method(**args)
finally:
os.chdir(cwd)
sys.path.pop(0)
def _import_string(import_name):
"""Imports an object based on a string.
Useful to delay import to not load everything on startup.
Use dotted notaion in `import_name`, e.g. 'dvc.remote.gs.RemoteGS'.
:return: imported object
"""
if "." in import_name:
module, obj = import_name.rsplit(".", 1)
else:
return importlib.import_module(import_name)
return getattr(importlib.import_module(module), obj)
def _require_dvc(repo):
if not isinstance(repo, Repo):
raise UrlNotDvcRepoError(repo.url)
dvc/cli.py
+33 -31
View File
@@ -3,37 +3,39 @@ import argparse
import logging
import sys
import dvc.command.add as add
import dvc.command.cache as cache
import dvc.command.checkout as checkout
import dvc.command.commit as commit
import dvc.command.config as config
import dvc.command.daemon as daemon
import dvc.command.data_sync as data_sync
import dvc.command.destroy as destroy
import dvc.command.diff as diff
import dvc.command.gc as gc
import dvc.command.get as get
import dvc.command.get_url as get_url
import dvc.command.imp as imp
import dvc.command.imp_url as imp_url
import dvc.command.init as init
import dvc.command.install as install
import dvc.command.lock as lock
import dvc.command.metrics as metrics
import dvc.command.move as move
import dvc.command.pipeline as pipeline
import dvc.command.remote as remote
import dvc.command.remove as remove
import dvc.command.repro as repro
import dvc.command.root as root
import dvc.command.run as run
import dvc.command.tag as tag
import dvc.command.unprotect as unprotect
import dvc.command.update as update
import dvc.command.version as version
from dvc.command.base import fix_subparsers
from dvc.exceptions import DvcParserError
from .command import (
add,
cache,
checkout,
commit,
config,
daemon,
data_sync,
destroy,
diff,
gc,
get,
get_url,
imp,
imp_url,
init,
install,
lock,
metrics,
move,
pipeline,
remote,
remove,
repro,
root,
run,
tag,
unprotect,
update,
version,
)
from .command.base import fix_subparsers
from .exceptions import DvcParserError
logger = logging.getLogger(__name__)
dvc/command/diff.py
+116 -123
View File
@@ -1,12 +1,11 @@
import argparse
import json
import logging
import os
import humanize
import inflect
from funcy import compact
import colorama
from dvc.command.base import append_doc_link
from dvc.command.base import CmdBase
from dvc.command.base import CmdBase, append_doc_link
from dvc.exceptions import DvcException
@@ -15,129 +14,114 @@ logger = logging.getLogger(__name__)
class CmdDiff(CmdBase):
@staticmethod
def _print_size(size):
if size < 0:
change = "decreased by {}"
elif size > 0:
change = "increased by {}"
else:
change = "not changed"
natur_size = humanize.naturalsize(abs(size))
return change.format(natur_size)
def _format(diff):
"""
Given a diff structure, generate a string of paths separated
by new lines and grouped together by their state.
@staticmethod
def _get_md5_string(sign, file_name, checksum):
sample_msg = ""
if file_name:
sample_msg = "{}{} with md5 {}\n"
sample_msg = sample_msg.format(sign, file_name, checksum)
return sample_msg
A group's header is colored and its entries are sorted to enhance
readability, for example:
@classmethod
def _get_dir_changes(cls, dct):
import dvc.repo.diff as diff
Added:
another_file.txt
backup.tar
dir/
dir/1
engine = inflect.engine()
changes_msg = (
"{} {} untouched, {} {} modified, {} {} added, "
"{} {} deleted, size was {}"
An example of a diff formatted when entries contain checksums:
Added:
d3b07384 foo
Modified:
c157a790..f98bf6f1 bar
If a group has no entries, it won't be included in the result.
At the bottom, include a summary with the number of files per state.
"""
def _digest(checksum):
if type(checksum) is str:
return checksum[0:8]
return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
colors = {
"added": colorama.Fore.GREEN,
"modified": colorama.Fore.YELLOW,
"deleted": colorama.Fore.RED,
}
summary = {}
groups = []
for state in ["added", "deleted", "modified"]:
summary[state] = 0
entries = diff[state]
if not entries:
continue
content = []
for entry in entries:
path = entry["path"]
checksum = entry.get("checksum")
summary[state] += 1 if not path.endswith(os.sep) else 0
content.append(
"{space}{checksum}{separator}{path}".format(
space=" ",
checksum=_digest(checksum) if checksum else "",
separator=" " if checksum else "",
path=entry["path"],
)
)
groups.append(
"{color}{header}{nc}:\n{content}".format(
color=colors[state],
header=state.capitalize(),
nc=colorama.Fore.RESET,
content="\n".join(content),
)
)
groups.append(
"summary: added ({added}), deleted ({deleted}),"
" modified ({modified})".format_map(summary)
)
changes_msg = changes_msg.format(
dct[diff.DIFF_IDENT],
engine.plural("file", dct[diff.DIFF_IDENT]),
dct[diff.DIFF_CHANGE],
engine.plural("file", dct[diff.DIFF_CHANGE]),
dct[diff.DIFF_NEW],
engine.plural("file", dct[diff.DIFF_NEW]),
dct[diff.DIFF_DEL],
engine.plural("file", dct[diff.DIFF_DEL]),
cls._print_size(dct[diff.DIFF_SIZE]),
)
return changes_msg
@classmethod
def _get_file_changes(cls, dct):
import dvc.repo.diff as diff
if (
dct.get(diff.DIFF_OLD_FILE)
and dct.get(diff.DIFF_NEW_FILE)
and dct[diff.DIFF_SIZE] == 0
):
msg = "file size was not changed"
elif dct.get(diff.DIFF_NEW_FILE):
msg = "added file with size {}".format(
humanize.naturalsize(dct[diff.DIFF_SIZE])
)
elif dct.get(diff.DIFF_OLD_FILE):
msg = "deleted file with size {}".format(
humanize.naturalsize(abs(dct[diff.DIFF_SIZE]))
)
else:
msg = "file was modified, file size {}".format(
cls._print_size(dct[diff.DIFF_SIZE])
)
return msg
@classmethod
def _get_royal_changes(cls, dct):
import dvc.repo.diff as diff
if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
if dct.get("is_dir"):
return cls._get_dir_changes(dct)
else:
return cls._get_file_changes(dct)
return "size is ?"
@classmethod
def _show(cls, diff_dct):
import dvc.repo.diff as diff
msg = "dvc diff from {} to {}".format(
diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
)
if diff_dct.get(diff.DIFF_EQUAL):
logger.info(msg)
return
for dct in diff_dct[diff.DIFF_LIST]:
msg += "\n\ndiff for '{}'\n".format(dct[diff.DIFF_TARGET])
msg += cls._get_md5_string(
"-",
dct.get(diff.DIFF_OLD_FILE),
dct.get(diff.DIFF_OLD_CHECKSUM),
)
msg += cls._get_md5_string(
"+",
dct.get(diff.DIFF_NEW_FILE),
dct.get(diff.DIFF_NEW_CHECKSUM),
)
msg += "\n"
msg += cls._get_royal_changes(dct)
logger.info(msg)
return msg
return "\n\n".join(groups)
def run(self):
try:
msg = self.repo.diff(
self.args.a_ref, target=self.args.target, b_ref=self.args.b_ref
)
self._show(msg)
diff = self.repo.diff(self.args.a_ref, self.args.b_ref)
if not any(diff.values()):
return 0
if not self.args.checksums:
for _, entries in diff.items():
for entry in entries:
del entry["checksum"]
if self.args.show_json:
res = json.dumps(diff)
else:
res = self._format(diff)
logger.info(res)
except DvcException:
msg = "failed to get 'diff {}'"
args = " ".join(
compact([self.args.target, self.args.a_ref, self.args.b_ref])
)
msg = msg.format(args)
logger.exception(msg)
logger.exception("failed to get diff")
return 1
return 0
def add_parser(subparsers, parent_parser):
DIFF_DESCRIPTION = (
"Show changes between revisions of the DVC repository.\n"
"Summary of how many files were deleted/changed, et al."
"Compare two versions of the DVC repository.\n"
"Shows the list of paths added, modified, or deleted"
)
diff_parser = subparsers.add_parser(
"diff",
@@ -147,23 +131,32 @@ def add_parser(subparsers, parent_parser):
formatter_class=argparse.RawDescriptionHelpFormatter,
)
diff_parser.add_argument(
"-t",
"--target",
"a_ref",
help=(
"Source path to a data file or directory. Default None. "
"If not specified, compares all files and directories "
"that are under DVC control in the current working space."
"Git reference to the older version to compare"
" (defaults to `HEAD`)"
),
)
diff_parser.add_argument(
"a_ref", help="Git reference from which the diff begins"
nargs="?",
default="HEAD",
)
diff_parser.add_argument(
"b_ref",
help=(
"Git reference until which the diff ends. "
"If omitted, `HEAD` (latest commit) is used."
"Git reference to the newer version to compare"
" (defaults to the current workspace including changes)"
),
nargs="?",
)
diff_parser.add_argument(
"--show-json",
help="Format the output into a JSON",
action="store_true",
default=False,
)
diff_parser.add_argument(
"--checksums",
help="Display checksums for each entry",
action="store_true",
default=False,
)
diff_parser.set_defaults(func=CmdDiff)
dvc/output/base.py
+1 -1
View File
@@ -405,7 +405,7 @@ class OutputBase(object):
if self.stage.is_repo_import:
cache = NamedCache()
dep, = self.stage.deps
(dep,) = self.stage.deps
cache.external[dep.repo_pair].add(dep.def_path)
return cache
dvc/progress.py
+13 -7
View File
@@ -4,7 +4,6 @@ import logging
import sys
from threading import RLock
from funcy import merge
from tqdm import tqdm
from dvc.utils import env2bool
@@ -31,7 +30,10 @@ class Tqdm(tqdm):
)
BAR_FMT_NOTOTAL = (
"{desc:{ncols_desc}.{ncols_desc}}{n_fmt}"
" [{elapsed}<??:??, {rate_fmt:>11}{postfix}]"
" [{elapsed}, {rate_fmt:>11}{postfix}]"
)
BYTES_DEFAULTS = dict(
unit="B", unit_scale=True, unit_divisor=1024, miniters=1
)
def __init__(
@@ -44,6 +46,7 @@ class Tqdm(tqdm):
bar_format=None,
bytes=False, # pylint: disable=W0622
file=None,
total=None,
**kwargs
):
"""
@@ -58,12 +61,10 @@ class Tqdm(tqdm):
kwargs : anything accepted by `tqdm.tqdm()`
"""
kwargs = kwargs.copy()
kwargs.setdefault("unit_scale", True)
if bytes:
bytes_defaults = dict(
unit="B", unit_scale=True, unit_divisor=1024, miniters=1
)
kwargs = merge(bytes_defaults, kwargs)
kwargs = {**self.BYTES_DEFAULTS, **kwargs}
else:
kwargs.setdefault("unit_scale", total > 999 if total else True)
if file is None:
file = sys.stderr
self.desc_persist = desc
@@ -84,6 +85,7 @@ class Tqdm(tqdm):
desc=desc,
bar_format="!",
lock_args=(False,),
total=total,
**kwargs
)
if bar_format is None:
@@ -114,6 +116,10 @@ class Tqdm(tqdm):
def close(self):
if self.desc_persist is not None:
self.set_description_str(self.desc_persist, refresh=False)
# unknown/zero ETA
self.bar_format = self.bar_format.replace("<{remaining}", "")
# remove completed bar
self.bar_format = self.bar_format.replace("|{bar:10}|", " ")
super().close()
@property
dvc/remote/base.py
+3 -1
View File
@@ -493,7 +493,9 @@ class RemoteBASE(object):
cache_info = self.checksum_to_path_info(checksum)
dir_info = self.get_dir_cache(checksum)
for entry in dir_info:
for entry in Tqdm(
dir_info, desc="Saving " + path_info.name, unit="file"
):
entry_info = path_info / entry[self.PARAM_RELPATH]
entry_checksum = entry[self.PARAM_CHECKSUM]
self._save_file(entry_info, entry_checksum, save_link=False)
dvc/remote/gdrive.py
+47 -11
View File
@@ -104,13 +104,15 @@ class RemoteGDrive(RemoteBASE):
),
)
)
self.remote_drive_id = None
@gdrive_retry
def gdrive_upload_file(
self, args, no_progress_bar=True, from_file="", progress_name=""
):
parent = {"id": args["parent_id"]}
item = self.drive.CreateFile(
{"title": args["title"], "parents": [{"id": args["parent_id"]}]}
{"title": args["title"], "parents": [parent]}
)
with open(from_file, "rb") as fobj:
@@ -133,7 +135,9 @@ class RemoteGDrive(RemoteBASE):
def gdrive_download_file(
self, file_id, to_file, progress_name, no_progress_bar
):
gdrive_file = self.drive.CreateFile({"id": file_id})
param = {"id": file_id}
# it does not create a file on the remote
gdrive_file = self.drive.CreateFile(param)
bar_format = (
"Donwloading {desc:{ncols_desc}.{ncols_desc}}... "
+ Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
@@ -144,7 +148,12 @@ class RemoteGDrive(RemoteBASE):
gdrive_file.GetContentFile(to_file)
def gdrive_list_item(self, query):
file_list = self.drive.ListFile({"q": query, "maxResults": 1000})
param = {"q": query, "maxResults": 1000, "corpora": self.corpora}
if self.remote_drive_id:
param["driveId"] = self.remote_drive_id
file_list = self.drive.ListFile(param)
# Isolate and decorate fetching of remote drive items in pages
get_list = gdrive_retry(lambda: next(file_list, None))
@@ -175,6 +184,12 @@ class RemoteGDrive(RemoteBASE):
self.drive
return self._cached_ids
@property
def corpora(self):
if not hasattr(self, "_corpora"):
self.drive
return self._corpora
@property
@wrap_with(threading.RLock())
def drive(self):
@@ -240,21 +255,22 @@ class RemoteGDrive(RemoteBASE):
self._gdrive = GoogleDrive(gauth)
if self.bucket != "root" and self.bucket != "appDataFolder":
self.remote_drive_id = self.get_remote_drive_id(self.bucket)
self._corpora = "drive" if self.remote_drive_id else "default"
self.remote_root_id = self.get_remote_id(
self.path_info, create=True
)
self._cached_dirs, self._cached_ids = self.cache_root_dirs()
return self._gdrive
@gdrive_retry
def create_remote_dir(self, parent_id, title):
parent = {"id": parent_id}
item = self.drive.CreateFile(
{
"title": title,
"parents": [{"id": parent_id}],
"mimeType": FOLDER_MIME_TYPE,
}
{"title": title, "parents": [parent], "mimeType": FOLDER_MIME_TYPE}
)
item.Upload()
return item
@@ -272,12 +288,28 @@ class RemoteGDrive(RemoteBASE):
query += " and trashed=false and title='{}'".format(name)
param = {
"q": query,
# Remote might contain items with duplicated titles
"maxResults": 1,
"corpora": self.corpora,
}
if self.remote_drive_id:
param["driveId"] = self.remote_drive_id
# Limit found remote items count to 1 in response
item_list = self.drive.ListFile(
{"q": query, "maxResults": 1}
).GetList()
item_list = self.drive.ListFile(param).GetList()
return next(iter(item_list), None)
@gdrive_retry
def get_remote_drive_id(self, remote_id):
param = {"id": remote_id}
# it does not create a file on the remote
item = self.drive.CreateFile(param)
item.FetchMetadata("driveId")
return item.get("driveId", None)
def resolve_remote_item_from_path(self, path_parts, create):
parents_ids = [self.bucket]
current_path = ""
@@ -301,6 +333,10 @@ class RemoteGDrive(RemoteBASE):
return []
def get_remote_id(self, path_info, create=False):
if not path_info.path and path_info.bucket:
# Case sensitive base path
return self.bucket
remote_ids = self.get_remote_id_from_cache(path_info.path)
if remote_ids:
dvc/repo/__init__.py
+3 -3
View File
@@ -125,7 +125,7 @@ class Repo(object):
self._reset()
def __repr__(self):
return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
return "{}: '{}'".format(self.__class__.__name__, self.root_dir)
@classmethod
def find_root(cls, root=None):
@@ -205,7 +205,7 @@ class Repo(object):
return [(stage, None) for stage in self.stages]
try:
out, = self.find_outs_by_path(target, strict=False)
(out,) = self.find_outs_by_path(target, strict=False)
filter_info = PathInfo(os.path.abspath(target))
return [(out.stage, filter_info)]
except OutputNotFoundError:
@@ -419,7 +419,7 @@ class Repo(object):
def find_out_by_relpath(self, relpath):
path = os.path.join(self.root_dir, relpath)
out, = self.find_outs_by_path(path)
(out,) = self.find_outs_by_path(path)
return out
def is_dvc_internal(self, path):
dvc/repo/add.py
+43 -15
View File
@@ -4,11 +4,12 @@ import os
import colorama
from . import locked
from dvc.exceptions import RecursiveAddingWhileUsingFilename
from dvc.progress import Tqdm
from dvc.repo.scm_context import scm_context
from dvc.stage import Stage
from dvc.utils import LARGE_DIR_SIZE
from ..exceptions import RecursiveAddingWhileUsingFilename
from ..output.base import OutputDoesNotExistError
from ..progress import Tqdm
from ..repo.scm_context import scm_context
from ..stage import Stage
from ..utils import LARGE_DIR_SIZE
logger = logging.getLogger(__name__)
@@ -23,7 +24,12 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
targets = [targets]
stages_list = []
with Tqdm(total=len(targets), desc="Add", unit="file", leave=True) as pbar:
num_targets = len(targets)
with Tqdm(total=num_targets, desc="Add", unit="file", leave=True) as pbar:
if num_targets == 1:
# clear unneeded top-level progress bar for single target
pbar.bar_format = "Adding..."
pbar.refresh()
for target in targets:
sub_targets = _find_all_targets(repo, target, recursive)
pbar.total += len(sub_targets) - 1
@@ -45,17 +51,29 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
repo.check_modified_graph(stages)
for stage in stages:
stage.save()
with Tqdm(
total=len(stages),
desc="Processing",
unit="file",
disable=True if len(stages) == 1 else None,
) as pbar_stages:
for stage in stages:
try:
stage.save()
except OutputDoesNotExistError:
pbar.n -= 1
raise
if not no_commit:
stage.commit()
if not no_commit:
stage.commit()
stage.dump()
stage.dump()
pbar_stages.update()
stages_list += stages
# remove filled bar bit of progress, leaving stats
pbar.bar_format = pbar.BAR_FMT_DEFAULT.replace("|{bar:10}|", " ")
if num_targets == 1: # restore bar format for stats
pbar.bar_format = pbar.BAR_FMT_DEFAULT
return stages_list
@@ -64,7 +82,12 @@ def _find_all_targets(repo, target, recursive):
if os.path.isdir(target) and recursive:
return [
fname
for fname in repo.tree.walk_files(target)
for fname in Tqdm(
repo.tree.walk_files(target),
desc="Searching " + target,
bar_format=Tqdm.BAR_FMT_NOTOTAL,
unit="file",
)
if not repo.is_dvc_internal(fname)
if not Stage.is_stage_file(fname)
if not repo.scm.belongs_to_scm(fname)
@@ -76,7 +99,12 @@ def _find_all_targets(repo, target, recursive):
def _create_stages(repo, targets, fname, pbar=None):
stages = []
for out in targets:
for out in Tqdm(
targets,
desc="Creating DVC-files",
disable=True if len(targets) < LARGE_DIR_SIZE else None,
unit="file",
):
stage = Stage.create(repo, outs=[out], add=True, fname=fname)
if not stage:
dvc/repo/diff.py
+69 -239
View File
@@ -1,246 +1,76 @@
import os
from errno import ENOENT
from dvc import logger
from dvc.scm.base import FileNotInCommitError
from dvc.scm.git import DIFF_A_REF, DIFF_B_REF, DIFF_A_TREE, DIFF_B_TREE
from dvc.scm.git import DIFF_EQUAL
from . import locked
DIFF_TARGET = "target"
DIFF_IS_DIR = "is_dir"
DIFF_OLD_FILE = "old_file"
DIFF_OLD_CHECKSUM = "old_checksum"
DIFF_NEW_FILE = "new_file"
DIFF_NEW_CHECKSUM = "new_checksum"
DIFF_SIZE = "size_diff"
DIFF_DEL = "del"
DIFF_IDENT = "ident"
DIFF_CHANGE = "changes"
DIFF_NEW = "new"
DIFF_MOVE = "moves"
DIFF_LIST = "diffs"
DIFF_SIZE_UNKNOWN = "?"
DIFF_A_OUTPUT = "a_output"
DIFF_B_OUTPUT = "b_output"
DIFF_DELETED = "deleted_file"
DIFF_IS_NEW = "created_file"
def _file_not_exists(error, result):
if error.errno == ENOENT:
result.update({DIFF_SIZE: DIFF_SIZE_UNKNOWN})
else:
raise error
def _extract_dir(self, dir_not_exists, output):
"""Extract the content of dvc tree file
Args:
self(object) - Repo class instance
dir_not_exists(bool) - flag for directory existence
output(object) - OutputLOCAL class instance
Returns:
dict - dictionary with keys - paths to file in .dvc/cache
values -checksums for that files
"""
if not dir_not_exists:
lst = output.dir_cache
return {i["relpath"]: i["md5"] for i in lst}
return {}
def _get_dir_info(dir_not_exists, a_output):
if not dir_not_exists:
return str(a_output), a_output.checksum
return "", ""
def _ident_files(a_entries, b_entries):
keys = [
key for key in a_entries.keys() if b_entries.get(key) == a_entries[key]
]
return len(keys)
def _modified_files(self, a_entries, b_entries):
keys = [key for key in a_entries.keys() if key in b_entries]
diff_size = 0
modified_count = 0
for key in keys:
if a_entries[key] != b_entries[key]:
modified_count += 1
diff_size += os.path.getsize(
self.cache.local.get(b_entries[key])
) - os.path.getsize(self.cache.local.get(a_entries[key]))
return modified_count, diff_size
def _deleted_files(self, a_entries, b_entries):
diff_size = 0
deleted_count = 0
for key, value in a_entries.items():
if key not in b_entries:
deleted_count += 1
diff_size -= os.path.getsize(self.cache.local.get(a_entries[key]))
return deleted_count, diff_size
def _new_files(self, a_entries, b_entries):
diff_size = 0
new_count = 0
for key, value in b_entries.items():
if key not in a_entries:
new_count += 1
diff_size += os.path.getsize(self.cache.local.get(b_entries[key]))
return new_count, diff_size
def _get_tree_changes(self, a_entries, b_entries):
result = {
DIFF_DEL: 0,
DIFF_IDENT: 0,
DIFF_CHANGE: 0,
DIFF_NEW: 0,
DIFF_MOVE: 0,
}
result[DIFF_IDENT] = _ident_files(a_entries, b_entries)
result[DIFF_CHANGE], diff_size = _modified_files(
self, a_entries, b_entries
)
result[DIFF_SIZE] = diff_size
result[DIFF_DEL], diff_size = _deleted_files(self, a_entries, b_entries)
result[DIFF_SIZE] += diff_size
result[DIFF_NEW], diff_size = _new_files(self, a_entries, b_entries)
result[DIFF_SIZE] += diff_size
return result
def _check_local_cache(a_out, is_checked):
if a_out is not None and a_out.scheme != "local":
is_checked.append(str(a_out))
return True
return False
def _is_dir(path, a_outs, b_outs):
if a_outs.get(path):
return a_outs[path].is_dir_checksum
else:
return b_outs[path].is_dir_checksum
def _get_diff_outs(self, diff_dct):
self.tree = diff_dct[DIFF_A_TREE]
a_outs = {str(out): out for st in self.stages for out in st.outs}
self.tree = diff_dct[DIFF_B_TREE]
b_outs = {str(out): out for st in self.stages for out in st.outs}
outs_paths = set(a_outs.keys())
outs_paths.update(b_outs.keys())
results = {}
non_local_cache = []
for path in outs_paths:
check1 = _check_local_cache(a_outs.get(path), non_local_cache)
check2 = _check_local_cache(b_outs.get(path), non_local_cache)
# skip files/directories with non-local cache for now
if check1 or check2:
continue
results[path] = {}
results[path][DIFF_A_OUTPUT] = a_outs.get(path)
results[path][DIFF_B_OUTPUT] = b_outs.get(path)
results[path][DIFF_IS_NEW] = path not in a_outs
results[path][DIFF_DELETED] = path not in b_outs
results[path][DIFF_IS_DIR] = _is_dir(path, a_outs, b_outs)
if non_local_cache:
logger.warning(
"Diff is not supported for non-local outputs. Ignoring: {}".format(
non_local_cache
)
)
return results
def _diff_dir(self, target, diff_dct):
result = {DIFF_TARGET: target}
result[DIFF_IS_DIR] = True
a_entries, b_entries = {}, {}
try:
a_entries = _extract_dir(
self, diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
)
b_entries = _extract_dir(
self, diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
)
result[DIFF_OLD_FILE], result[DIFF_OLD_CHECKSUM] = _get_dir_info(
diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
)
result[DIFF_NEW_FILE], result[DIFF_NEW_CHECKSUM] = _get_dir_info(
diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
)
result.update(_get_tree_changes(self, a_entries, b_entries))
except IOError as e:
_file_not_exists(e, result)
return result
def _diff_file(self, target, diff_dct):
result = {DIFF_TARGET: target}
size = 0
try:
if not diff_dct[DIFF_IS_NEW]:
result[DIFF_OLD_FILE] = target
result[DIFF_OLD_CHECKSUM] = diff_dct[DIFF_A_OUTPUT].checksum
size -= os.path.getsize(
self.cache.local.get(diff_dct[DIFF_A_OUTPUT].checksum)
)
if not diff_dct[DIFF_DELETED]:
result[DIFF_NEW_FILE] = target
result[DIFF_NEW_CHECKSUM] = diff_dct[DIFF_B_OUTPUT].checksum
size += os.path.getsize(
self.cache.local.get(diff_dct[DIFF_B_OUTPUT].checksum)
)
result[DIFF_SIZE] = size
except IOError as e:
_file_not_exists(e, result)
return result
def _diff_royal(self, target, diff_dct):
if diff_dct[DIFF_IS_DIR]:
return _diff_dir(self, target, diff_dct)
return _diff_file(self, target, diff_dct)
from dvc.exceptions import DvcException
from dvc.repo import locked
from dvc.scm.git import Git
@locked
def diff(self, a_ref, target=None, b_ref=None):
"""Generates diff message string output
Args:
target(str) - file/directory to check diff of
a_ref(str) - first tag
(optional) b_ref(str) - second git tag
Returns:
string: string of output message with diff info
def diff(self, a_ref="HEAD", b_ref=None):
"""
result = {}
diff_dct = self.scm.get_diff_trees(a_ref, b_ref=b_ref)
result[DIFF_A_REF] = diff_dct[DIFF_A_REF]
result[DIFF_B_REF] = diff_dct[DIFF_B_REF]
if diff_dct[DIFF_EQUAL]:
result[DIFF_EQUAL] = True
By default, it compares the working tree with the last commit's tree.
This implementation differs from `git diff` since DVC doesn't have
the concept of `index`, but it keeps the same interface, thus,
`dvc diff` would be the same as `dvc diff HEAD`.
"""
if type(self.scm) is not Git:
raise DvcException("only supported for Git repositories")
def _paths_checksums():
"""
A dictionary of checksums addressed by relpaths collected from
the current tree outputs.
Unpack directories to include their entries
To help distinguish between a directory and a file output,
the former one will come with a trailing slash in the path:
directory: "data/"
file: "data"
"""
result = {}
for stage in self.stages:
for output in stage.outs:
if not output.is_dir_checksum:
result.update({str(output): output.checksum})
continue
result.update({os.path.join(str(output), ""): output.checksum})
for entry in output.dir_cache:
path = str(output.path_info / entry["relpath"])
result.update({path: entry["md5"]})
return result
result[DIFF_LIST] = []
diff_outs = _get_diff_outs(self, diff_dct)
if target is None:
result[DIFF_LIST] = [
_diff_royal(self, path, diff_outs[path]) for path in diff_outs
]
elif target in diff_outs:
result[DIFF_LIST] = [_diff_royal(self, target, diff_outs[target])]
else:
msg = "Have not found file/directory '{}' in the commits"
raise FileNotInCommitError(msg.format(target))
return result
working_tree = self.tree
a_tree = self.scm.get_tree(a_ref)
b_tree = self.scm.get_tree(b_ref) if b_ref else working_tree
try:
self.tree = a_tree
old = _paths_checksums()
self.tree = b_tree
new = _paths_checksums()
finally:
self.tree = working_tree
# Compare paths between the old and new tree.
# set() efficiently converts dict keys to a set
added = sorted(set(new) - set(old))
deleted = sorted(set(old) - set(new))
modified = sorted(set(old) & set(new))
return {
"added": [{"path": path, "checksum": new[path]} for path in added],
"deleted": [{"path": path, "checksum": old[path]} for path in deleted],
"modified": [
{"path": path, "checksum": {"old": old[path], "new": new[path]}}
for path in modified
if old[path] != new[path]
],
}
dvc/repo/get_url.py
+2 -2
View File
@@ -14,6 +14,6 @@ def get_url(url, out=None):
out = os.path.abspath(out)
dep, = dependency.loads_from(None, [url])
out, = output.loads_from(None, [out], use_cache=False)
(dep,) = dependency.loads_from(None, [url])
(out,) = output.loads_from(None, [out], use_cache=False)
dep.download(out)
dvc/scm/base.py
+1 -10
View File
@@ -15,22 +15,13 @@ class FileNotInRepoError(SCMError):
"""
class FileNotInCommitError(SCMError):
"""Thrown when trying to find a file/directory that is not
in the specified commit in the repository.
"""
class CloneError(SCMError):
def __init__(self, url, path):
super().__init__("Failed to clone repo '{}' to '{}'".format(url, path))
class RevError(SCMError):
def __init__(self, url, rev):
super().__init__(
"Failed to access revision '{}' for repo '{}'".format(rev, url)
)
pass
class Base(object):
dvc/scm/git/__init__.py
+26 -66
View File
@@ -22,13 +22,6 @@ from dvc.utils.fs import path_isin
logger = logging.getLogger(__name__)
DIFF_A_TREE = "a_tree"
DIFF_B_TREE = "b_tree"
DIFF_A_REF = "a_ref"
DIFF_B_REF = "b_ref"
DIFF_EQUAL = "equal"
class Git(Base):
"""Class for managing Git."""
@@ -94,7 +87,11 @@ class Git(Base):
try:
repo.checkout(rev)
except git.exc.GitCommandError as exc:
raise RevError(url, rev) from exc
raise RevError(
"failed to access revision '{}' for repo '{}'".format(
rev, url
)
) from exc
return repo
@@ -215,6 +212,16 @@ class Git(Base):
else:
self.repo.git.checkout(branch)
def pull(self):
info, = self.repo.remote().pull()
if info.flags & info.ERROR:
raise SCMError("pull failed: {}".format(info.note))
def push(self):
info, = self.repo.remote().push()
if info.flags & info.ERROR:
raise SCMError("push failed: {}".format(info.summary))
def branch(self, branch):
self.repo.git.branch(branch)
@@ -275,9 +282,10 @@ class Git(Base):
[
# checking out some reference and not specific file.
'[ "$3" = "1" ]',
# check that we are on some branch/tag and not in detached HEAD
# state, so we don't accidentally break a rebase.
'[ "$(git rev-parse --abbrev-ref HEAD)" != "HEAD" ]',
# make sure we are not in the middle of a rebase/merge, so we
# don't accidentally break it with an unsuccessful checkout.
# Note that git hooks are always running in repo root.
"[ ! -d .git/rebase-merge ]",
],
"checkout",
)
@@ -315,66 +323,18 @@ class Git(Base):
return basename == self.ignore_file or Git.GIT_DIR in path_parts
def get_tree(self, rev):
return GitTree(self.repo, rev)
def _get_diff_trees(self, a_ref, b_ref):
"""Private method for getting the trees and commit hashes of 2 git
references. Requires `gitdb` module (from gitpython package).
Args:
a_ref (str): git reference
b_ref (str): second git reference. If None, uses HEAD
Returns:
tuple: tuple with elements: (trees, commits)
"""
from gitdb.exc import BadObject, BadName
trees = {DIFF_A_TREE: None, DIFF_B_TREE: None}
commits = []
if b_ref is None:
b_ref = self.repo.head.commit
try:
a_commit = self.repo.git.rev_parse(a_ref, short=True)
b_commit = self.repo.git.rev_parse(b_ref, short=True)
# See https://gitpython.readthedocs.io
# /en/2.1.11/reference.html#git.objects.base.Object.__str__
commits.append(a_commit)
commits.append(b_commit)
trees[DIFF_A_TREE] = self.get_tree(commits[0])
trees[DIFF_B_TREE] = self.get_tree(commits[1])
except (BadName, BadObject) as exc:
raise SCMError("git problem") from exc
return trees, commits
def get_diff_trees(self, a_ref, b_ref=None):
"""Method for getting two repo trees between two git tag commits.
Returns the dvc hash names of changed file/directory
Args:
a_ref (str): git reference
b_ref (str): optional second git reference, default None
Returns:
dict: dictionary with keys: {a_ref, b_ref, equal}
or {a_ref, b_ref, a_tree, b_tree}
"""
diff_dct = {DIFF_EQUAL: False}
trees, commits = self._get_diff_trees(a_ref, b_ref)
diff_dct[DIFF_A_REF] = commits[0]
diff_dct[DIFF_B_REF] = commits[1]
if commits[0] == commits[1]:
diff_dct[DIFF_EQUAL] = True
return diff_dct
diff_dct[DIFF_A_TREE] = trees[DIFF_A_TREE]
diff_dct[DIFF_B_TREE] = trees[DIFF_B_TREE]
return diff_dct
return GitTree(self.repo, self.resolve_rev(rev))
def get_rev(self):
return self.repo.git.rev_parse("HEAD")
def resolve_rev(self, rev):
return self.repo.git.rev_parse(rev)
from git.exc import GitCommandError
try:
return self.repo.git.rev_parse(rev)
except GitCommandError:
raise RevError("unknown Git revision '{}'".format(rev))
def close(self):
self.repo.close()
dvc/version.py
+1 -1
View File
@@ -7,7 +7,7 @@ import os
import subprocess
_BASE_VERSION = "0.82.3"
_BASE_VERSION = "0.82.6"
def _generate_version(base_version):
scripts/completion/dvc.bash
+1 -1
View File
@@ -22,7 +22,7 @@ _dvc_checkout='-d --with-deps -R --recursive -f --force --relink $(compgen -G *.
_dvc_commit='-f --force -d --with-deps -R --recursive $(compgen -G *.dvc)'
_dvc_config='-u --unset --local --system --global'
_dvc_destroy='-f --force'
_dvc_diff='-t --target'
_dvc_diff='-t --show-json --checksums'
_dvc_fetch='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive $(compgen -G *.dvc)'
_dvc_get_url=''
_dvc_get='-o --out --rev --show-url'
scripts/completion/dvc.zsh
+3 -1
View File
@@ -97,7 +97,9 @@ _dvc_destroy=(
)
_dvc_diff=(
{-t,--target}"[Source path to a data file or directory.]:Target files:"
"--show-json[Format the output into a JSON]"
"--checksums[Display checksums for each entry]"
{1,2}":Git revision (e.g. branch, tag, SHA):"
)
_dvc_fetch=(
setup.py
+1 -1
View File
@@ -84,7 +84,7 @@ install_requires = [
# Extra dependencies for remote integrations
gs = ["google-cloud-storage==1.19.0"]
gdrive = ["pydrive2>=1.4.1"]
gdrive = ["pydrive2>=1.4.2"]
s3 = ["boto3>=1.9.201"]
azure = ["azure-storage-blob==2.1.0"]
oss = ["oss2==2.6.1"]
tests/func/test_api.py
+1 -68
View File
@@ -1,12 +1,10 @@
import os
import shutil
import copy
import ruamel.yaml
import pytest
from dvc import api
from dvc.api import SummonError, UrlNotDvcRepoError
from dvc.api import UrlNotDvcRepoError
from dvc.compat import fspath
from dvc.exceptions import FileMissingError
from dvc.main import main
@@ -141,68 +139,3 @@ def test_open_not_cached(dvc):
os.remove(metric_file)
with pytest.raises(FileMissingError):
api.read(metric_file)
def test_summon(tmp_dir, dvc, erepo_dir):
objects = {
"objects": [
{
"name": "sum",
"meta": {"description": "Add <x> to <number>"},
"summon": {
"type": "python",
"call": "calculator.add_to_num",
"args": {"x": 1},
"deps": ["number"],
},
}
]
}
other_objects = copy.deepcopy(objects)
other_objects["objects"][0]["summon"]["args"]["x"] = 100
dup_objects = copy.deepcopy(objects)
dup_objects["objects"] *= 2
with erepo_dir.chdir():
erepo_dir.dvc_gen("number", "100", commit="Add number.dvc")
erepo_dir.scm_gen("dvcsummon.yaml", ruamel.yaml.dump(objects))
erepo_dir.scm_gen("other.yaml", ruamel.yaml.dump(other_objects))
erepo_dir.scm_gen("dup.yaml", ruamel.yaml.dump(dup_objects))
erepo_dir.scm_gen("invalid.yaml", ruamel.yaml.dump({"name": "sum"}))
erepo_dir.scm_gen("not_yaml.yaml", "a: - this is not a YAML file")
erepo_dir.scm_gen(
"calculator.py",
"def add_to_num(x): return x + int(open('number').read())",
)
erepo_dir.scm.commit("Add files")
repo_url = "file://{}".format(erepo_dir)
assert api.summon("sum", repo=repo_url) == 101
assert api.summon("sum", repo=repo_url, args={"x": 2}) == 102
assert api.summon("sum", repo=repo_url, summon_file="other.yaml") == 200
try:
api.summon("sum", repo=repo_url, summon_file="missing.yaml")
except SummonError as exc:
assert "Summon file not found" in str(exc)
assert "missing.yaml" in str(exc)
assert repo_url in str(exc)
else:
pytest.fail("Did not raise on missing summon file")
with pytest.raises(SummonError, match=r"No object with name 'missing'"):
api.summon("missing", repo=repo_url)
with pytest.raises(
SummonError, match=r"More than one object with name 'sum'"
):
api.summon("sum", repo=repo_url, summon_file="dup.yaml")
with pytest.raises(SummonError, match=r"extra keys not allowed"):
api.summon("sum", repo=repo_url, summon_file="invalid.yaml")
with pytest.raises(SummonError, match=r"Failed to parse summon file"):
api.summon("sum", repo=repo_url, summon_file="not_yaml.yaml")
tests/func/test_diff.py
+145 -240
View File
@@ -1,267 +1,172 @@
import hashlib
import os
import pytest
import shutil
import dvc.repo.diff as diff
from dvc.main import main
from dvc.scm.base import FileNotInCommitError
from tests.basic_env import TestDvcGit
from dvc.compat import fspath
from dvc.exceptions import DvcException
class TestDiff(TestDvcGit):
def setUp(self):
super().setUp()
self.new_file = "new_test_file"
self.create(self.new_file, self.new_file)
stage = self.dvc.add(self.new_file)[0]
self.a_ref = self.git.git.rev_parse(self.git.head.commit, short=True)
self.new_checksum = stage.outs[0].checksum
self.git.index.add([self.new_file + ".dvc"])
self.git.index.commit("adds new_file")
self.test_dct = {
diff.DIFF_A_REF: self.a_ref,
diff.DIFF_B_REF: self.git.git.rev_parse(
self.git.head.commit, short=True
),
diff.DIFF_LIST: [
{
diff.DIFF_TARGET: self.new_file,
diff.DIFF_NEW_FILE: self.new_file,
diff.DIFF_NEW_CHECKSUM: self.new_checksum,
diff.DIFF_SIZE: 13,
}
],
}
def test(self):
out = self.dvc.scm.get_diff_trees(self.a_ref)
self.assertFalse(out[diff.DIFF_EQUAL])
self.assertEqual(self.a_ref, out[diff.DIFF_A_REF])
self.assertEqual(
self.git.git.rev_parse(self.git.head.commit, short=True),
out[diff.DIFF_B_REF],
)
def digest(text):
return hashlib.md5(bytes(text, "utf-8")).hexdigest()
class TestDiffRepo(TestDiff):
def test(self):
result = self.dvc.diff(self.a_ref, target=self.new_file)
self.assertEqual(self.test_dct, result)
def test_no_scm(tmp_dir, dvc):
tmp_dir.dvc_gen("file", "text")
with pytest.raises(DvcException, match=r"only supported for Git repos"):
dvc.diff()
class TestDiffCmdLine(TestDiff):
def test(self):
ret = main(["diff", "-t", self.new_file, self.a_ref])
self.assertEqual(ret, 0)
def test_added(tmp_dir, scm, dvc):
tmp_dir.dvc_gen("file", "text")
assert dvc.diff() == {
"added": [{"path": "file", "checksum": digest("text")}],
"deleted": [],
"modified": [],
}
class TestDiffDir(TestDvcGit):
def setUp(self):
super().setUp()
def test_no_cache_entry(tmp_dir, scm, dvc):
tmp_dir.dvc_gen("file", "first", commit="add a file")
stage = self.dvc.add(self.DATA_DIR)[0]
self.git.index.add([self.DATA_DIR + ".dvc"])
self.git.index.commit("adds data_dir")
self.a_ref = self.git.git.rev_parse(
self.dvc.scm.repo.head.commit, short=True
)
self.old_checksum = stage.outs[0].checksum
self.new_file = os.path.join(self.DATA_SUB_DIR, diff.DIFF_NEW_FILE)
self.create(self.new_file, self.new_file)
stage = self.dvc.add(self.DATA_DIR)[0]
self.git.index.add([self.DATA_DIR + ".dvc"])
self.git.index.commit(message="adds data_dir with new_file")
self.new_checksum = stage.outs[0].checksum
tmp_dir.dvc_gen({"dir": {"1": "1", "2": "2"}})
tmp_dir.dvc_gen("file", "second")
def test(self):
out = self.dvc.scm.get_diff_trees(self.a_ref)
self.assertFalse(out[diff.DIFF_EQUAL])
self.assertEqual(self.a_ref, out[diff.DIFF_A_REF])
self.assertEqual(
self.git.git.rev_parse(self.git.head.commit, short=True),
out[diff.DIFF_B_REF],
)
shutil.rmtree(fspath(tmp_dir / ".dvc" / "cache"))
(tmp_dir / ".dvc" / "state").unlink()
dir_checksum = "5fb6b29836c388e093ca0715c872fe2a.dir"
assert dvc.diff() == {
"added": [
{"path": os.path.join("dir", ""), "checksum": dir_checksum},
{"path": os.path.join("dir", "1"), "checksum": digest("1")},
{"path": os.path.join("dir", "2"), "checksum": digest("2")},
],
"deleted": [],
"modified": [
{
"path": "file",
"checksum": {"old": digest("first"), "new": digest("second")},
}
],
}
class TestDiffDirRepo(TestDiffDir):
maxDiff = None
def test_deleted(tmp_dir, scm, dvc):
tmp_dir.dvc_gen("file", "text", commit="add file")
(tmp_dir / "file.dvc").unlink()
def test(self):
result = self.dvc.diff(self.a_ref, target=self.DATA_DIR)
test_dct = {
diff.DIFF_A_REF: self.git.git.rev_parse(self.a_ref, short=True),
diff.DIFF_B_REF: self.git.git.rev_parse(
self.git.head.commit, short=True
),
diff.DIFF_LIST: [
{
diff.DIFF_CHANGE: 0,
diff.DIFF_DEL: 0,
diff.DIFF_IDENT: 2,
diff.DIFF_MOVE: 0,
diff.DIFF_NEW: 1,
diff.DIFF_IS_DIR: True,
diff.DIFF_TARGET: self.DATA_DIR,
diff.DIFF_NEW_FILE: self.DATA_DIR,
diff.DIFF_NEW_CHECKSUM: self.new_checksum,
diff.DIFF_OLD_FILE: self.DATA_DIR,
diff.DIFF_OLD_CHECKSUM: self.old_checksum,
diff.DIFF_SIZE: 30,
}
],
}
self.assertEqual(test_dct, result)
assert dvc.diff() == {
"added": [],
"deleted": [{"path": "file", "checksum": digest("text")}],
"modified": [],
}
class TestDiffDirRepoDeletedFile(TestDiffDir):
maxDiff = None
def test_modified(tmp_dir, scm, dvc):
tmp_dir.dvc_gen("file", "first", commit="first version")
tmp_dir.dvc_gen("file", "second")
def setUp(self):
super().setUp()
self.b_ref = self.a_ref
tmp = self.new_checksum
self.new_checksum = self.old_checksum
self.a_ref = str(self.dvc.scm.repo.head.commit)
self.old_checksum = tmp
def test(self):
result = self.dvc.diff(
self.a_ref, b_ref=self.b_ref, target=self.DATA_DIR
)
test_dct = {
diff.DIFF_A_REF: self.git.git.rev_parse(self.a_ref, short=True),
diff.DIFF_B_REF: self.git.git.rev_parse(self.b_ref, short=True),
diff.DIFF_LIST: [
{
diff.DIFF_CHANGE: 0,
diff.DIFF_DEL: 1,
diff.DIFF_IDENT: 2,
diff.DIFF_MOVE: 0,
diff.DIFF_NEW: 0,
diff.DIFF_IS_DIR: True,
diff.DIFF_TARGET: self.DATA_DIR,
diff.DIFF_NEW_FILE: self.DATA_DIR,
diff.DIFF_NEW_CHECKSUM: self.new_checksum,
diff.DIFF_OLD_FILE: self.DATA_DIR,
diff.DIFF_OLD_CHECKSUM: self.old_checksum,
diff.DIFF_SIZE: -30,
}
],
}
self.assertEqual(test_dct, result)
assert dvc.diff() == {
"added": [],
"deleted": [],
"modified": [
{
"path": "file",
"checksum": {"old": digest("first"), "new": digest("second")},
}
],
}
class TestDiffFileNotFound(TestDiffDir):
def setUp(self):
super().setUp()
self.unknown_file = "unknown_file_" + str(id(self))
def test_refs(tmp_dir, scm, dvc):
tmp_dir.dvc_gen("file", "first", commit="first version")
tmp_dir.dvc_gen("file", "second", commit="second version")
tmp_dir.dvc_gen("file", "third", commit="third version")
def test(self):
with self.assertRaises(FileNotInCommitError):
self.dvc.diff(self.a_ref, target=self.unknown_file)
HEAD_2 = digest("first")
HEAD_1 = digest("second")
HEAD = digest("third")
assert dvc.diff("HEAD~1") == {
"added": [],
"deleted": [],
"modified": [
{"path": "file", "checksum": {"old": HEAD_1, "new": HEAD}}
],
}
assert dvc.diff("HEAD~2", "HEAD~1") == {
"added": [],
"deleted": [],
"modified": [
{"path": "file", "checksum": {"old": HEAD_2, "new": HEAD_1}}
],
}
with pytest.raises(DvcException, match=r"unknown Git revision 'missing'"):
dvc.diff("missing")
class TestDiffModifiedFile(TestDiff):
maxDiff = None
def test_directories(tmp_dir, scm, dvc):
tmp_dir.dvc_gen({"dir": {"1": "1", "2": "2"}}, commit="add a directory")
tmp_dir.dvc_gen({"dir": {"3": "3"}}, commit="add a file")
tmp_dir.dvc_gen({"dir": {"2": "two"}}, commit="modify a file")
def setUp(self):
super().setUp()
(tmp_dir / "dir" / "2").unlink()
dvc.add("dir")
scm.add("dir.dvc")
scm.commit("delete a file")
self.old_checksum = self.new_checksum
self.new_file_content = "new_test_file_bigger_content_123456789"
self.diff_len = len(self.new_file) + len(self.new_file_content)
self.create(self.new_file, self.new_file_content)
stage = self.dvc.add(self.new_file)[0]
self.git.index.add([self.new_file + ".dvc"])
self.git.index.commit("change new_file content to be bigger")
self.new_checksum = stage.outs[0].checksum
self.b_ref = self.git.git.rev_parse(self.git.head.commit, short=True)
# The ":/<text>" format is a way to specify revisions by commit message:
# https://git-scm.com/docs/revisions
#
assert dvc.diff(":/init", ":/directory") == {
"added": [
{
"path": os.path.join("dir", ""),
"checksum": "5fb6b29836c388e093ca0715c872fe2a.dir",
},
{"path": os.path.join("dir", "1"), "checksum": digest("1")},
{"path": os.path.join("dir", "2"), "checksum": digest("2")},
],
"deleted": [],
"modified": [],
}
def test(self):
result = self.dvc.diff(
self.a_ref, b_ref=self.b_ref, target=self.new_file
)
test_dct = {
diff.DIFF_A_REF: self.git.git.rev_parse(self.a_ref, short=True),
diff.DIFF_B_REF: self.git.git.rev_parse(self.b_ref, short=True),
diff.DIFF_LIST: [
{
diff.DIFF_NEW_CHECKSUM: self.new_checksum,
diff.DIFF_NEW_FILE: self.new_file,
diff.DIFF_TARGET: self.new_file,
diff.DIFF_SIZE: self.diff_len,
}
],
}
self.assertEqual(test_dct, result)
assert dvc.diff(":/directory", ":/modify") == {
"added": [{"path": os.path.join("dir", "3"), "checksum": digest("3")}],
"deleted": [],
"modified": [
{
"path": os.path.join("dir", ""),
"checksum": {
"old": "5fb6b29836c388e093ca0715c872fe2a.dir",
"new": "9b5faf37366b3370fd98e3e60ca439c1.dir",
},
},
{
"path": os.path.join("dir", "2"),
"checksum": {"old": digest("2"), "new": digest("two")},
},
],
}
class TestDiffDirWithFile(TestDiffDir):
maxDiff = None
def setUp(self):
super().setUp()
self.a_ref = self.git.git.rev_parse(self.git.head.commit, short=True)
self.old_checksum = self.new_checksum
self.new_file_content = "new_test_file_bigger_content_123456789"
self.diff_len = len(self.new_file_content)
self.create(self.new_file, self.new_file_content)
stage = self.dvc.add(self.DATA_DIR)[0]
self.git.index.add([self.DATA_DIR + ".dvc"])
self.git.index.commit(message="modify file in the data dir")
self.new_checksum = stage.outs[0].checksum
self.b_ref = self.git.git.rev_parse(self.git.head.commit, short=True)
def test(self):
result = self.dvc.diff(self.a_ref, target=self.DATA_DIR)
test_dct = {
diff.DIFF_A_REF: self.git.git.rev_parse(self.a_ref, short=True),
diff.DIFF_B_REF: self.git.git.rev_parse(self.b_ref, short=True),
diff.DIFF_LIST: [
{
diff.DIFF_IDENT: 2,
diff.DIFF_CHANGE: 1,
diff.DIFF_DEL: 0,
diff.DIFF_MOVE: 0,
diff.DIFF_NEW: 0,
diff.DIFF_IS_DIR: True,
diff.DIFF_TARGET: self.DATA_DIR,
diff.DIFF_NEW_FILE: self.DATA_DIR,
diff.DIFF_NEW_CHECKSUM: self.new_checksum,
diff.DIFF_OLD_FILE: self.DATA_DIR,
diff.DIFF_OLD_CHECKSUM: self.old_checksum,
diff.DIFF_SIZE: self.diff_len,
}
],
}
self.assertEqual(test_dct, result)
class TestDiffCmdMessage(TestDiff):
maxDiff = None
def test(self):
ret = main(
[
"diff",
self.test_dct[diff.DIFF_A_REF],
self.test_dct[diff.DIFF_B_REF],
]
)
self.assertEqual(ret, 0)
msg1 = "dvc diff from {0} to {1}".format(
self.git.git.rev_parse(self.test_dct[diff.DIFF_A_REF], short=True),
self.git.git.rev_parse(self.test_dct[diff.DIFF_B_REF], short=True),
)
msg2 = "diff for '{0}'".format(
self.test_dct[diff.DIFF_LIST][0][diff.DIFF_TARGET]
)
msg3 = "+{0} with md5 {1}".format(
self.test_dct[diff.DIFF_LIST][0][diff.DIFF_TARGET],
self.test_dct[diff.DIFF_LIST][0][diff.DIFF_NEW_CHECKSUM],
)
msg4 = "added file with size 13 Bytes"
for m in [msg1, msg2, msg3, msg4]:
assert m in self._caplog.text
assert dvc.diff(":/modify", ":/delete") == {
"added": [],
"deleted": [
{"path": os.path.join("dir", "2"), "checksum": digest("two")}
],
"modified": [
{
"path": os.path.join("dir", ""),
"checksum": {
"old": "9b5faf37366b3370fd98e3e60ca439c1.dir",
"new": "83ae82fb367ac9926455870773ff09e6.dir",
},
}
],
}
tests/unit/command/test_diff.py
+93
View File
@@ -0,0 +1,93 @@
import collections
import os
from dvc.cli import parse_args
def test_default(mocker, caplog):
args = parse_args(["diff"])
cmd = args.func(args)
diff = {
"added": [{"path": "file", "checksum": "00000000"}],
"deleted": [],
"modified": [],
}
mocker.patch("dvc.repo.Repo.diff", return_value=diff)
assert 0 == cmd.run()
assert (
"Added:\n"
" file\n"
"\n"
"summary: added (1), deleted (0), modified (0)"
) in caplog.text
def test_checksums(mocker, caplog):
args = parse_args(["diff", "--checksums"])
cmd = args.func(args)
diff = {
"added": [],
"deleted": [
{"path": os.path.join("data", ""), "checksum": "XXXXXXXX.dir"},
{"path": os.path.join("data", "bar"), "checksum": "00000000"},
{"path": os.path.join("data", "foo"), "checksum": "11111111"},
],
"modified": [
{
"path": "file",
"checksum": {"old": "AAAAAAAA", "new": "BBBBBBBB"},
}
],
}
mocker.patch("dvc.repo.Repo.diff", return_value=diff)
assert 0 == cmd.run()
assert (
"Deleted:\n"
" XXXXXXXX " + os.path.join("data", "") + "\n"
" 00000000 " + os.path.join("data", "bar") + "\n"
" 11111111 " + os.path.join("data", "foo") + "\n"
"\n"
"Modified:\n"
" AAAAAAAA..BBBBBBBB file\n"
"\n"
"summary: added (0), deleted (2), modified (1)"
) in caplog.text
def test_json(mocker, caplog):
args = parse_args(["diff", "--show-json"])
cmd = args.func(args)
diff = {
"added": [{"path": "file", "checksum": "00000000"}],
"deleted": [],
"modified": [],
}
mocker.patch("dvc.repo.Repo.diff", return_value=diff)
assert 0 == cmd.run()
assert '"added": [{"path": "file"}]' in caplog.text
assert '"deleted": []' in caplog.text
assert '"modified": []' in caplog.text
def test_json_checksums(mocker, caplog):
args = parse_args(["diff", "--show-json", "--checksums"])
cmd = args.func(args)
diff = {
"added": [
# py35: maintain a consistent key order for tests purposes
collections.OrderedDict(
[("path", "file"), ("checksum", "00000000")]
)
],
"deleted": [],
"modified": [],
}
mocker.patch("dvc.repo.Repo.diff", return_value=diff)
assert 0 == cmd.run()
assert '"added": [{"path": "file", "checksum": "00000000"}]' in caplog.text
assert '"deleted": []' in caplog.text
assert '"modified": []' in caplog.text