diff --git a/dvc/api.py b/dvc/api.py index 7efaf524a..48a24c494 100644 --- a/dvc/api.py +++ b/dvc/api.py @@ -1,48 +1,11 @@ -from builtins import open as builtin_open -import importlib import os -import sys from contextlib import contextmanager, _GeneratorContextManager as GCM -import threading - -from funcy import wrap_with -import ruamel.yaml -from voluptuous import Schema, Required, Invalid from dvc.repo import Repo from dvc.exceptions import DvcException, NotDvcRepoError from dvc.external_repo import external_repo -SUMMON_FILE_SCHEMA = Schema( - { - Required("objects"): [ - { - Required("name"): str, - "meta": dict, - Required("summon"): { - Required("type"): str, - "deps": [str], - str: object, - }, - } - ] - } -) -SUMMON_PYTHON_SCHEMA = Schema( - { - Required("type"): "python", - Required("call"): str, - "args": dict, - "deps": [str], - } -) - - -class SummonError(DvcException): - pass - - class UrlNotDvcRepoError(DvcException): """Thrown if given url is not a DVC repository. @@ -120,128 +83,6 @@ def _make_repo(repo_url=None, rev=None): yield repo -def summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml", args=None): - """Instantiate an object described in the `summon_file`.""" - with prepare_summon( - name, repo=repo, rev=rev, summon_file=summon_file - ) as desc: - try: - summon_dict = SUMMON_PYTHON_SCHEMA(desc.obj["summon"]) - except Invalid as exc: - raise SummonError(str(exc)) from exc - - _args = {**summon_dict.get("args", {}), **(args or {})} - return _invoke_method(summon_dict["call"], _args, desc.repo.root_dir) - - -@contextmanager -def prepare_summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml"): - """Does a couple of things every summon needs as a prerequisite: - clones the repo, parses the summon file and pulls the deps. - - Calling code is expected to complete the summon logic following - instructions stated in "summon" dict of the object spec. - - Returns a SummonDesc instance, which contains references to a Repo object, - named object specification and resolved paths to deps. - """ - with _make_repo(repo, rev=rev) as _repo: - _require_dvc(_repo) - try: - path = os.path.join(_repo.root_dir, summon_file) - obj = _get_object_spec(name, path) - yield SummonDesc(_repo, obj) - except SummonError as exc: - raise SummonError( - str(exc) + " at '{}' in '{}'".format(summon_file, repo) - ) from exc.__cause__ - - -class SummonDesc: - def __init__(self, repo, obj): - self.repo = repo - self.obj = obj - self._pull_deps() - - @property - def deps(self): - return [os.path.join(self.repo.root_dir, d) for d in self._deps] - - @property - def _deps(self): - return self.obj["summon"].get("deps", []) - - def _pull_deps(self): - if not self._deps: - return - - outs = [self.repo.find_out_by_relpath(d) for d in self._deps] - - with self.repo.state: - for out in outs: - self.repo.cloud.pull(out.get_used_cache()) - out.checkout() - - -def _get_object_spec(name, path): - """ - Given a summonable object's name, search for it on the given file - and return its description. - """ - try: - with builtin_open(path, "r") as fobj: - content = SUMMON_FILE_SCHEMA(ruamel.yaml.safe_load(fobj.read())) - objects = [x for x in content["objects"] if x["name"] == name] - - if not objects: - raise SummonError("No object with name '{}'".format(name)) - elif len(objects) >= 2: - raise SummonError( - "More than one object with name '{}'".format(name) - ) - - return objects[0] - - except FileNotFoundError as exc: - raise SummonError("Summon file not found") from exc - except ruamel.yaml.YAMLError as exc: - raise SummonError("Failed to parse summon file") from exc - except Invalid as exc: - raise SummonError(str(exc)) from exc - - -@wrap_with(threading.Lock()) -def _invoke_method(call, args, path): - # XXX: Some issues with this approach: - # * Import will pollute sys.modules - # * sys.path manipulation is "theoretically" not needed, - # but tests are failing for an unknown reason. - cwd = os.getcwd() - - try: - os.chdir(path) - sys.path.insert(0, path) - method = _import_string(call) - return method(**args) - finally: - os.chdir(cwd) - sys.path.pop(0) - - -def _import_string(import_name): - """Imports an object based on a string. - Useful to delay import to not load everything on startup. - Use dotted notaion in `import_name`, e.g. 'dvc.remote.gs.RemoteGS'. - - :return: imported object - """ - if "." in import_name: - module, obj = import_name.rsplit(".", 1) - else: - return importlib.import_module(import_name) - return getattr(importlib.import_module(module), obj) - - def _require_dvc(repo): if not isinstance(repo, Repo): raise UrlNotDvcRepoError(repo.url) diff --git a/dvc/cli.py b/dvc/cli.py index 7deb82a70..0d612f85d 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -3,37 +3,39 @@ import argparse import logging import sys -import dvc.command.add as add -import dvc.command.cache as cache -import dvc.command.checkout as checkout -import dvc.command.commit as commit -import dvc.command.config as config -import dvc.command.daemon as daemon -import dvc.command.data_sync as data_sync -import dvc.command.destroy as destroy -import dvc.command.diff as diff -import dvc.command.gc as gc -import dvc.command.get as get -import dvc.command.get_url as get_url -import dvc.command.imp as imp -import dvc.command.imp_url as imp_url -import dvc.command.init as init -import dvc.command.install as install -import dvc.command.lock as lock -import dvc.command.metrics as metrics -import dvc.command.move as move -import dvc.command.pipeline as pipeline -import dvc.command.remote as remote -import dvc.command.remove as remove -import dvc.command.repro as repro -import dvc.command.root as root -import dvc.command.run as run -import dvc.command.tag as tag -import dvc.command.unprotect as unprotect -import dvc.command.update as update -import dvc.command.version as version -from dvc.command.base import fix_subparsers -from dvc.exceptions import DvcParserError +from .command import ( + add, + cache, + checkout, + commit, + config, + daemon, + data_sync, + destroy, + diff, + gc, + get, + get_url, + imp, + imp_url, + init, + install, + lock, + metrics, + move, + pipeline, + remote, + remove, + repro, + root, + run, + tag, + unprotect, + update, + version, +) +from .command.base import fix_subparsers +from .exceptions import DvcParserError logger = logging.getLogger(__name__) diff --git a/dvc/command/diff.py b/dvc/command/diff.py index e606f692b..df1f61f27 100644 --- a/dvc/command/diff.py +++ b/dvc/command/diff.py @@ -1,12 +1,11 @@ import argparse +import json import logging +import os -import humanize -import inflect -from funcy import compact +import colorama -from dvc.command.base import append_doc_link -from dvc.command.base import CmdBase +from dvc.command.base import CmdBase, append_doc_link from dvc.exceptions import DvcException @@ -15,129 +14,114 @@ logger = logging.getLogger(__name__) class CmdDiff(CmdBase): @staticmethod - def _print_size(size): - if size < 0: - change = "decreased by {}" - elif size > 0: - change = "increased by {}" - else: - change = "not changed" - natur_size = humanize.naturalsize(abs(size)) - return change.format(natur_size) + def _format(diff): + """ + Given a diff structure, generate a string of paths separated + by new lines and grouped together by their state. - @staticmethod - def _get_md5_string(sign, file_name, checksum): - sample_msg = "" - if file_name: - sample_msg = "{}{} with md5 {}\n" - sample_msg = sample_msg.format(sign, file_name, checksum) - return sample_msg + A group's header is colored and its entries are sorted to enhance + readability, for example: - @classmethod - def _get_dir_changes(cls, dct): - import dvc.repo.diff as diff + Added: + another_file.txt + backup.tar + dir/ + dir/1 - engine = inflect.engine() - changes_msg = ( - "{} {} untouched, {} {} modified, {} {} added, " - "{} {} deleted, size was {}" + An example of a diff formatted when entries contain checksums: + + Added: + d3b07384 foo + + Modified: + c157a790..f98bf6f1 bar + + If a group has no entries, it won't be included in the result. + + At the bottom, include a summary with the number of files per state. + """ + + def _digest(checksum): + if type(checksum) is str: + return checksum[0:8] + return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8]) + + colors = { + "added": colorama.Fore.GREEN, + "modified": colorama.Fore.YELLOW, + "deleted": colorama.Fore.RED, + } + + summary = {} + groups = [] + + for state in ["added", "deleted", "modified"]: + summary[state] = 0 + entries = diff[state] + + if not entries: + continue + + content = [] + + for entry in entries: + path = entry["path"] + checksum = entry.get("checksum") + summary[state] += 1 if not path.endswith(os.sep) else 0 + content.append( + "{space}{checksum}{separator}{path}".format( + space=" ", + checksum=_digest(checksum) if checksum else "", + separator=" " if checksum else "", + path=entry["path"], + ) + ) + + groups.append( + "{color}{header}{nc}:\n{content}".format( + color=colors[state], + header=state.capitalize(), + nc=colorama.Fore.RESET, + content="\n".join(content), + ) + ) + + groups.append( + "summary: added ({added}), deleted ({deleted})," + " modified ({modified})".format_map(summary) ) - changes_msg = changes_msg.format( - dct[diff.DIFF_IDENT], - engine.plural("file", dct[diff.DIFF_IDENT]), - dct[diff.DIFF_CHANGE], - engine.plural("file", dct[diff.DIFF_CHANGE]), - dct[diff.DIFF_NEW], - engine.plural("file", dct[diff.DIFF_NEW]), - dct[diff.DIFF_DEL], - engine.plural("file", dct[diff.DIFF_DEL]), - cls._print_size(dct[diff.DIFF_SIZE]), - ) - return changes_msg - @classmethod - def _get_file_changes(cls, dct): - import dvc.repo.diff as diff - - if ( - dct.get(diff.DIFF_OLD_FILE) - and dct.get(diff.DIFF_NEW_FILE) - and dct[diff.DIFF_SIZE] == 0 - ): - msg = "file size was not changed" - elif dct.get(diff.DIFF_NEW_FILE): - msg = "added file with size {}".format( - humanize.naturalsize(dct[diff.DIFF_SIZE]) - ) - elif dct.get(diff.DIFF_OLD_FILE): - msg = "deleted file with size {}".format( - humanize.naturalsize(abs(dct[diff.DIFF_SIZE])) - ) - else: - msg = "file was modified, file size {}".format( - cls._print_size(dct[diff.DIFF_SIZE]) - ) - return msg - - @classmethod - def _get_royal_changes(cls, dct): - import dvc.repo.diff as diff - - if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN: - if dct.get("is_dir"): - return cls._get_dir_changes(dct) - else: - return cls._get_file_changes(dct) - return "size is ?" - - @classmethod - def _show(cls, diff_dct): - import dvc.repo.diff as diff - - msg = "dvc diff from {} to {}".format( - diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF] - ) - if diff_dct.get(diff.DIFF_EQUAL): - logger.info(msg) - return - for dct in diff_dct[diff.DIFF_LIST]: - msg += "\n\ndiff for '{}'\n".format(dct[diff.DIFF_TARGET]) - msg += cls._get_md5_string( - "-", - dct.get(diff.DIFF_OLD_FILE), - dct.get(diff.DIFF_OLD_CHECKSUM), - ) - msg += cls._get_md5_string( - "+", - dct.get(diff.DIFF_NEW_FILE), - dct.get(diff.DIFF_NEW_CHECKSUM), - ) - msg += "\n" - msg += cls._get_royal_changes(dct) - logger.info(msg) - return msg + return "\n\n".join(groups) def run(self): try: - msg = self.repo.diff( - self.args.a_ref, target=self.args.target, b_ref=self.args.b_ref - ) - self._show(msg) + diff = self.repo.diff(self.args.a_ref, self.args.b_ref) + + if not any(diff.values()): + return 0 + + if not self.args.checksums: + for _, entries in diff.items(): + for entry in entries: + del entry["checksum"] + + if self.args.show_json: + res = json.dumps(diff) + else: + res = self._format(diff) + + logger.info(res) + except DvcException: - msg = "failed to get 'diff {}'" - args = " ".join( - compact([self.args.target, self.args.a_ref, self.args.b_ref]) - ) - msg = msg.format(args) - logger.exception(msg) + logger.exception("failed to get diff") return 1 return 0 def add_parser(subparsers, parent_parser): DIFF_DESCRIPTION = ( - "Show changes between revisions of the DVC repository.\n" - "Summary of how many files were deleted/changed, et al." + "Compare two versions of the DVC repository.\n" + "Shows the list of paths added, modified, or deleted" ) diff_parser = subparsers.add_parser( "diff", @@ -147,23 +131,32 @@ def add_parser(subparsers, parent_parser): formatter_class=argparse.RawDescriptionHelpFormatter, ) diff_parser.add_argument( - "-t", - "--target", + "a_ref", help=( - "Source path to a data file or directory. Default None. " - "If not specified, compares all files and directories " - "that are under DVC control in the current working space." + "Git reference to the older version to compare" + " (defaults to `HEAD`)" ), - ) - diff_parser.add_argument( - "a_ref", help="Git reference from which the diff begins" + nargs="?", + default="HEAD", ) diff_parser.add_argument( "b_ref", help=( - "Git reference until which the diff ends. " - "If omitted, `HEAD` (latest commit) is used." + "Git reference to the newer version to compare" + " (defaults to the current workspace including changes)" ), nargs="?", ) + diff_parser.add_argument( + "--show-json", + help="Format the output into a JSON", + action="store_true", + default=False, + ) + diff_parser.add_argument( + "--checksums", + help="Display checksums for each entry", + action="store_true", + default=False, + ) diff_parser.set_defaults(func=CmdDiff) diff --git a/dvc/output/base.py b/dvc/output/base.py index da0e7de90..7770c30cc 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -405,7 +405,7 @@ class OutputBase(object): if self.stage.is_repo_import: cache = NamedCache() - dep, = self.stage.deps + (dep,) = self.stage.deps cache.external[dep.repo_pair].add(dep.def_path) return cache diff --git a/dvc/progress.py b/dvc/progress.py index fefbd98f6..d03b037c0 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -4,7 +4,6 @@ import logging import sys from threading import RLock -from funcy import merge from tqdm import tqdm from dvc.utils import env2bool @@ -31,7 +30,10 @@ class Tqdm(tqdm): ) BAR_FMT_NOTOTAL = ( "{desc:{ncols_desc}.{ncols_desc}}{n_fmt}" - " [{elapsed}11}{postfix}]" + " [{elapsed}, {rate_fmt:>11}{postfix}]" + ) + BYTES_DEFAULTS = dict( + unit="B", unit_scale=True, unit_divisor=1024, miniters=1 ) def __init__( @@ -44,6 +46,7 @@ class Tqdm(tqdm): bar_format=None, bytes=False, # pylint: disable=W0622 file=None, + total=None, **kwargs ): """ @@ -58,12 +61,10 @@ class Tqdm(tqdm): kwargs : anything accepted by `tqdm.tqdm()` """ kwargs = kwargs.copy() - kwargs.setdefault("unit_scale", True) if bytes: - bytes_defaults = dict( - unit="B", unit_scale=True, unit_divisor=1024, miniters=1 - ) - kwargs = merge(bytes_defaults, kwargs) + kwargs = {**self.BYTES_DEFAULTS, **kwargs} + else: + kwargs.setdefault("unit_scale", total > 999 if total else True) if file is None: file = sys.stderr self.desc_persist = desc @@ -84,6 +85,7 @@ class Tqdm(tqdm): desc=desc, bar_format="!", lock_args=(False,), + total=total, **kwargs ) if bar_format is None: @@ -114,6 +116,10 @@ class Tqdm(tqdm): def close(self): if self.desc_persist is not None: self.set_description_str(self.desc_persist, refresh=False) + # unknown/zero ETA + self.bar_format = self.bar_format.replace("<{remaining}", "") + # remove completed bar + self.bar_format = self.bar_format.replace("|{bar:10}|", " ") super().close() @property diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 6096af65f..33cb8a300 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -493,7 +493,9 @@ class RemoteBASE(object): cache_info = self.checksum_to_path_info(checksum) dir_info = self.get_dir_cache(checksum) - for entry in dir_info: + for entry in Tqdm( + dir_info, desc="Saving " + path_info.name, unit="file" + ): entry_info = path_info / entry[self.PARAM_RELPATH] entry_checksum = entry[self.PARAM_CHECKSUM] self._save_file(entry_info, entry_checksum, save_link=False) diff --git a/dvc/remote/gdrive.py b/dvc/remote/gdrive.py index 9b141402f..9cbca729e 100644 --- a/dvc/remote/gdrive.py +++ b/dvc/remote/gdrive.py @@ -104,13 +104,15 @@ class RemoteGDrive(RemoteBASE): ), ) ) + self.remote_drive_id = None @gdrive_retry def gdrive_upload_file( self, args, no_progress_bar=True, from_file="", progress_name="" ): + parent = {"id": args["parent_id"]} item = self.drive.CreateFile( - {"title": args["title"], "parents": [{"id": args["parent_id"]}]} + {"title": args["title"], "parents": [parent]} ) with open(from_file, "rb") as fobj: @@ -133,7 +135,9 @@ class RemoteGDrive(RemoteBASE): def gdrive_download_file( self, file_id, to_file, progress_name, no_progress_bar ): - gdrive_file = self.drive.CreateFile({"id": file_id}) + param = {"id": file_id} + # it does not create a file on the remote + gdrive_file = self.drive.CreateFile(param) bar_format = ( "Donwloading {desc:{ncols_desc}.{ncols_desc}}... " + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024) @@ -144,7 +148,12 @@ class RemoteGDrive(RemoteBASE): gdrive_file.GetContentFile(to_file) def gdrive_list_item(self, query): - file_list = self.drive.ListFile({"q": query, "maxResults": 1000}) + param = {"q": query, "maxResults": 1000, "corpora": self.corpora} + + if self.remote_drive_id: + param["driveId"] = self.remote_drive_id + + file_list = self.drive.ListFile(param) # Isolate and decorate fetching of remote drive items in pages get_list = gdrive_retry(lambda: next(file_list, None)) @@ -175,6 +184,12 @@ class RemoteGDrive(RemoteBASE): self.drive return self._cached_ids + @property + def corpora(self): + if not hasattr(self, "_corpora"): + self.drive + return self._corpora + @property @wrap_with(threading.RLock()) def drive(self): @@ -240,21 +255,22 @@ class RemoteGDrive(RemoteBASE): self._gdrive = GoogleDrive(gauth) + if self.bucket != "root" and self.bucket != "appDataFolder": + self.remote_drive_id = self.get_remote_drive_id(self.bucket) + self._corpora = "drive" if self.remote_drive_id else "default" self.remote_root_id = self.get_remote_id( self.path_info, create=True ) + self._cached_dirs, self._cached_ids = self.cache_root_dirs() return self._gdrive @gdrive_retry def create_remote_dir(self, parent_id, title): + parent = {"id": parent_id} item = self.drive.CreateFile( - { - "title": title, - "parents": [{"id": parent_id}], - "mimeType": FOLDER_MIME_TYPE, - } + {"title": title, "parents": [parent], "mimeType": FOLDER_MIME_TYPE} ) item.Upload() return item @@ -272,12 +288,28 @@ class RemoteGDrive(RemoteBASE): query += " and trashed=false and title='{}'".format(name) + param = { + "q": query, + # Remote might contain items with duplicated titles + "maxResults": 1, + "corpora": self.corpora, + } + + if self.remote_drive_id: + param["driveId"] = self.remote_drive_id + # Limit found remote items count to 1 in response - item_list = self.drive.ListFile( - {"q": query, "maxResults": 1} - ).GetList() + item_list = self.drive.ListFile(param).GetList() return next(iter(item_list), None) + @gdrive_retry + def get_remote_drive_id(self, remote_id): + param = {"id": remote_id} + # it does not create a file on the remote + item = self.drive.CreateFile(param) + item.FetchMetadata("driveId") + return item.get("driveId", None) + def resolve_remote_item_from_path(self, path_parts, create): parents_ids = [self.bucket] current_path = "" @@ -301,6 +333,10 @@ class RemoteGDrive(RemoteBASE): return [] def get_remote_id(self, path_info, create=False): + if not path_info.path and path_info.bucket: + # Case sensitive base path + return self.bucket + remote_ids = self.get_remote_id_from_cache(path_info.path) if remote_ids: diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 9d0a506a8..581f8bfef 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -125,7 +125,7 @@ class Repo(object): self._reset() def __repr__(self): - return "Repo: '{root_dir}'".format(root_dir=self.root_dir) + return "{}: '{}'".format(self.__class__.__name__, self.root_dir) @classmethod def find_root(cls, root=None): @@ -205,7 +205,7 @@ class Repo(object): return [(stage, None) for stage in self.stages] try: - out, = self.find_outs_by_path(target, strict=False) + (out,) = self.find_outs_by_path(target, strict=False) filter_info = PathInfo(os.path.abspath(target)) return [(out.stage, filter_info)] except OutputNotFoundError: @@ -419,7 +419,7 @@ class Repo(object): def find_out_by_relpath(self, relpath): path = os.path.join(self.root_dir, relpath) - out, = self.find_outs_by_path(path) + (out,) = self.find_outs_by_path(path) return out def is_dvc_internal(self, path): diff --git a/dvc/repo/add.py b/dvc/repo/add.py index 51fc417f8..e469119b9 100644 --- a/dvc/repo/add.py +++ b/dvc/repo/add.py @@ -4,11 +4,12 @@ import os import colorama from . import locked -from dvc.exceptions import RecursiveAddingWhileUsingFilename -from dvc.progress import Tqdm -from dvc.repo.scm_context import scm_context -from dvc.stage import Stage -from dvc.utils import LARGE_DIR_SIZE +from ..exceptions import RecursiveAddingWhileUsingFilename +from ..output.base import OutputDoesNotExistError +from ..progress import Tqdm +from ..repo.scm_context import scm_context +from ..stage import Stage +from ..utils import LARGE_DIR_SIZE logger = logging.getLogger(__name__) @@ -23,7 +24,12 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None): targets = [targets] stages_list = [] - with Tqdm(total=len(targets), desc="Add", unit="file", leave=True) as pbar: + num_targets = len(targets) + with Tqdm(total=num_targets, desc="Add", unit="file", leave=True) as pbar: + if num_targets == 1: + # clear unneeded top-level progress bar for single target + pbar.bar_format = "Adding..." + pbar.refresh() for target in targets: sub_targets = _find_all_targets(repo, target, recursive) pbar.total += len(sub_targets) - 1 @@ -45,17 +51,29 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None): repo.check_modified_graph(stages) - for stage in stages: - stage.save() + with Tqdm( + total=len(stages), + desc="Processing", + unit="file", + disable=True if len(stages) == 1 else None, + ) as pbar_stages: + for stage in stages: + try: + stage.save() + except OutputDoesNotExistError: + pbar.n -= 1 + raise - if not no_commit: - stage.commit() + if not no_commit: + stage.commit() - stage.dump() + stage.dump() + pbar_stages.update() stages_list += stages - # remove filled bar bit of progress, leaving stats - pbar.bar_format = pbar.BAR_FMT_DEFAULT.replace("|{bar:10}|", " ") + + if num_targets == 1: # restore bar format for stats + pbar.bar_format = pbar.BAR_FMT_DEFAULT return stages_list @@ -64,7 +82,12 @@ def _find_all_targets(repo, target, recursive): if os.path.isdir(target) and recursive: return [ fname - for fname in repo.tree.walk_files(target) + for fname in Tqdm( + repo.tree.walk_files(target), + desc="Searching " + target, + bar_format=Tqdm.BAR_FMT_NOTOTAL, + unit="file", + ) if not repo.is_dvc_internal(fname) if not Stage.is_stage_file(fname) if not repo.scm.belongs_to_scm(fname) @@ -76,7 +99,12 @@ def _find_all_targets(repo, target, recursive): def _create_stages(repo, targets, fname, pbar=None): stages = [] - for out in targets: + for out in Tqdm( + targets, + desc="Creating DVC-files", + disable=True if len(targets) < LARGE_DIR_SIZE else None, + unit="file", + ): stage = Stage.create(repo, outs=[out], add=True, fname=fname) if not stage: diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index 165b286b9..b8970c7b3 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -1,246 +1,76 @@ import os -from errno import ENOENT -from dvc import logger -from dvc.scm.base import FileNotInCommitError -from dvc.scm.git import DIFF_A_REF, DIFF_B_REF, DIFF_A_TREE, DIFF_B_TREE -from dvc.scm.git import DIFF_EQUAL -from . import locked - - -DIFF_TARGET = "target" -DIFF_IS_DIR = "is_dir" -DIFF_OLD_FILE = "old_file" -DIFF_OLD_CHECKSUM = "old_checksum" -DIFF_NEW_FILE = "new_file" -DIFF_NEW_CHECKSUM = "new_checksum" -DIFF_SIZE = "size_diff" -DIFF_DEL = "del" -DIFF_IDENT = "ident" -DIFF_CHANGE = "changes" -DIFF_NEW = "new" -DIFF_MOVE = "moves" -DIFF_LIST = "diffs" -DIFF_SIZE_UNKNOWN = "?" -DIFF_A_OUTPUT = "a_output" -DIFF_B_OUTPUT = "b_output" -DIFF_DELETED = "deleted_file" -DIFF_IS_NEW = "created_file" - - -def _file_not_exists(error, result): - if error.errno == ENOENT: - result.update({DIFF_SIZE: DIFF_SIZE_UNKNOWN}) - else: - raise error - - -def _extract_dir(self, dir_not_exists, output): - """Extract the content of dvc tree file - Args: - self(object) - Repo class instance - dir_not_exists(bool) - flag for directory existence - output(object) - OutputLOCAL class instance - Returns: - dict - dictionary with keys - paths to file in .dvc/cache - values -checksums for that files - """ - if not dir_not_exists: - lst = output.dir_cache - return {i["relpath"]: i["md5"] for i in lst} - return {} - - -def _get_dir_info(dir_not_exists, a_output): - if not dir_not_exists: - return str(a_output), a_output.checksum - return "", "" - - -def _ident_files(a_entries, b_entries): - keys = [ - key for key in a_entries.keys() if b_entries.get(key) == a_entries[key] - ] - return len(keys) - - -def _modified_files(self, a_entries, b_entries): - keys = [key for key in a_entries.keys() if key in b_entries] - diff_size = 0 - modified_count = 0 - for key in keys: - if a_entries[key] != b_entries[key]: - modified_count += 1 - diff_size += os.path.getsize( - self.cache.local.get(b_entries[key]) - ) - os.path.getsize(self.cache.local.get(a_entries[key])) - return modified_count, diff_size - - -def _deleted_files(self, a_entries, b_entries): - diff_size = 0 - deleted_count = 0 - for key, value in a_entries.items(): - if key not in b_entries: - deleted_count += 1 - diff_size -= os.path.getsize(self.cache.local.get(a_entries[key])) - return deleted_count, diff_size - - -def _new_files(self, a_entries, b_entries): - diff_size = 0 - new_count = 0 - for key, value in b_entries.items(): - if key not in a_entries: - new_count += 1 - diff_size += os.path.getsize(self.cache.local.get(b_entries[key])) - return new_count, diff_size - - -def _get_tree_changes(self, a_entries, b_entries): - result = { - DIFF_DEL: 0, - DIFF_IDENT: 0, - DIFF_CHANGE: 0, - DIFF_NEW: 0, - DIFF_MOVE: 0, - } - result[DIFF_IDENT] = _ident_files(a_entries, b_entries) - result[DIFF_CHANGE], diff_size = _modified_files( - self, a_entries, b_entries - ) - result[DIFF_SIZE] = diff_size - result[DIFF_DEL], diff_size = _deleted_files(self, a_entries, b_entries) - result[DIFF_SIZE] += diff_size - result[DIFF_NEW], diff_size = _new_files(self, a_entries, b_entries) - result[DIFF_SIZE] += diff_size - return result - - -def _check_local_cache(a_out, is_checked): - if a_out is not None and a_out.scheme != "local": - is_checked.append(str(a_out)) - return True - return False - - -def _is_dir(path, a_outs, b_outs): - if a_outs.get(path): - return a_outs[path].is_dir_checksum - else: - return b_outs[path].is_dir_checksum - - -def _get_diff_outs(self, diff_dct): - self.tree = diff_dct[DIFF_A_TREE] - a_outs = {str(out): out for st in self.stages for out in st.outs} - self.tree = diff_dct[DIFF_B_TREE] - b_outs = {str(out): out for st in self.stages for out in st.outs} - outs_paths = set(a_outs.keys()) - outs_paths.update(b_outs.keys()) - results = {} - non_local_cache = [] - for path in outs_paths: - check1 = _check_local_cache(a_outs.get(path), non_local_cache) - check2 = _check_local_cache(b_outs.get(path), non_local_cache) - # skip files/directories with non-local cache for now - if check1 or check2: - continue - results[path] = {} - results[path][DIFF_A_OUTPUT] = a_outs.get(path) - results[path][DIFF_B_OUTPUT] = b_outs.get(path) - results[path][DIFF_IS_NEW] = path not in a_outs - results[path][DIFF_DELETED] = path not in b_outs - results[path][DIFF_IS_DIR] = _is_dir(path, a_outs, b_outs) - if non_local_cache: - logger.warning( - "Diff is not supported for non-local outputs. Ignoring: {}".format( - non_local_cache - ) - ) - - return results - - -def _diff_dir(self, target, diff_dct): - result = {DIFF_TARGET: target} - result[DIFF_IS_DIR] = True - a_entries, b_entries = {}, {} - try: - a_entries = _extract_dir( - self, diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT] - ) - b_entries = _extract_dir( - self, diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT] - ) - result[DIFF_OLD_FILE], result[DIFF_OLD_CHECKSUM] = _get_dir_info( - diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT] - ) - result[DIFF_NEW_FILE], result[DIFF_NEW_CHECKSUM] = _get_dir_info( - diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT] - ) - result.update(_get_tree_changes(self, a_entries, b_entries)) - except IOError as e: - _file_not_exists(e, result) - return result - - -def _diff_file(self, target, diff_dct): - result = {DIFF_TARGET: target} - size = 0 - try: - if not diff_dct[DIFF_IS_NEW]: - result[DIFF_OLD_FILE] = target - result[DIFF_OLD_CHECKSUM] = diff_dct[DIFF_A_OUTPUT].checksum - size -= os.path.getsize( - self.cache.local.get(diff_dct[DIFF_A_OUTPUT].checksum) - ) - if not diff_dct[DIFF_DELETED]: - result[DIFF_NEW_FILE] = target - result[DIFF_NEW_CHECKSUM] = diff_dct[DIFF_B_OUTPUT].checksum - size += os.path.getsize( - self.cache.local.get(diff_dct[DIFF_B_OUTPUT].checksum) - ) - result[DIFF_SIZE] = size - except IOError as e: - _file_not_exists(e, result) - return result - - -def _diff_royal(self, target, diff_dct): - if diff_dct[DIFF_IS_DIR]: - return _diff_dir(self, target, diff_dct) - return _diff_file(self, target, diff_dct) +from dvc.exceptions import DvcException +from dvc.repo import locked +from dvc.scm.git import Git @locked -def diff(self, a_ref, target=None, b_ref=None): - """Generates diff message string output - - Args: - target(str) - file/directory to check diff of - a_ref(str) - first tag - (optional) b_ref(str) - second git tag - - Returns: - string: string of output message with diff info +def diff(self, a_ref="HEAD", b_ref=None): """ - result = {} - diff_dct = self.scm.get_diff_trees(a_ref, b_ref=b_ref) - result[DIFF_A_REF] = diff_dct[DIFF_A_REF] - result[DIFF_B_REF] = diff_dct[DIFF_B_REF] - if diff_dct[DIFF_EQUAL]: - result[DIFF_EQUAL] = True + By default, it compares the working tree with the last commit's tree. + + This implementation differs from `git diff` since DVC doesn't have + the concept of `index`, but it keeps the same interface, thus, + `dvc diff` would be the same as `dvc diff HEAD`. + """ + if type(self.scm) is not Git: + raise DvcException("only supported for Git repositories") + + def _paths_checksums(): + """ + A dictionary of checksums addressed by relpaths collected from + the current tree outputs. + + Unpack directories to include their entries + + To help distinguish between a directory and a file output, + the former one will come with a trailing slash in the path: + + directory: "data/" + file: "data" + """ + result = {} + + for stage in self.stages: + for output in stage.outs: + if not output.is_dir_checksum: + result.update({str(output): output.checksum}) + continue + + result.update({os.path.join(str(output), ""): output.checksum}) + + for entry in output.dir_cache: + path = str(output.path_info / entry["relpath"]) + result.update({path: entry["md5"]}) + return result - result[DIFF_LIST] = [] - diff_outs = _get_diff_outs(self, diff_dct) - if target is None: - result[DIFF_LIST] = [ - _diff_royal(self, path, diff_outs[path]) for path in diff_outs - ] - elif target in diff_outs: - result[DIFF_LIST] = [_diff_royal(self, target, diff_outs[target])] - else: - msg = "Have not found file/directory '{}' in the commits" - raise FileNotInCommitError(msg.format(target)) - return result + + working_tree = self.tree + a_tree = self.scm.get_tree(a_ref) + b_tree = self.scm.get_tree(b_ref) if b_ref else working_tree + + try: + self.tree = a_tree + old = _paths_checksums() + + self.tree = b_tree + new = _paths_checksums() + finally: + self.tree = working_tree + + # Compare paths between the old and new tree. + # set() efficiently converts dict keys to a set + added = sorted(set(new) - set(old)) + deleted = sorted(set(old) - set(new)) + modified = sorted(set(old) & set(new)) + + return { + "added": [{"path": path, "checksum": new[path]} for path in added], + "deleted": [{"path": path, "checksum": old[path]} for path in deleted], + "modified": [ + {"path": path, "checksum": {"old": old[path], "new": new[path]}} + for path in modified + if old[path] != new[path] + ], + } diff --git a/dvc/repo/get_url.py b/dvc/repo/get_url.py index fbc173b64..5718b20ec 100644 --- a/dvc/repo/get_url.py +++ b/dvc/repo/get_url.py @@ -14,6 +14,6 @@ def get_url(url, out=None): out = os.path.abspath(out) - dep, = dependency.loads_from(None, [url]) - out, = output.loads_from(None, [out], use_cache=False) + (dep,) = dependency.loads_from(None, [url]) + (out,) = output.loads_from(None, [out], use_cache=False) dep.download(out) diff --git a/dvc/scm/base.py b/dvc/scm/base.py index 412731ca8..51a52217b 100644 --- a/dvc/scm/base.py +++ b/dvc/scm/base.py @@ -15,22 +15,13 @@ class FileNotInRepoError(SCMError): """ -class FileNotInCommitError(SCMError): - """Thrown when trying to find a file/directory that is not - in the specified commit in the repository. - """ - - class CloneError(SCMError): def __init__(self, url, path): super().__init__("Failed to clone repo '{}' to '{}'".format(url, path)) class RevError(SCMError): - def __init__(self, url, rev): - super().__init__( - "Failed to access revision '{}' for repo '{}'".format(rev, url) - ) + pass class Base(object): diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index ad1fcb2da..79fbdb360 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -22,13 +22,6 @@ from dvc.utils.fs import path_isin logger = logging.getLogger(__name__) -DIFF_A_TREE = "a_tree" -DIFF_B_TREE = "b_tree" -DIFF_A_REF = "a_ref" -DIFF_B_REF = "b_ref" -DIFF_EQUAL = "equal" - - class Git(Base): """Class for managing Git.""" @@ -94,7 +87,11 @@ class Git(Base): try: repo.checkout(rev) except git.exc.GitCommandError as exc: - raise RevError(url, rev) from exc + raise RevError( + "failed to access revision '{}' for repo '{}'".format( + rev, url + ) + ) from exc return repo @@ -215,6 +212,16 @@ class Git(Base): else: self.repo.git.checkout(branch) + def pull(self): + info, = self.repo.remote().pull() + if info.flags & info.ERROR: + raise SCMError("pull failed: {}".format(info.note)) + + def push(self): + info, = self.repo.remote().push() + if info.flags & info.ERROR: + raise SCMError("push failed: {}".format(info.summary)) + def branch(self, branch): self.repo.git.branch(branch) @@ -275,9 +282,10 @@ class Git(Base): [ # checking out some reference and not specific file. '[ "$3" = "1" ]', - # check that we are on some branch/tag and not in detached HEAD - # state, so we don't accidentally break a rebase. - '[ "$(git rev-parse --abbrev-ref HEAD)" != "HEAD" ]', + # make sure we are not in the middle of a rebase/merge, so we + # don't accidentally break it with an unsuccessful checkout. + # Note that git hooks are always running in repo root. + "[ ! -d .git/rebase-merge ]", ], "checkout", ) @@ -315,66 +323,18 @@ class Git(Base): return basename == self.ignore_file or Git.GIT_DIR in path_parts def get_tree(self, rev): - return GitTree(self.repo, rev) - - def _get_diff_trees(self, a_ref, b_ref): - """Private method for getting the trees and commit hashes of 2 git - references. Requires `gitdb` module (from gitpython package). - - Args: - a_ref (str): git reference - b_ref (str): second git reference. If None, uses HEAD - - Returns: - tuple: tuple with elements: (trees, commits) - """ - from gitdb.exc import BadObject, BadName - - trees = {DIFF_A_TREE: None, DIFF_B_TREE: None} - commits = [] - if b_ref is None: - b_ref = self.repo.head.commit - try: - a_commit = self.repo.git.rev_parse(a_ref, short=True) - b_commit = self.repo.git.rev_parse(b_ref, short=True) - # See https://gitpython.readthedocs.io - # /en/2.1.11/reference.html#git.objects.base.Object.__str__ - commits.append(a_commit) - commits.append(b_commit) - trees[DIFF_A_TREE] = self.get_tree(commits[0]) - trees[DIFF_B_TREE] = self.get_tree(commits[1]) - except (BadName, BadObject) as exc: - raise SCMError("git problem") from exc - return trees, commits - - def get_diff_trees(self, a_ref, b_ref=None): - """Method for getting two repo trees between two git tag commits. - Returns the dvc hash names of changed file/directory - - Args: - a_ref (str): git reference - b_ref (str): optional second git reference, default None - - Returns: - dict: dictionary with keys: {a_ref, b_ref, equal} - or {a_ref, b_ref, a_tree, b_tree} - """ - diff_dct = {DIFF_EQUAL: False} - trees, commits = self._get_diff_trees(a_ref, b_ref) - diff_dct[DIFF_A_REF] = commits[0] - diff_dct[DIFF_B_REF] = commits[1] - if commits[0] == commits[1]: - diff_dct[DIFF_EQUAL] = True - return diff_dct - diff_dct[DIFF_A_TREE] = trees[DIFF_A_TREE] - diff_dct[DIFF_B_TREE] = trees[DIFF_B_TREE] - return diff_dct + return GitTree(self.repo, self.resolve_rev(rev)) def get_rev(self): return self.repo.git.rev_parse("HEAD") def resolve_rev(self, rev): - return self.repo.git.rev_parse(rev) + from git.exc import GitCommandError + + try: + return self.repo.git.rev_parse(rev) + except GitCommandError: + raise RevError("unknown Git revision '{}'".format(rev)) def close(self): self.repo.close() diff --git a/dvc/version.py b/dvc/version.py index 0b2db6be5..7eb03c22c 100644 --- a/dvc/version.py +++ b/dvc/version.py @@ -7,7 +7,7 @@ import os import subprocess -_BASE_VERSION = "0.82.3" +_BASE_VERSION = "0.82.6" def _generate_version(base_version): diff --git a/scripts/completion/dvc.bash b/scripts/completion/dvc.bash index 2af965a67..8c2d3b061 100644 --- a/scripts/completion/dvc.bash +++ b/scripts/completion/dvc.bash @@ -22,7 +22,7 @@ _dvc_checkout='-d --with-deps -R --recursive -f --force --relink $(compgen -G *. _dvc_commit='-f --force -d --with-deps -R --recursive $(compgen -G *.dvc)' _dvc_config='-u --unset --local --system --global' _dvc_destroy='-f --force' -_dvc_diff='-t --target' +_dvc_diff='-t --show-json --checksums' _dvc_fetch='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive $(compgen -G *.dvc)' _dvc_get_url='' _dvc_get='-o --out --rev --show-url' diff --git a/scripts/completion/dvc.zsh b/scripts/completion/dvc.zsh index 6129fce0f..15c949b28 100644 --- a/scripts/completion/dvc.zsh +++ b/scripts/completion/dvc.zsh @@ -97,7 +97,9 @@ _dvc_destroy=( ) _dvc_diff=( - {-t,--target}"[Source path to a data file or directory.]:Target files:" + "--show-json[Format the output into a JSON]" + "--checksums[Display checksums for each entry]" + {1,2}":Git revision (e.g. branch, tag, SHA):" ) _dvc_fetch=( diff --git a/setup.py b/setup.py index 9f211084c..9e9b64f42 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ install_requires = [ # Extra dependencies for remote integrations gs = ["google-cloud-storage==1.19.0"] -gdrive = ["pydrive2>=1.4.1"] +gdrive = ["pydrive2>=1.4.2"] s3 = ["boto3>=1.9.201"] azure = ["azure-storage-blob==2.1.0"] oss = ["oss2==2.6.1"] diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 04f707826..02d77866e 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -1,12 +1,10 @@ import os import shutil -import copy -import ruamel.yaml import pytest from dvc import api -from dvc.api import SummonError, UrlNotDvcRepoError +from dvc.api import UrlNotDvcRepoError from dvc.compat import fspath from dvc.exceptions import FileMissingError from dvc.main import main @@ -141,68 +139,3 @@ def test_open_not_cached(dvc): os.remove(metric_file) with pytest.raises(FileMissingError): api.read(metric_file) - - -def test_summon(tmp_dir, dvc, erepo_dir): - objects = { - "objects": [ - { - "name": "sum", - "meta": {"description": "Add to "}, - "summon": { - "type": "python", - "call": "calculator.add_to_num", - "args": {"x": 1}, - "deps": ["number"], - }, - } - ] - } - - other_objects = copy.deepcopy(objects) - other_objects["objects"][0]["summon"]["args"]["x"] = 100 - - dup_objects = copy.deepcopy(objects) - dup_objects["objects"] *= 2 - - with erepo_dir.chdir(): - erepo_dir.dvc_gen("number", "100", commit="Add number.dvc") - erepo_dir.scm_gen("dvcsummon.yaml", ruamel.yaml.dump(objects)) - erepo_dir.scm_gen("other.yaml", ruamel.yaml.dump(other_objects)) - erepo_dir.scm_gen("dup.yaml", ruamel.yaml.dump(dup_objects)) - erepo_dir.scm_gen("invalid.yaml", ruamel.yaml.dump({"name": "sum"})) - erepo_dir.scm_gen("not_yaml.yaml", "a: - this is not a YAML file") - erepo_dir.scm_gen( - "calculator.py", - "def add_to_num(x): return x + int(open('number').read())", - ) - erepo_dir.scm.commit("Add files") - - repo_url = "file://{}".format(erepo_dir) - - assert api.summon("sum", repo=repo_url) == 101 - assert api.summon("sum", repo=repo_url, args={"x": 2}) == 102 - assert api.summon("sum", repo=repo_url, summon_file="other.yaml") == 200 - - try: - api.summon("sum", repo=repo_url, summon_file="missing.yaml") - except SummonError as exc: - assert "Summon file not found" in str(exc) - assert "missing.yaml" in str(exc) - assert repo_url in str(exc) - else: - pytest.fail("Did not raise on missing summon file") - - with pytest.raises(SummonError, match=r"No object with name 'missing'"): - api.summon("missing", repo=repo_url) - - with pytest.raises( - SummonError, match=r"More than one object with name 'sum'" - ): - api.summon("sum", repo=repo_url, summon_file="dup.yaml") - - with pytest.raises(SummonError, match=r"extra keys not allowed"): - api.summon("sum", repo=repo_url, summon_file="invalid.yaml") - - with pytest.raises(SummonError, match=r"Failed to parse summon file"): - api.summon("sum", repo=repo_url, summon_file="not_yaml.yaml") diff --git a/tests/func/test_diff.py b/tests/func/test_diff.py index 15d2cc982..7af8f3afc 100644 --- a/tests/func/test_diff.py +++ b/tests/func/test_diff.py @@ -1,267 +1,172 @@ +import hashlib import os +import pytest +import shutil -import dvc.repo.diff as diff -from dvc.main import main -from dvc.scm.base import FileNotInCommitError -from tests.basic_env import TestDvcGit +from dvc.compat import fspath +from dvc.exceptions import DvcException -class TestDiff(TestDvcGit): - def setUp(self): - super().setUp() - - self.new_file = "new_test_file" - self.create(self.new_file, self.new_file) - stage = self.dvc.add(self.new_file)[0] - self.a_ref = self.git.git.rev_parse(self.git.head.commit, short=True) - self.new_checksum = stage.outs[0].checksum - self.git.index.add([self.new_file + ".dvc"]) - self.git.index.commit("adds new_file") - self.test_dct = { - diff.DIFF_A_REF: self.a_ref, - diff.DIFF_B_REF: self.git.git.rev_parse( - self.git.head.commit, short=True - ), - diff.DIFF_LIST: [ - { - diff.DIFF_TARGET: self.new_file, - diff.DIFF_NEW_FILE: self.new_file, - diff.DIFF_NEW_CHECKSUM: self.new_checksum, - diff.DIFF_SIZE: 13, - } - ], - } - - def test(self): - out = self.dvc.scm.get_diff_trees(self.a_ref) - self.assertFalse(out[diff.DIFF_EQUAL]) - self.assertEqual(self.a_ref, out[diff.DIFF_A_REF]) - self.assertEqual( - self.git.git.rev_parse(self.git.head.commit, short=True), - out[diff.DIFF_B_REF], - ) +def digest(text): + return hashlib.md5(bytes(text, "utf-8")).hexdigest() -class TestDiffRepo(TestDiff): - def test(self): - result = self.dvc.diff(self.a_ref, target=self.new_file) - self.assertEqual(self.test_dct, result) +def test_no_scm(tmp_dir, dvc): + tmp_dir.dvc_gen("file", "text") + + with pytest.raises(DvcException, match=r"only supported for Git repos"): + dvc.diff() -class TestDiffCmdLine(TestDiff): - def test(self): - ret = main(["diff", "-t", self.new_file, self.a_ref]) - self.assertEqual(ret, 0) +def test_added(tmp_dir, scm, dvc): + tmp_dir.dvc_gen("file", "text") + + assert dvc.diff() == { + "added": [{"path": "file", "checksum": digest("text")}], + "deleted": [], + "modified": [], + } -class TestDiffDir(TestDvcGit): - def setUp(self): - super().setUp() +def test_no_cache_entry(tmp_dir, scm, dvc): + tmp_dir.dvc_gen("file", "first", commit="add a file") - stage = self.dvc.add(self.DATA_DIR)[0] - self.git.index.add([self.DATA_DIR + ".dvc"]) - self.git.index.commit("adds data_dir") - self.a_ref = self.git.git.rev_parse( - self.dvc.scm.repo.head.commit, short=True - ) - self.old_checksum = stage.outs[0].checksum - self.new_file = os.path.join(self.DATA_SUB_DIR, diff.DIFF_NEW_FILE) - self.create(self.new_file, self.new_file) - stage = self.dvc.add(self.DATA_DIR)[0] - self.git.index.add([self.DATA_DIR + ".dvc"]) - self.git.index.commit(message="adds data_dir with new_file") - self.new_checksum = stage.outs[0].checksum + tmp_dir.dvc_gen({"dir": {"1": "1", "2": "2"}}) + tmp_dir.dvc_gen("file", "second") - def test(self): - out = self.dvc.scm.get_diff_trees(self.a_ref) - self.assertFalse(out[diff.DIFF_EQUAL]) - self.assertEqual(self.a_ref, out[diff.DIFF_A_REF]) - self.assertEqual( - self.git.git.rev_parse(self.git.head.commit, short=True), - out[diff.DIFF_B_REF], - ) + shutil.rmtree(fspath(tmp_dir / ".dvc" / "cache")) + (tmp_dir / ".dvc" / "state").unlink() + + dir_checksum = "5fb6b29836c388e093ca0715c872fe2a.dir" + + assert dvc.diff() == { + "added": [ + {"path": os.path.join("dir", ""), "checksum": dir_checksum}, + {"path": os.path.join("dir", "1"), "checksum": digest("1")}, + {"path": os.path.join("dir", "2"), "checksum": digest("2")}, + ], + "deleted": [], + "modified": [ + { + "path": "file", + "checksum": {"old": digest("first"), "new": digest("second")}, + } + ], + } -class TestDiffDirRepo(TestDiffDir): - maxDiff = None +def test_deleted(tmp_dir, scm, dvc): + tmp_dir.dvc_gen("file", "text", commit="add file") + (tmp_dir / "file.dvc").unlink() - def test(self): - result = self.dvc.diff(self.a_ref, target=self.DATA_DIR) - test_dct = { - diff.DIFF_A_REF: self.git.git.rev_parse(self.a_ref, short=True), - diff.DIFF_B_REF: self.git.git.rev_parse( - self.git.head.commit, short=True - ), - diff.DIFF_LIST: [ - { - diff.DIFF_CHANGE: 0, - diff.DIFF_DEL: 0, - diff.DIFF_IDENT: 2, - diff.DIFF_MOVE: 0, - diff.DIFF_NEW: 1, - diff.DIFF_IS_DIR: True, - diff.DIFF_TARGET: self.DATA_DIR, - diff.DIFF_NEW_FILE: self.DATA_DIR, - diff.DIFF_NEW_CHECKSUM: self.new_checksum, - diff.DIFF_OLD_FILE: self.DATA_DIR, - diff.DIFF_OLD_CHECKSUM: self.old_checksum, - diff.DIFF_SIZE: 30, - } - ], - } - self.assertEqual(test_dct, result) + assert dvc.diff() == { + "added": [], + "deleted": [{"path": "file", "checksum": digest("text")}], + "modified": [], + } -class TestDiffDirRepoDeletedFile(TestDiffDir): - maxDiff = None +def test_modified(tmp_dir, scm, dvc): + tmp_dir.dvc_gen("file", "first", commit="first version") + tmp_dir.dvc_gen("file", "second") - def setUp(self): - super().setUp() - - self.b_ref = self.a_ref - tmp = self.new_checksum - self.new_checksum = self.old_checksum - self.a_ref = str(self.dvc.scm.repo.head.commit) - self.old_checksum = tmp - - def test(self): - result = self.dvc.diff( - self.a_ref, b_ref=self.b_ref, target=self.DATA_DIR - ) - test_dct = { - diff.DIFF_A_REF: self.git.git.rev_parse(self.a_ref, short=True), - diff.DIFF_B_REF: self.git.git.rev_parse(self.b_ref, short=True), - diff.DIFF_LIST: [ - { - diff.DIFF_CHANGE: 0, - diff.DIFF_DEL: 1, - diff.DIFF_IDENT: 2, - diff.DIFF_MOVE: 0, - diff.DIFF_NEW: 0, - diff.DIFF_IS_DIR: True, - diff.DIFF_TARGET: self.DATA_DIR, - diff.DIFF_NEW_FILE: self.DATA_DIR, - diff.DIFF_NEW_CHECKSUM: self.new_checksum, - diff.DIFF_OLD_FILE: self.DATA_DIR, - diff.DIFF_OLD_CHECKSUM: self.old_checksum, - diff.DIFF_SIZE: -30, - } - ], - } - self.assertEqual(test_dct, result) + assert dvc.diff() == { + "added": [], + "deleted": [], + "modified": [ + { + "path": "file", + "checksum": {"old": digest("first"), "new": digest("second")}, + } + ], + } -class TestDiffFileNotFound(TestDiffDir): - def setUp(self): - super().setUp() - self.unknown_file = "unknown_file_" + str(id(self)) +def test_refs(tmp_dir, scm, dvc): + tmp_dir.dvc_gen("file", "first", commit="first version") + tmp_dir.dvc_gen("file", "second", commit="second version") + tmp_dir.dvc_gen("file", "third", commit="third version") - def test(self): - with self.assertRaises(FileNotInCommitError): - self.dvc.diff(self.a_ref, target=self.unknown_file) + HEAD_2 = digest("first") + HEAD_1 = digest("second") + HEAD = digest("third") + + assert dvc.diff("HEAD~1") == { + "added": [], + "deleted": [], + "modified": [ + {"path": "file", "checksum": {"old": HEAD_1, "new": HEAD}} + ], + } + + assert dvc.diff("HEAD~2", "HEAD~1") == { + "added": [], + "deleted": [], + "modified": [ + {"path": "file", "checksum": {"old": HEAD_2, "new": HEAD_1}} + ], + } + + with pytest.raises(DvcException, match=r"unknown Git revision 'missing'"): + dvc.diff("missing") -class TestDiffModifiedFile(TestDiff): - maxDiff = None +def test_directories(tmp_dir, scm, dvc): + tmp_dir.dvc_gen({"dir": {"1": "1", "2": "2"}}, commit="add a directory") + tmp_dir.dvc_gen({"dir": {"3": "3"}}, commit="add a file") + tmp_dir.dvc_gen({"dir": {"2": "two"}}, commit="modify a file") - def setUp(self): - super().setUp() + (tmp_dir / "dir" / "2").unlink() + dvc.add("dir") + scm.add("dir.dvc") + scm.commit("delete a file") - self.old_checksum = self.new_checksum - self.new_file_content = "new_test_file_bigger_content_123456789" - self.diff_len = len(self.new_file) + len(self.new_file_content) - self.create(self.new_file, self.new_file_content) - stage = self.dvc.add(self.new_file)[0] - self.git.index.add([self.new_file + ".dvc"]) - self.git.index.commit("change new_file content to be bigger") - self.new_checksum = stage.outs[0].checksum - self.b_ref = self.git.git.rev_parse(self.git.head.commit, short=True) + # The ":/" format is a way to specify revisions by commit message: + # https://git-scm.com/docs/revisions + # + assert dvc.diff(":/init", ":/directory") == { + "added": [ + { + "path": os.path.join("dir", ""), + "checksum": "5fb6b29836c388e093ca0715c872fe2a.dir", + }, + {"path": os.path.join("dir", "1"), "checksum": digest("1")}, + {"path": os.path.join("dir", "2"), "checksum": digest("2")}, + ], + "deleted": [], + "modified": [], + } - def test(self): - result = self.dvc.diff( - self.a_ref, b_ref=self.b_ref, target=self.new_file - ) - test_dct = { - diff.DIFF_A_REF: self.git.git.rev_parse(self.a_ref, short=True), - diff.DIFF_B_REF: self.git.git.rev_parse(self.b_ref, short=True), - diff.DIFF_LIST: [ - { - diff.DIFF_NEW_CHECKSUM: self.new_checksum, - diff.DIFF_NEW_FILE: self.new_file, - diff.DIFF_TARGET: self.new_file, - diff.DIFF_SIZE: self.diff_len, - } - ], - } - self.assertEqual(test_dct, result) + assert dvc.diff(":/directory", ":/modify") == { + "added": [{"path": os.path.join("dir", "3"), "checksum": digest("3")}], + "deleted": [], + "modified": [ + { + "path": os.path.join("dir", ""), + "checksum": { + "old": "5fb6b29836c388e093ca0715c872fe2a.dir", + "new": "9b5faf37366b3370fd98e3e60ca439c1.dir", + }, + }, + { + "path": os.path.join("dir", "2"), + "checksum": {"old": digest("2"), "new": digest("two")}, + }, + ], + } - -class TestDiffDirWithFile(TestDiffDir): - maxDiff = None - - def setUp(self): - super().setUp() - - self.a_ref = self.git.git.rev_parse(self.git.head.commit, short=True) - self.old_checksum = self.new_checksum - self.new_file_content = "new_test_file_bigger_content_123456789" - self.diff_len = len(self.new_file_content) - self.create(self.new_file, self.new_file_content) - stage = self.dvc.add(self.DATA_DIR)[0] - self.git.index.add([self.DATA_DIR + ".dvc"]) - self.git.index.commit(message="modify file in the data dir") - self.new_checksum = stage.outs[0].checksum - self.b_ref = self.git.git.rev_parse(self.git.head.commit, short=True) - - def test(self): - result = self.dvc.diff(self.a_ref, target=self.DATA_DIR) - test_dct = { - diff.DIFF_A_REF: self.git.git.rev_parse(self.a_ref, short=True), - diff.DIFF_B_REF: self.git.git.rev_parse(self.b_ref, short=True), - diff.DIFF_LIST: [ - { - diff.DIFF_IDENT: 2, - diff.DIFF_CHANGE: 1, - diff.DIFF_DEL: 0, - diff.DIFF_MOVE: 0, - diff.DIFF_NEW: 0, - diff.DIFF_IS_DIR: True, - diff.DIFF_TARGET: self.DATA_DIR, - diff.DIFF_NEW_FILE: self.DATA_DIR, - diff.DIFF_NEW_CHECKSUM: self.new_checksum, - diff.DIFF_OLD_FILE: self.DATA_DIR, - diff.DIFF_OLD_CHECKSUM: self.old_checksum, - diff.DIFF_SIZE: self.diff_len, - } - ], - } - self.assertEqual(test_dct, result) - - -class TestDiffCmdMessage(TestDiff): - maxDiff = None - - def test(self): - ret = main( - [ - "diff", - self.test_dct[diff.DIFF_A_REF], - self.test_dct[diff.DIFF_B_REF], - ] - ) - self.assertEqual(ret, 0) - - msg1 = "dvc diff from {0} to {1}".format( - self.git.git.rev_parse(self.test_dct[diff.DIFF_A_REF], short=True), - self.git.git.rev_parse(self.test_dct[diff.DIFF_B_REF], short=True), - ) - msg2 = "diff for '{0}'".format( - self.test_dct[diff.DIFF_LIST][0][diff.DIFF_TARGET] - ) - msg3 = "+{0} with md5 {1}".format( - self.test_dct[diff.DIFF_LIST][0][diff.DIFF_TARGET], - self.test_dct[diff.DIFF_LIST][0][diff.DIFF_NEW_CHECKSUM], - ) - msg4 = "added file with size 13 Bytes" - for m in [msg1, msg2, msg3, msg4]: - assert m in self._caplog.text + assert dvc.diff(":/modify", ":/delete") == { + "added": [], + "deleted": [ + {"path": os.path.join("dir", "2"), "checksum": digest("two")} + ], + "modified": [ + { + "path": os.path.join("dir", ""), + "checksum": { + "old": "9b5faf37366b3370fd98e3e60ca439c1.dir", + "new": "83ae82fb367ac9926455870773ff09e6.dir", + }, + } + ], + } diff --git a/tests/unit/command/test_diff.py b/tests/unit/command/test_diff.py new file mode 100644 index 000000000..5d5a59461 --- /dev/null +++ b/tests/unit/command/test_diff.py @@ -0,0 +1,93 @@ +import collections +import os + +from dvc.cli import parse_args + + +def test_default(mocker, caplog): + args = parse_args(["diff"]) + cmd = args.func(args) + diff = { + "added": [{"path": "file", "checksum": "00000000"}], + "deleted": [], + "modified": [], + } + mocker.patch("dvc.repo.Repo.diff", return_value=diff) + + assert 0 == cmd.run() + assert ( + "Added:\n" + " file\n" + "\n" + "summary: added (1), deleted (0), modified (0)" + ) in caplog.text + + +def test_checksums(mocker, caplog): + args = parse_args(["diff", "--checksums"]) + cmd = args.func(args) + diff = { + "added": [], + "deleted": [ + {"path": os.path.join("data", ""), "checksum": "XXXXXXXX.dir"}, + {"path": os.path.join("data", "bar"), "checksum": "00000000"}, + {"path": os.path.join("data", "foo"), "checksum": "11111111"}, + ], + "modified": [ + { + "path": "file", + "checksum": {"old": "AAAAAAAA", "new": "BBBBBBBB"}, + } + ], + } + mocker.patch("dvc.repo.Repo.diff", return_value=diff) + assert 0 == cmd.run() + assert ( + "Deleted:\n" + " XXXXXXXX " + os.path.join("data", "") + "\n" + " 00000000 " + os.path.join("data", "bar") + "\n" + " 11111111 " + os.path.join("data", "foo") + "\n" + "\n" + "Modified:\n" + " AAAAAAAA..BBBBBBBB file\n" + "\n" + "summary: added (0), deleted (2), modified (1)" + ) in caplog.text + + +def test_json(mocker, caplog): + args = parse_args(["diff", "--show-json"]) + cmd = args.func(args) + diff = { + "added": [{"path": "file", "checksum": "00000000"}], + "deleted": [], + "modified": [], + } + mocker.patch("dvc.repo.Repo.diff", return_value=diff) + + assert 0 == cmd.run() + assert '"added": [{"path": "file"}]' in caplog.text + assert '"deleted": []' in caplog.text + assert '"modified": []' in caplog.text + + +def test_json_checksums(mocker, caplog): + args = parse_args(["diff", "--show-json", "--checksums"]) + cmd = args.func(args) + + diff = { + "added": [ + # py35: maintain a consistent key order for tests purposes + collections.OrderedDict( + [("path", "file"), ("checksum", "00000000")] + ) + ], + "deleted": [], + "modified": [], + } + mocker.patch("dvc.repo.Repo.diff", return_value=diff) + + assert 0 == cmd.run() + assert '"added": [{"path": "file", "checksum": "00000000"}]' in caplog.text + assert '"deleted": []' in caplog.text + assert '"modified": []' in caplog.text