From c2639cfb3a8d5087fac1d581270009b63cccd041 Mon Sep 17 00:00:00 2001 From: Zhanibek Date: Sat, 23 Mar 2019 15:21:16 +0600 Subject: [PATCH] dvc: add support for dvc diff command Some summary statistics regarding the folder/file. How many files were added/deleted/modified. How size changed. TO DO things: 1)To support for detecting diff after `dvc move` command. 2)To add unit tests. --- dvc/cli.py | 2 + dvc/command/diff.py | 36 +++++++++++ dvc/repo/__init__.py | 1 + dvc/repo/diff.py | 135 ++++++++++++++++++++++++++++++++++++++++ dvc/scm/git/__init__.py | 80 ++++++++++++++++++++++++ tests/test_diff.py | 96 ++++++++++++++++++++++++++++ 6 files changed, 350 insertions(+) create mode 100644 dvc/command/diff.py create mode 100644 dvc/repo/diff.py create mode 100644 tests/test_diff.py diff --git a/dvc/cli.py b/dvc/cli.py index d8af5717a..7fe70dbef 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -32,6 +32,7 @@ import dvc.command.pipeline as pipeline import dvc.command.daemon as daemon import dvc.command.commit as commit import dvc.command.tag as tag +import dvc.command.diff as diff from dvc.exceptions import DvcParserError from dvc import VERSION @@ -62,6 +63,7 @@ COMMANDS = [ daemon, commit, tag, + diff, ] diff --git a/dvc/command/diff.py b/dvc/command/diff.py new file mode 100644 index 000000000..01a12c4cc --- /dev/null +++ b/dvc/command/diff.py @@ -0,0 +1,36 @@ +from __future__ import unicode_literals + +from dvc.command.base import CmdBase + + +class CmdMove(CmdBase): + def run(self): + self.repo.diff( + self.args.target, a_ref=self.args.a_ref, b_ref=self.args.b_ref + ) + return 0 + + +def add_parser(subparsers, parent_parser): + description = ( + "show diff of a data file or a directory that " + "is under DVC control. Some basic statistics " + "summary, how many files were deleted/changed." + ) + help = "Show a diff of a DVC controlled data file or a directory." + diff_parser = subparsers.add_parser( + "diff", parents=[parent_parser], description=description, help=help + ) + diff_parser.add_argument( + "target", help="Source path to a data file or directory." + ) + diff_parser.add_argument( + "a_ref", help="Git tag/reference from which diff calculates" + ) + diff_parser.add_argument( + "b_ref", + help="Git tag/reference till which diff calculates", + nargs="?", + default=None, + ) + diff_parser.set_defaults(func=CmdMove) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 43ecbe600..a082e7c0b 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -33,6 +33,7 @@ class Repo(object): from dvc.repo.gc import gc from dvc.repo.commit import commit from dvc.repo.pkg import install_pkg + from dvc.repo.diff import diff from dvc.repo.brancher import brancher def __init__(self, root_dir=None): diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py new file mode 100644 index 000000000..e8681266e --- /dev/null +++ b/dvc/repo/diff.py @@ -0,0 +1,135 @@ +import os +import math + +import dvc.logger as logger + + +def _extract_tree(self, cache_dir): + lst = self.cache.local.load_dir_cache(cache_dir) + return {i["relpath"]: i["md5"] for i in lst} + + +def _convert_size(size_bytes): + sign = "" if size_bytes > 0 else "-" + if size_bytes > 0: + sign = "" + else: + size_bytes = -size_bytes + sign = "-" + size_bytes = abs(size_bytes) + if size_bytes == 0: + return "0B" + size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") + i = int(math.floor(math.log(size_bytes, 1024))) + p = math.pow(1024, i) + s = round(size_bytes / p, 2) + return "%s %s" % (s, size_name[i]), sign + + +def _get_tree_changes(self, a_entries, b_entries): + result = {"del": 0, "ident": 0, "changes": 0, "new": 0, "moves": 0} + keys = set(a_entries.keys()) + keys.update(b_entries.keys()) + diff_size = 0 + for key in keys: + if key in a_entries and key in b_entries: + if a_entries[key] == b_entries[key]: + result["ident"] += 1 + else: + result["changes"] += 1 + diff_size += os.path.getsize( + self.cache.local.get(b_entries[key]) + ) - os.path.getsize(self.cache.local.get(a_entries[key])) + elif key in a_entries: + result["del"] += 1 + diff_size -= os.path.getsize(self.cache.local.get(a_entries[key])) + else: + result["new"] += 1 + diff_size += os.path.getsize(self.cache.local.get(b_entries[key])) + result["diff_size"], result["diff_sign"] = _convert_size(diff_size) + return result + + +def plural(s): + return str(s) + " file" + "s" * (s > 1) + + +def _diff_dir(self, target, diff_dct): + msg = "" + a_entries, b_entries = {}, {} + if not diff_dct["new_file"]: + msg += "-{} with md5 {}\n".format( + target, diff_dct["a_fname"].split(".dir")[0] + ) + a_entries = _extract_tree(self, diff_dct["a_fname"]) + if not diff_dct["deleted_file"]: + msg += "+{} with md5 {}\n".format( + target, diff_dct["b_fname"].split(".dir")[0] + ) + b_entries = _extract_tree(self, diff_dct["b_fname"]) + msg += "\n" + result = _get_tree_changes(self, a_entries, b_entries) + msg += plural(result["ident"]) + " didn't change, " + msg += plural(result["changes"]) + " modified, " + msg += plural(result["new"]) + " added, " + msg += plural(result["del"]) + " deleted, " + msg += "size was " + "increased" * (result["diff_sign"] != "-") + msg += ( + "decreased" * (result["diff_sign"] == "-") + + " by " + + result["diff_size"] + ) + return msg + + +def _diff_file(self, target, diff_dct): + msg = "" + size = 0 + # os.path.getsize(self.cache.local.get(diff_obj['a_fname'])) + if not diff_dct["new_file"]: + msg += "-{} with md5 {}\n".format( + target, diff_dct["a_fname"].split(".dir")[0] + ) + size -= os.path.getsize(self.cache.local.get(diff_dct["a_fname"])) + if not diff_dct["deleted_file"]: + msg += "+{} with md5 {}\n".format( + target, diff_dct["b_fname"].split(".dir")[0] + ) + size += os.path.getsize(self.cache.local.get(diff_dct["b_fname"])) + if not diff_dct["new_file"] and not diff_dct["deleted_file"] and size == 0: + msg += "file did not change\n" + elif diff_dct["new_file"]: + msg += "added file with size %s" % _convert_size(size)[0] + elif diff_dct["deleted_file"]: + msg += "deleted file with size %s" % _convert_size(size)[0] + else: + msg += "file was modified, file size " + "increased" * (size >= 0) + msg += "decreased" * (size < 0) + " by " + _convert_size(size)[0] + return msg + + +def diff(self, target, a_ref=None, b_ref=None): + """ + Gerenates diff message string output + input: + target - file/folder to check diff of + a_ref - first git ref/tag + b_ref(optional) - second git ref/tag + output: + string of output message with diff info + """ + diff_dct = self.scm.diff(target, a_ref=a_ref, b_ref=b_ref) + msg = "dvc diff for '{}' from {} to {} \n\n".format( + target, diff_dct["a_tag"], diff_dct["b_tag"] + ) + if diff_dct["equal"]: + logger.info(msg) + return msg + if diff_dct["is_dir"]: + dir_info = _diff_dir(self, target, diff_dct) + msg += dir_info + else: + file_info = _diff_file(self, target, diff_dct) + msg += file_info + logger.info(msg) + return msg diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index cd585f841..1b5a0fc0e 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import os +import re from dvc.utils.compat import str, open from dvc.utils import fix_env @@ -235,3 +236,82 @@ class Git(Base): def get_tree(self, rev): return GitTree(self.git, rev) + + def _get_diff_obj(self, file_name, a_ref, **kwargs): + + from gitdb.exc import BadObject, BadName + + file_name = file_name[:-1] if file_name.endswith("/") else file_name + file_name += ".dvc" + commits = [] + try: + if "b_ref" in kwargs and kwargs["b_ref"] is not None: + b_ref = kwargs["b_ref"] + a_commit = self.git.commit(a_ref) + b_commit = self.git.commit(b_ref) + commits.append(str(a_commit)) + commits.append(str(b_commit)) + diff_obj = a_commit.diff(b_commit, paths=file_name) + # other way of getting the diff between commits : + # self.git.diff(a_commit, b_commit) returns the diff string + # by reading the files in a, b commits and computing the diff + else: + a_commit = self.git.head.commit + b_commit = self.git.commit(a_ref) + commits.append(str(a_commit)) + commits.append(str(b_commit)) + if a_commit == b_commit: + return None, commits + diff_obj = a_commit.diff(b_commit, paths=file_name) + except (BadName, BadObject) as e: + raise SCMError(e) + if len(diff_obj) == 0: + msg = "Have not found file/folder '{}' in the commits" + raise FileNotInRepoError(msg.format(file_name)) + return diff_obj[0], commits + + def _extract_fname(self, is_deleted, blob_obj): + if not is_deleted: + s1 = blob_obj.data_stream.read().decode("utf-8") + a_hash, a_fname = re.findall(r"md5:\s([a-fA-F\d]{32}[.dir]*)", s1) + return a_hash, a_fname + else: + return None, None + + def diff(self, file_name, a_ref, **kwargs): + """ + function for diff between two git tag commits + returns the dvc hash names of changed file/folder + file_name - file on which to calculate diff + returns dictionary with keys: (new_file, a_file_name, b_file_name) + """ + diff_dct = {"equal": False} + diff_obj, commit_refs = self._get_diff_obj(file_name, a_ref, **kwargs) + diff_dct["a_tag"] = commit_refs[0] + diff_dct["b_tag"] = commit_refs[1] + if diff_obj is None: + diff_dct["equal"] = True + return diff_dct + if not diff_obj.a_path: + msg = ( + "file path {} doesn't exist maybe you forgot to add dataset" + " to dvc?" + ) + raise SCMError(msg.format(file_name)) + # this function doesn't support an indicating of moved of a file/folder + diff_dct["new_file"] = diff_obj.new_file + diff_dct["deleted_file"] = diff_obj.deleted_file + diff_dct["a_hash"], diff_dct["a_fname"] = self._extract_fname( + diff_dct["new_file"], diff_obj.a_blob + ) + diff_dct["b_hash"], diff_dct["b_fname"] = self._extract_fname( + diff_dct["deleted_file"], diff_obj.b_blob + ) + actual_path = ( + diff_dct["b_fname"] if diff_dct["b_fname"] else diff_dct["a_fname"] + ) + diff_dct["is_dir"] = actual_path.endswith(".dir") + diff_dct["file_path"] = ( + diff_obj.b_path if diff_obj.b_path else diff_obj.a_path + ) + return diff_dct diff --git a/tests/test_diff.py b/tests/test_diff.py new file mode 100644 index 000000000..5c9223a60 --- /dev/null +++ b/tests/test_diff.py @@ -0,0 +1,96 @@ +from __future__ import unicode_literals +import os + +from dvc.main import main + + +from dvc.scm.base import FileNotInRepoError + + +from tests.basic_env import TestDvc + + +class TestDiff(TestDvc): + def setUp(self): + super(TestDiff, self).setUp() + + self.new_file = "new_file" + self.create(self.new_file, self.new_file) + self.dvc.add(self.new_file) + self.a_ref = self.git.head.commit + self.git.index.add([self.new_file + ".dvc"]) + self.git.index.commit("adds new_file") + + def test(self): + out = self.dvc.scm.diff(self.new_file, self.a_ref) + self.assertTrue(out["deleted_file"]) + self.assertEqual("new_file.dvc", out["file_path"]) + + +class TestDiffRepo(TestDiff): + def test(self): + msg = self.dvc.diff(self.new_file, a_ref=self.a_ref) + self.assertTrue("deleted file" in msg) + self.assertTrue(self.new_file in msg) + + +class TestDiffCmdLine(TestDiff): + def test(self): + ret = main(["diff", self.new_file, str(self.a_ref)]) + self.assertEqual(ret, 0) + + +class TestDiffDir(TestDvc): + def setUp(self): + super(TestDiffDir, self).setUp() + + self.dvc.add(self.DATA_DIR) + self.git.index.add([self.DATA_DIR + ".dvc"]) + self.git.index.commit("adds data_dir") + self.a_ref = str(self.dvc.scm.git.head.commit) + self.new_file = os.path.join(self.DATA_SUB_DIR, "new_file") + self.create(self.new_file, self.new_file) + self.dvc.add(self.DATA_DIR) + self.git.index.add([self.DATA_DIR + ".dvc"]) + self.git.index.commit(message="adds data_dir with new_file") + + def test(self): + out = self.dvc.scm.diff(self.DATA_DIR, self.a_ref) + self.assertFalse(out["deleted_file"], "new file added checker failed") + self.assertEqual( + self.DATA_DIR + ".dvc", + out["file_path"], + "path to file/folder is incorrect", + ) + + +class TestDiffDirRepo(TestDiffDir): + def test(self): + msg = self.dvc.diff(self.DATA_DIR, a_ref=self.a_ref) + self.assertTrue("1 file deleted" in msg, msg) + self.assertTrue("size was decreased by " in msg, msg) + + +class TestDiffFileNotFound(TestDiffDir): + def setUp(self): + super(TestDiffFileNotFound, self).setUp() + self.unknown_file = "unknown_file_" + str(id(self)) + + def test(self): + with self.assertRaises( + FileNotInRepoError, msg="diff with incorrect arg" + ) as context: + self.dvc.scm.diff(self.unknown_file, self.a_ref) + self.assertTrue( + "Have not found file/folder" in str(context.exception), + "scm raises with incorrect error message", + ) + + +class TestDiffFileNotFoundRepo(TestDiffFileNotFound): + def test(self): + with self.assertRaises( + FileNotInRepoError, msg="diff with incorrect arg" + ) as context: + self.dvc.diff(self.unknown_file, a_ref=self.a_ref) + self.assertTrue("Have not found file/folder" in str(context.exception))