From 7b3790e70136b6ef98e84b05bea0c75e9db0caa9 Mon Sep 17 00:00:00 2001 From: Zhanibek Date: Mon, 1 Apr 2019 20:19:06 +0600 Subject: [PATCH] repo: fix macros placement from `dvc/scm` to `dvc/repo/diff` --- dvc/command/diff.py | 54 ++++++++-------- dvc/repo/diff.py | 139 ++++++++++++++++++++++++---------------- dvc/scm/base.py | 21 ------ dvc/scm/git/__init__.py | 12 ++-- tests/test_diff.py | 66 +++++++++---------- 5 files changed, 149 insertions(+), 143 deletions(-) diff --git a/dvc/command/diff.py b/dvc/command/diff.py index 2aae03bda..5feadce28 100644 --- a/dvc/command/diff.py +++ b/dvc/command/diff.py @@ -8,7 +8,7 @@ import dvc.logger as logger from dvc.exceptions import DvcException from dvc.command.base import CmdBase from dvc.utils.collections import compact -import dvc.scm.base as git +import dvc.repo.diff as diff class CmdDiff(CmdBase): @@ -25,61 +25,61 @@ class CmdDiff(CmdBase): def _show(self, diff_dct): engine = inflect.engine() msg = "dvc diff from {} to {}".format( - diff_dct[git.DIFF_A_REF], diff_dct[git.DIFF_B_REF] + diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF] ) - if diff_dct.get(git.DIFF_EQUAL): + if diff_dct.get(diff.DIFF_EQUAL): logger.info(msg) return - for dct in diff_dct[git.DIFF_LIST]: - msg += "\n\ndiff for '{}' \n".format(dct[git.DIFF_TARGET]) - if dct.get(git.DIFF_OLD_FILE): + for dct in diff_dct[diff.DIFF_LIST]: + msg += "\n\ndiff for '{}' \n".format(dct[diff.DIFF_TARGET]) + if dct.get(diff.DIFF_OLD_FILE): msg += "-{} with md5 {}\n".format( - dct[git.DIFF_OLD_FILE], dct[git.DIFF_OLD_CHECKSUM] + dct[diff.DIFF_OLD_FILE], dct[diff.DIFF_OLD_CHECKSUM] ) - if dct.get(git.DIFF_NEW_FILE): + if dct.get(diff.DIFF_NEW_FILE): msg += "+{} with md5 {}\n".format( - dct[git.DIFF_NEW_FILE], dct[git.DIFF_NEW_CHECKSUM] + dct[diff.DIFF_NEW_FILE], dct[diff.DIFF_NEW_CHECKSUM] ) msg += "\n" - if dct[git.DIFF_SIZE] != git.DIFF_SIZE_UNKNOWN: + if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN: if dct.get("is_dir"): msg += "{} {} not changed, ".format( - dct[git.DIFF_IDENT], - engine.plural("file", dct[git.DIFF_IDENT]), + dct[diff.DIFF_IDENT], + engine.plural("file", dct[diff.DIFF_IDENT]), ) msg += "{} {} modified, ".format( - dct[git.DIFF_CHANGE], - engine.plural("file", dct[git.DIFF_CHANGE]), + dct[diff.DIFF_CHANGE], + engine.plural("file", dct[diff.DIFF_CHANGE]), ) msg += "{} {} added, ".format( - dct[git.DIFF_NEW], - engine.plural("file", dct[git.DIFF_NEW]), + dct[diff.DIFF_NEW], + engine.plural("file", dct[diff.DIFF_NEW]), ) msg += "{} {} deleted, ".format( - dct[git.DIFF_DEL], - engine.plural("file", dct[git.DIFF_DEL]), + dct[diff.DIFF_DEL], + engine.plural("file", dct[diff.DIFF_DEL]), ) msg += "size was {}".format( - self._print_size(dct[git.DIFF_SIZE]) + self._print_size(dct[diff.DIFF_SIZE]) ) else: if ( - dct.get(git.DIFF_OLD_FILE) - and dct.get(git.DIFF_NEW_FILE) - and dct[git.DIFF_SIZE] == 0 + dct.get(diff.DIFF_OLD_FILE) + and dct.get(diff.DIFF_NEW_FILE) + and dct[diff.DIFF_SIZE] == 0 ): msg += "file size was not changed" - elif dct.get(git.DIFF_NEW_FILE): + elif dct.get(diff.DIFF_NEW_FILE): msg += "added file with size {}".format( - humanize.naturalsize(dct[git.DIFF_SIZE]) + humanize.naturalsize(dct[diff.DIFF_SIZE]) ) - elif dct.get(git.DIFF_OLD_FILE): + elif dct.get(diff.DIFF_OLD_FILE): msg += "deleted file with size {}".format( - humanize.naturalsize(abs(dct[git.DIFF_SIZE])) + humanize.naturalsize(abs(dct[diff.DIFF_SIZE])) ) else: msg += "file was modified, file size {}".format( - self._print_size(dct[git.DIFF_SIZE]) + self._print_size(dct[diff.DIFF_SIZE]) ) else: msg += "size is ?" diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index 90e474455..011be69a7 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -4,7 +4,32 @@ import os from dvc.scm.base import FileNotInCommitError -import dvc.scm.base as git +from dvc.scm.git import ( + DIFF_A_TREE, + DIFF_B_TREE, + DIFF_A_REF, + DIFF_B_REF, + DIFF_EQUAL, +) + + +DIFF_TARGET = "target" +DIFF_IS_DIR = "is_dir" +DIFF_OLD_FILE = "old_file" +DIFF_OLD_CHECKSUM = "old_checksum" +DIFF_NEW_FILE = "new_file" +DIFF_NEW_CHECKSUM = "new_checksum" +DIFF_SIZE = "size_diff" +DIFF_DEL = "del" +DIFF_IDENT = "ident" +DIFF_CHANGE = "changes" +DIFF_NEW = "new" +DIFF_MOVE = "moves" +DIFF_LIST = "diffs" +DIFF_SIZE_UNKNOWN = "?" +DIFF_A_OUTPUT = "a_output" +DIFF_B_OUTPUT = "b_output" +DIFF_DELETED = "deleted_file" def _extract_dir(self, output): @@ -22,11 +47,11 @@ def _extract_dir(self, output): def _get_tree_changes(self, a_entries, b_entries): result = { - git.DIFF_DEL: 0, - git.DIFF_IDENT: 0, - git.DIFF_CHANGE: 0, - git.DIFF_NEW: 0, - git.DIFF_MOVE: 0, + DIFF_DEL: 0, + DIFF_IDENT: 0, + DIFF_CHANGE: 0, + DIFF_NEW: 0, + DIFF_MOVE: 0, } keys = set(a_entries.keys()) keys.update(b_entries.keys()) @@ -34,26 +59,26 @@ def _get_tree_changes(self, a_entries, b_entries): for key in keys: if key in a_entries and key in b_entries: if a_entries[key] == b_entries[key]: - result[git.DIFF_IDENT] += 1 + result[DIFF_IDENT] += 1 else: - result[git.DIFF_CHANGE] += 1 + result[DIFF_CHANGE] += 1 diff_size += os.path.getsize( self.cache.local.get(b_entries[key]) ) - os.path.getsize(self.cache.local.get(a_entries[key])) elif key in a_entries: - result[git.DIFF_DEL] += 1 + result[DIFF_DEL] += 1 diff_size -= os.path.getsize(self.cache.local.get(a_entries[key])) else: - result[git.DIFF_NEW] += 1 + result[DIFF_NEW] += 1 diff_size += os.path.getsize(self.cache.local.get(b_entries[key])) - result[git.DIFF_SIZE] = diff_size + result[DIFF_SIZE] = diff_size return result def _get_diff_outs(self, diff_dct): - self.tree = diff_dct[git.DIFF_A_TREE] + self.tree = diff_dct[DIFF_A_TREE] a_outs = {str(out): out for st in self.stages() for out in st.outs} - self.tree = diff_dct[git.DIFF_B_TREE] + self.tree = diff_dct[DIFF_B_TREE] b_outs = {str(out): out for st in self.stages() for out in st.outs} outs_paths = set(a_outs.keys()) outs_paths.update(b_outs.keys()) @@ -61,73 +86,73 @@ def _get_diff_outs(self, diff_dct): for path in outs_paths: if path in a_outs and path in b_outs: results[path] = { - "a_output": a_outs[path], - "b_output": b_outs[path], - git.DIFF_NEW_FILE: False, - "deleted_file": False, + DIFF_A_OUTPUT: a_outs[path], + DIFF_B_OUTPUT: b_outs[path], + DIFF_NEW_FILE: False, + DIFF_DELETED: False, # possible drawback: regular file ->directory movement - git.DIFF_IS_DIR: a_outs[path].is_dir_cache, + DIFF_IS_DIR: a_outs[path].is_dir_cache, } elif path in a_outs: results[path] = { - "a_output": a_outs[path], - git.DIFF_NEW_FILE: False, - "deleted_file": True, - git.DIFF_IS_DIR: a_outs[path].is_dir_cache, + DIFF_A_OUTPUT: a_outs[path], + DIFF_NEW_FILE: False, + DIFF_DELETED: True, + DIFF_IS_DIR: a_outs[path].is_dir_cache, } else: results[path] = { - "b_output": b_outs[path], - git.DIFF_NEW_FILE: True, - "deleted_file": False, - git.DIFF_IS_DIR: b_outs[path].is_dir_cache, + DIFF_B_OUTPUT: b_outs[path], + DIFF_NEW_FILE: True, + DIFF_DELETED: False, + DIFF_IS_DIR: b_outs[path].is_dir_cache, } return results def _diff_dir(self, target, diff_dct): - result = {git.DIFF_TARGET: target} - result[git.DIFF_IS_DIR] = True + result = {DIFF_TARGET: target} + result[DIFF_IS_DIR] = True a_entries, b_entries = {}, {} try: - if not diff_dct[git.DIFF_NEW_FILE]: - result[git.DIFF_OLD_FILE] = target - result[git.DIFF_OLD_CHECKSUM] = diff_dct["a_output"].checksum - a_entries = _extract_dir(self, diff_dct["a_output"]) - if not diff_dct["deleted_file"]: - result[git.DIFF_NEW_FILE] = target - result[git.DIFF_NEW_CHECKSUM] = diff_dct["b_output"].checksum - b_entries = _extract_dir(self, diff_dct["b_output"]) + if not diff_dct[DIFF_NEW_FILE]: + result[DIFF_OLD_FILE] = target + result[DIFF_OLD_CHECKSUM] = diff_dct[DIFF_A_OUTPUT].checksum + a_entries = _extract_dir(self, diff_dct[DIFF_A_OUTPUT]) + if not diff_dct[DIFF_DELETED]: + result[DIFF_NEW_FILE] = target + result[DIFF_NEW_CHECKSUM] = diff_dct[DIFF_B_OUTPUT].checksum + b_entries = _extract_dir(self, diff_dct[DIFF_B_OUTPUT]) result.update(_get_tree_changes(self, a_entries, b_entries)) except FileNotFoundError: - result[git.DIFF_SIZE] = git.DIFF_SIZE_UNKNOWN + result[DIFF_SIZE] = DIFF_SIZE_UNKNOWN return result def _diff_file(self, target, diff_dct): - result = {git.DIFF_TARGET: target} + result = {DIFF_TARGET: target} size = 0 try: - if not diff_dct[git.DIFF_NEW_FILE]: - result[git.DIFF_OLD_FILE] = target - result[git.DIFF_OLD_CHECKSUM] = diff_dct["a_output"].checksum + if not diff_dct[DIFF_NEW_FILE]: + result[DIFF_OLD_FILE] = target + result[DIFF_OLD_CHECKSUM] = diff_dct[DIFF_A_OUTPUT].checksum size -= os.path.getsize( - self.cache.local.get(diff_dct["a_output"].checksum) + self.cache.local.get(diff_dct[DIFF_A_OUTPUT].checksum) ) - if not diff_dct["deleted_file"]: - result[git.DIFF_NEW_FILE] = target - result[git.DIFF_NEW_CHECKSUM] = diff_dct["b_output"].checksum + if not diff_dct[DIFF_DELETED]: + result[DIFF_NEW_FILE] = target + result[DIFF_NEW_CHECKSUM] = diff_dct[DIFF_B_OUTPUT].checksum size += os.path.getsize( - self.cache.local.get(diff_dct["b_output"].checksum) + self.cache.local.get(diff_dct[DIFF_B_OUTPUT].checksum) ) except FileNotFoundError: - size = git.DIFF_SIZE_UNKNOWN - result[git.DIFF_SIZE] = size + size = DIFF_SIZE_UNKNOWN + result[DIFF_SIZE] = size return result def _diff_royal(self, target, diff_dct): - if diff_dct[git.DIFF_IS_DIR]: + if diff_dct[DIFF_IS_DIR]: return _diff_dir(self, target, diff_dct) return _diff_file(self, target, diff_dct) @@ -137,7 +162,7 @@ def diff(self, a_ref, target=None, b_ref=None): Args: target(str) - file/directory to check diff of - a_ref(str) - first git tag + a_ref(str) - first tag (optional) b_ref(str) - second git tag Returns: @@ -145,19 +170,19 @@ def diff(self, a_ref, target=None, b_ref=None): """ result = {} diff_dct = self.scm.get_diff_trees(a_ref, b_ref=b_ref) - result[git.DIFF_A_REF] = diff_dct[git.DIFF_A_REF] - result[git.DIFF_B_REF] = diff_dct[git.DIFF_B_REF] - if diff_dct[git.DIFF_EQUAL]: - result[git.DIFF_EQUAL] = True + result[DIFF_A_REF] = diff_dct[DIFF_A_REF] + result[DIFF_B_REF] = diff_dct[DIFF_B_REF] + if diff_dct[DIFF_EQUAL]: + result[DIFF_EQUAL] = True return result - result[git.DIFF_LIST] = [] + result[DIFF_LIST] = [] diff_outs = _get_diff_outs(self, diff_dct) if target is None: - result[git.DIFF_LIST] = [ + result[DIFF_LIST] = [ _diff_royal(self, path, diff_outs[path]) for path in diff_outs ] elif target in diff_outs: - result[git.DIFF_LIST] = [_diff_royal(self, target, diff_outs[target])] + result[DIFF_LIST] = [_diff_royal(self, target, diff_outs[target])] else: msg = "Have not found file/directory '{}' in the commits" raise FileNotInCommitError(msg.format(target)) diff --git a/dvc/scm/base.py b/dvc/scm/base.py index be054345a..d7f5333e3 100644 --- a/dvc/scm/base.py +++ b/dvc/scm/base.py @@ -7,27 +7,6 @@ import os from dvc.exceptions import DvcException -DIFF_TARGET = "target" -DIFF_IS_DIR = "is_dir" -DIFF_OLD_FILE = "old_file" -DIFF_OLD_CHECKSUM = "old_checksum" -DIFF_NEW_FILE = "new_file" -DIFF_NEW_CHECKSUM = "new_checksum" -DIFF_SIZE = "size_diff" -DIFF_DEL = "del" -DIFF_IDENT = "ident" -DIFF_CHANGE = "changes" -DIFF_NEW = "new" -DIFF_MOVE = "moves" -DIFF_A_TREE = "a_tree" -DIFF_B_TREE = "b_tree" -DIFF_A_REF = "a_ref" -DIFF_B_REF = "b_ref" -DIFF_EQUAL = "equal" -DIFF_LIST = "diffs" -DIFF_SIZE_UNKNOWN = "?" - - class SCMError(DvcException): """Base class for source control management errors.""" diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 186fad525..9eb49f096 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -11,16 +11,18 @@ from dvc.scm.base import ( SCMError, FileNotInRepoError, FileNotInTargetSubdirError, - DIFF_A_TREE, - DIFF_B_TREE, - DIFF_A_REF, - DIFF_B_REF, - DIFF_EQUAL, ) from dvc.scm.git.tree import GitTree import dvc.logger as logger +DIFF_A_TREE = "a_tree" +DIFF_B_TREE = "b_tree" +DIFF_A_REF = "a_ref" +DIFF_B_REF = "b_ref" +DIFF_EQUAL = "equal" + + class Git(Base): """Class for managing Git.""" diff --git a/tests/test_diff.py b/tests/test_diff.py index a8147a8ec..e22d78620 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -7,7 +7,7 @@ from dvc.main import main from dvc.scm.base import FileNotInCommitError -import dvc.scm.base as git +import dvc.repo.diff as diff from tests.basic_env import TestDvc @@ -34,23 +34,23 @@ class TestDiff(TestDvc): def test(self): out = self.dvc.scm.get_diff_trees(self.a_ref) - self.assertFalse(out[git.DIFF_EQUAL]) - self.assertEqual(str(self.a_ref), out[git.DIFF_A_REF]) - self.assertEqual(str(self.git.head.commit), out[git.DIFF_B_REF]) + self.assertFalse(out[diff.DIFF_EQUAL]) + self.assertEqual(str(self.a_ref), out[diff.DIFF_A_REF]) + self.assertEqual(str(self.git.head.commit), out[diff.DIFF_B_REF]) class TestDiffRepo(TestDiff): def test(self): - result = self.dvc.diff(self.new_file, a_ref=self.a_ref) + result = self.dvc.diff(self.a_ref, target=self.new_file) test_dct = { - git.DIFF_A_REF: str(self.a_ref), - git.DIFF_B_REF: str(self.git.head.commit), - git.DIFF_LIST: [ + diff.DIFF_A_REF: str(self.a_ref), + diff.DIFF_B_REF: str(self.git.head.commit), + diff.DIFF_LIST: [ { - git.DIFF_TARGET: self.new_file, - git.DIFF_NEW_FILE: self.new_file, - git.DIFF_NEW_CHECKSUM: self.new_checksum, - git.DIFF_SIZE: 13, + diff.DIFF_TARGET: self.new_file, + diff.DIFF_NEW_FILE: self.new_file, + diff.DIFF_NEW_CHECKSUM: self.new_checksum, + diff.DIFF_SIZE: 13, } ], } @@ -76,7 +76,7 @@ class TestDiffDir(TestDvc): self.git.index.commit("adds data_dir") self.a_ref = str(self.dvc.scm.git.head.commit) self.old_checksum = _get_checksum(self.dvc, self.DATA_DIR) - self.new_file = os.path.join(self.DATA_SUB_DIR, git.DIFF_NEW_FILE) + self.new_file = os.path.join(self.DATA_SUB_DIR, diff.DIFF_NEW_FILE) self.create(self.new_file, self.new_file) self.dvc.add(self.DATA_DIR) self.git.index.add([self.DATA_DIR + ".dvc"]) @@ -85,33 +85,33 @@ class TestDiffDir(TestDvc): def test(self): out = self.dvc.scm.get_diff_trees(self.a_ref) - self.assertFalse(out[git.DIFF_EQUAL]) - self.assertEqual(str(self.a_ref), out[git.DIFF_A_REF]) - self.assertEqual(str(self.git.head.commit), out[git.DIFF_B_REF]) + self.assertFalse(out[diff.DIFF_EQUAL]) + self.assertEqual(str(self.a_ref), out[diff.DIFF_A_REF]) + self.assertEqual(str(self.git.head.commit), out[diff.DIFF_B_REF]) class TestDiffDirRepo(TestDiffDir): maxDiff = None def test(self): - result = self.dvc.diff(self.DATA_DIR, a_ref=self.a_ref) + result = self.dvc.diff(self.a_ref, target=self.DATA_DIR) test_dct = { - git.DIFF_A_REF: str(self.a_ref), - git.DIFF_B_REF: str(self.git.head.commit), - git.DIFF_LIST: [ + diff.DIFF_A_REF: str(self.a_ref), + diff.DIFF_B_REF: str(self.git.head.commit), + diff.DIFF_LIST: [ { - git.DIFF_CHANGE: 0, - git.DIFF_DEL: 0, - git.DIFF_IDENT: 2, - git.DIFF_MOVE: 0, - git.DIFF_NEW: 1, - git.DIFF_IS_DIR: True, - git.DIFF_TARGET: self.DATA_DIR, - git.DIFF_NEW_FILE: self.DATA_DIR, - git.DIFF_NEW_CHECKSUM: self.new_checksum, - git.DIFF_OLD_FILE: self.DATA_DIR, - git.DIFF_OLD_CHECKSUM: self.old_checksum, - git.DIFF_SIZE: 30, + diff.DIFF_CHANGE: 0, + diff.DIFF_DEL: 0, + diff.DIFF_IDENT: 2, + diff.DIFF_MOVE: 0, + diff.DIFF_NEW: 1, + diff.DIFF_IS_DIR: True, + diff.DIFF_TARGET: self.DATA_DIR, + diff.DIFF_NEW_FILE: self.DATA_DIR, + diff.DIFF_NEW_CHECKSUM: self.new_checksum, + diff.DIFF_OLD_FILE: self.DATA_DIR, + diff.DIFF_OLD_CHECKSUM: self.old_checksum, + diff.DIFF_SIZE: 30, } ], } @@ -125,4 +125,4 @@ class TestDiffFileNotFound(TestDiffDir): def test(self): with self.assertRaises(FileNotInCommitError): - self.dvc.diff(self.unknown_file, a_ref=self.a_ref) + self.dvc.diff(self.a_ref, target=self.unknown_file)