mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-11 07:18:28 +01:00
dvc: add support for dvc diff command
Some summary statistics regarding the folder/file. How many files were added/deleted/modified. How size changed. TO DO things: 1)To support for detecting diff after `dvc move` command. 2)To add unit tests.
This commit is contained in:
Zhanibek
@@ -32,6 +32,7 @@ import dvc.command.pipeline as pipeline
|
||||
import dvc.command.daemon as daemon
|
||||
import dvc.command.commit as commit
|
||||
import dvc.command.tag as tag
|
||||
import dvc.command.diff as diff
|
||||
from dvc.exceptions import DvcParserError
|
||||
from dvc import VERSION
|
||||
|
||||
@@ -62,6 +63,7 @@ COMMANDS = [
|
||||
daemon,
|
||||
commit,
|
||||
tag,
|
||||
diff,
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from dvc.command.base import CmdBase
|
||||
|
||||
|
||||
class CmdMove(CmdBase):
|
||||
def run(self):
|
||||
self.repo.diff(
|
||||
self.args.target, a_ref=self.args.a_ref, b_ref=self.args.b_ref
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
description = (
|
||||
"show diff of a data file or a directory that "
|
||||
"is under DVC control. Some basic statistics "
|
||||
"summary, how many files were deleted/changed."
|
||||
)
|
||||
help = "Show a diff of a DVC controlled data file or a directory."
|
||||
diff_parser = subparsers.add_parser(
|
||||
"diff", parents=[parent_parser], description=description, help=help
|
||||
)
|
||||
diff_parser.add_argument(
|
||||
"target", help="Source path to a data file or directory."
|
||||
)
|
||||
diff_parser.add_argument(
|
||||
"a_ref", help="Git tag/reference from which diff calculates"
|
||||
)
|
||||
diff_parser.add_argument(
|
||||
"b_ref",
|
||||
help="Git tag/reference till which diff calculates",
|
||||
nargs="?",
|
||||
default=None,
|
||||
)
|
||||
diff_parser.set_defaults(func=CmdMove)
|
||||
@@ -33,6 +33,7 @@ class Repo(object):
|
||||
from dvc.repo.gc import gc
|
||||
from dvc.repo.commit import commit
|
||||
from dvc.repo.pkg import install_pkg
|
||||
from dvc.repo.diff import diff
|
||||
from dvc.repo.brancher import brancher
|
||||
|
||||
def __init__(self, root_dir=None):
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import os
|
||||
import math
|
||||
|
||||
import dvc.logger as logger
|
||||
|
||||
|
||||
def _extract_tree(self, cache_dir):
|
||||
lst = self.cache.local.load_dir_cache(cache_dir)
|
||||
return {i["relpath"]: i["md5"] for i in lst}
|
||||
|
||||
|
||||
def _convert_size(size_bytes):
|
||||
sign = "" if size_bytes > 0 else "-"
|
||||
if size_bytes > 0:
|
||||
sign = ""
|
||||
else:
|
||||
size_bytes = -size_bytes
|
||||
sign = "-"
|
||||
size_bytes = abs(size_bytes)
|
||||
if size_bytes == 0:
|
||||
return "0B"
|
||||
size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
|
||||
i = int(math.floor(math.log(size_bytes, 1024)))
|
||||
p = math.pow(1024, i)
|
||||
s = round(size_bytes / p, 2)
|
||||
return "%s %s" % (s, size_name[i]), sign
|
||||
|
||||
|
||||
def _get_tree_changes(self, a_entries, b_entries):
|
||||
result = {"del": 0, "ident": 0, "changes": 0, "new": 0, "moves": 0}
|
||||
keys = set(a_entries.keys())
|
||||
keys.update(b_entries.keys())
|
||||
diff_size = 0
|
||||
for key in keys:
|
||||
if key in a_entries and key in b_entries:
|
||||
if a_entries[key] == b_entries[key]:
|
||||
result["ident"] += 1
|
||||
else:
|
||||
result["changes"] += 1
|
||||
diff_size += os.path.getsize(
|
||||
self.cache.local.get(b_entries[key])
|
||||
) - os.path.getsize(self.cache.local.get(a_entries[key]))
|
||||
elif key in a_entries:
|
||||
result["del"] += 1
|
||||
diff_size -= os.path.getsize(self.cache.local.get(a_entries[key]))
|
||||
else:
|
||||
result["new"] += 1
|
||||
diff_size += os.path.getsize(self.cache.local.get(b_entries[key]))
|
||||
result["diff_size"], result["diff_sign"] = _convert_size(diff_size)
|
||||
return result
|
||||
|
||||
|
||||
def plural(s):
|
||||
return str(s) + " file" + "s" * (s > 1)
|
||||
|
||||
|
||||
def _diff_dir(self, target, diff_dct):
|
||||
msg = ""
|
||||
a_entries, b_entries = {}, {}
|
||||
if not diff_dct["new_file"]:
|
||||
msg += "-{} with md5 {}\n".format(
|
||||
target, diff_dct["a_fname"].split(".dir")[0]
|
||||
)
|
||||
a_entries = _extract_tree(self, diff_dct["a_fname"])
|
||||
if not diff_dct["deleted_file"]:
|
||||
msg += "+{} with md5 {}\n".format(
|
||||
target, diff_dct["b_fname"].split(".dir")[0]
|
||||
)
|
||||
b_entries = _extract_tree(self, diff_dct["b_fname"])
|
||||
msg += "\n"
|
||||
result = _get_tree_changes(self, a_entries, b_entries)
|
||||
msg += plural(result["ident"]) + " didn't change, "
|
||||
msg += plural(result["changes"]) + " modified, "
|
||||
msg += plural(result["new"]) + " added, "
|
||||
msg += plural(result["del"]) + " deleted, "
|
||||
msg += "size was " + "increased" * (result["diff_sign"] != "-")
|
||||
msg += (
|
||||
"decreased" * (result["diff_sign"] == "-")
|
||||
+ " by "
|
||||
+ result["diff_size"]
|
||||
)
|
||||
return msg
|
||||
|
||||
|
||||
def _diff_file(self, target, diff_dct):
|
||||
msg = ""
|
||||
size = 0
|
||||
# os.path.getsize(self.cache.local.get(diff_obj['a_fname']))
|
||||
if not diff_dct["new_file"]:
|
||||
msg += "-{} with md5 {}\n".format(
|
||||
target, diff_dct["a_fname"].split(".dir")[0]
|
||||
)
|
||||
size -= os.path.getsize(self.cache.local.get(diff_dct["a_fname"]))
|
||||
if not diff_dct["deleted_file"]:
|
||||
msg += "+{} with md5 {}\n".format(
|
||||
target, diff_dct["b_fname"].split(".dir")[0]
|
||||
)
|
||||
size += os.path.getsize(self.cache.local.get(diff_dct["b_fname"]))
|
||||
if not diff_dct["new_file"] and not diff_dct["deleted_file"] and size == 0:
|
||||
msg += "file did not change\n"
|
||||
elif diff_dct["new_file"]:
|
||||
msg += "added file with size %s" % _convert_size(size)[0]
|
||||
elif diff_dct["deleted_file"]:
|
||||
msg += "deleted file with size %s" % _convert_size(size)[0]
|
||||
else:
|
||||
msg += "file was modified, file size " + "increased" * (size >= 0)
|
||||
msg += "decreased" * (size < 0) + " by " + _convert_size(size)[0]
|
||||
return msg
|
||||
|
||||
|
||||
def diff(self, target, a_ref=None, b_ref=None):
|
||||
"""
|
||||
Gerenates diff message string output
|
||||
input:
|
||||
target - file/folder to check diff of
|
||||
a_ref - first git ref/tag
|
||||
b_ref(optional) - second git ref/tag
|
||||
output:
|
||||
string of output message with diff info
|
||||
"""
|
||||
diff_dct = self.scm.diff(target, a_ref=a_ref, b_ref=b_ref)
|
||||
msg = "dvc diff for '{}' from {} to {} \n\n".format(
|
||||
target, diff_dct["a_tag"], diff_dct["b_tag"]
|
||||
)
|
||||
if diff_dct["equal"]:
|
||||
logger.info(msg)
|
||||
return msg
|
||||
if diff_dct["is_dir"]:
|
||||
dir_info = _diff_dir(self, target, diff_dct)
|
||||
msg += dir_info
|
||||
else:
|
||||
file_info = _diff_file(self, target, diff_dct)
|
||||
msg += file_info
|
||||
logger.info(msg)
|
||||
return msg
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from dvc.utils.compat import str, open
|
||||
from dvc.utils import fix_env
|
||||
@@ -235,3 +236,82 @@ class Git(Base):
|
||||
|
||||
def get_tree(self, rev):
|
||||
return GitTree(self.git, rev)
|
||||
|
||||
def _get_diff_obj(self, file_name, a_ref, **kwargs):
|
||||
|
||||
from gitdb.exc import BadObject, BadName
|
||||
|
||||
file_name = file_name[:-1] if file_name.endswith("/") else file_name
|
||||
file_name += ".dvc"
|
||||
commits = []
|
||||
try:
|
||||
if "b_ref" in kwargs and kwargs["b_ref"] is not None:
|
||||
b_ref = kwargs["b_ref"]
|
||||
a_commit = self.git.commit(a_ref)
|
||||
b_commit = self.git.commit(b_ref)
|
||||
commits.append(str(a_commit))
|
||||
commits.append(str(b_commit))
|
||||
diff_obj = a_commit.diff(b_commit, paths=file_name)
|
||||
# other way of getting the diff between commits :
|
||||
# self.git.diff(a_commit, b_commit) returns the diff string
|
||||
# by reading the files in a, b commits and computing the diff
|
||||
else:
|
||||
a_commit = self.git.head.commit
|
||||
b_commit = self.git.commit(a_ref)
|
||||
commits.append(str(a_commit))
|
||||
commits.append(str(b_commit))
|
||||
if a_commit == b_commit:
|
||||
return None, commits
|
||||
diff_obj = a_commit.diff(b_commit, paths=file_name)
|
||||
except (BadName, BadObject) as e:
|
||||
raise SCMError(e)
|
||||
if len(diff_obj) == 0:
|
||||
msg = "Have not found file/folder '{}' in the commits"
|
||||
raise FileNotInRepoError(msg.format(file_name))
|
||||
return diff_obj[0], commits
|
||||
|
||||
def _extract_fname(self, is_deleted, blob_obj):
|
||||
if not is_deleted:
|
||||
s1 = blob_obj.data_stream.read().decode("utf-8")
|
||||
a_hash, a_fname = re.findall(r"md5:\s([a-fA-F\d]{32}[.dir]*)", s1)
|
||||
return a_hash, a_fname
|
||||
else:
|
||||
return None, None
|
||||
|
||||
def diff(self, file_name, a_ref, **kwargs):
|
||||
"""
|
||||
function for diff between two git tag commits
|
||||
returns the dvc hash names of changed file/folder
|
||||
file_name - file on which to calculate diff
|
||||
returns dictionary with keys: (new_file, a_file_name, b_file_name)
|
||||
"""
|
||||
diff_dct = {"equal": False}
|
||||
diff_obj, commit_refs = self._get_diff_obj(file_name, a_ref, **kwargs)
|
||||
diff_dct["a_tag"] = commit_refs[0]
|
||||
diff_dct["b_tag"] = commit_refs[1]
|
||||
if diff_obj is None:
|
||||
diff_dct["equal"] = True
|
||||
return diff_dct
|
||||
if not diff_obj.a_path:
|
||||
msg = (
|
||||
"file path {} doesn't exist maybe you forgot to add dataset"
|
||||
" to dvc?"
|
||||
)
|
||||
raise SCMError(msg.format(file_name))
|
||||
# this function doesn't support an indicating of moved of a file/folder
|
||||
diff_dct["new_file"] = diff_obj.new_file
|
||||
diff_dct["deleted_file"] = diff_obj.deleted_file
|
||||
diff_dct["a_hash"], diff_dct["a_fname"] = self._extract_fname(
|
||||
diff_dct["new_file"], diff_obj.a_blob
|
||||
)
|
||||
diff_dct["b_hash"], diff_dct["b_fname"] = self._extract_fname(
|
||||
diff_dct["deleted_file"], diff_obj.b_blob
|
||||
)
|
||||
actual_path = (
|
||||
diff_dct["b_fname"] if diff_dct["b_fname"] else diff_dct["a_fname"]
|
||||
)
|
||||
diff_dct["is_dir"] = actual_path.endswith(".dir")
|
||||
diff_dct["file_path"] = (
|
||||
diff_obj.b_path if diff_obj.b_path else diff_obj.a_path
|
||||
)
|
||||
return diff_dct
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
from __future__ import unicode_literals
|
||||
import os
|
||||
|
||||
from dvc.main import main
|
||||
|
||||
|
||||
from dvc.scm.base import FileNotInRepoError
|
||||
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestDiff(TestDvc):
|
||||
def setUp(self):
|
||||
super(TestDiff, self).setUp()
|
||||
|
||||
self.new_file = "new_file"
|
||||
self.create(self.new_file, self.new_file)
|
||||
self.dvc.add(self.new_file)
|
||||
self.a_ref = self.git.head.commit
|
||||
self.git.index.add([self.new_file + ".dvc"])
|
||||
self.git.index.commit("adds new_file")
|
||||
|
||||
def test(self):
|
||||
out = self.dvc.scm.diff(self.new_file, self.a_ref)
|
||||
self.assertTrue(out["deleted_file"])
|
||||
self.assertEqual("new_file.dvc", out["file_path"])
|
||||
|
||||
|
||||
class TestDiffRepo(TestDiff):
|
||||
def test(self):
|
||||
msg = self.dvc.diff(self.new_file, a_ref=self.a_ref)
|
||||
self.assertTrue("deleted file" in msg)
|
||||
self.assertTrue(self.new_file in msg)
|
||||
|
||||
|
||||
class TestDiffCmdLine(TestDiff):
|
||||
def test(self):
|
||||
ret = main(["diff", self.new_file, str(self.a_ref)])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
class TestDiffDir(TestDvc):
|
||||
def setUp(self):
|
||||
super(TestDiffDir, self).setUp()
|
||||
|
||||
self.dvc.add(self.DATA_DIR)
|
||||
self.git.index.add([self.DATA_DIR + ".dvc"])
|
||||
self.git.index.commit("adds data_dir")
|
||||
self.a_ref = str(self.dvc.scm.git.head.commit)
|
||||
self.new_file = os.path.join(self.DATA_SUB_DIR, "new_file")
|
||||
self.create(self.new_file, self.new_file)
|
||||
self.dvc.add(self.DATA_DIR)
|
||||
self.git.index.add([self.DATA_DIR + ".dvc"])
|
||||
self.git.index.commit(message="adds data_dir with new_file")
|
||||
|
||||
def test(self):
|
||||
out = self.dvc.scm.diff(self.DATA_DIR, self.a_ref)
|
||||
self.assertFalse(out["deleted_file"], "new file added checker failed")
|
||||
self.assertEqual(
|
||||
self.DATA_DIR + ".dvc",
|
||||
out["file_path"],
|
||||
"path to file/folder is incorrect",
|
||||
)
|
||||
|
||||
|
||||
class TestDiffDirRepo(TestDiffDir):
|
||||
def test(self):
|
||||
msg = self.dvc.diff(self.DATA_DIR, a_ref=self.a_ref)
|
||||
self.assertTrue("1 file deleted" in msg, msg)
|
||||
self.assertTrue("size was decreased by " in msg, msg)
|
||||
|
||||
|
||||
class TestDiffFileNotFound(TestDiffDir):
|
||||
def setUp(self):
|
||||
super(TestDiffFileNotFound, self).setUp()
|
||||
self.unknown_file = "unknown_file_" + str(id(self))
|
||||
|
||||
def test(self):
|
||||
with self.assertRaises(
|
||||
FileNotInRepoError, msg="diff with incorrect arg"
|
||||
) as context:
|
||||
self.dvc.scm.diff(self.unknown_file, self.a_ref)
|
||||
self.assertTrue(
|
||||
"Have not found file/folder" in str(context.exception),
|
||||
"scm raises with incorrect error message",
|
||||
)
|
||||
|
||||
|
||||
class TestDiffFileNotFoundRepo(TestDiffFileNotFound):
|
||||
def test(self):
|
||||
with self.assertRaises(
|
||||
FileNotInRepoError, msg="diff with incorrect arg"
|
||||
) as context:
|
||||
self.dvc.diff(self.unknown_file, a_ref=self.a_ref)
|
||||
self.assertTrue("Have not found file/folder" in str(context.exception))
|
||||
Reference in New Issue
Block a user