From ab3abbe87a785983da2fbb840076a0c68b28cbe5 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sat, 6 Jan 2018 04:17:55 -0800 Subject: [PATCH] New command: dvc fsck. Close #381. --- dvc/cli.py | 23 ++++++++++++++++ dvc/command/fsck.py | 67 +++++++++++++++++++++++++++++++++++++++++++++ dvc/project.py | 7 +++++ dvc/stage.py | 26 ++++++++++++++++-- dvc/system.py | 6 +++- 5 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 dvc/command/fsck.py diff --git a/dvc/cli.py b/dvc/cli.py index f277e7df0..a64b8fd53 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -6,6 +6,7 @@ import argparse from multiprocessing import cpu_count from dvc.command.find import CmdFind +from dvc.command.fsck import CmdFsck from dvc.command.init import CmdInit from dvc.command.remove import CmdRemove from dvc.command.run import CmdRun @@ -414,6 +415,28 @@ def parse_args(argv=None): default=False, help='Show metrics value.') + fsck_parser = subparsers.add_parser( + 'fsck', + parents=[parent_parser], + help='Data file consistency check') + fsck_parser.add_argument( + 'targets', + nargs='*', + help='Data files to check') + fsck_parser.add_argument( + '-a', + '--all', + action='store_true', + default=False, + help='Check all data files') + fsck_parser.add_argument( + '-p', + '--physical', + action='store_true', + default=False, + help='Compute actual md5') + fsck_parser.set_defaults(func=CmdFsck) + if isinstance(argv, str): argv = argv.split() diff --git a/dvc/command/fsck.py b/dvc/command/fsck.py new file mode 100644 index 000000000..ec4d98af9 --- /dev/null +++ b/dvc/command/fsck.py @@ -0,0 +1,67 @@ +import os + +from dvc.command.common.base import CmdBase +from dvc.data_cloud import file_md5 + + +class CmdFsck(CmdBase): + def run(self): + if not self.args.targets and not self.args.all: + self.project.logger.error(u'Data files are not specified') + return 1 + + directions = self.project.all_directions() + files_and_stages = self.directions_by_datafile(directions) + + if self.args.all: + dvc_files = files_and_stages.keys() + else: + dvc_files = [os.path.relpath(os.path.abspath(f), self.project.root_dir) + for f in self.args.targets] + + caches = self.project.cache.find_cache(dvc_files) + + self.print_fsck(caches, dvc_files, files_and_stages) + return 0 + + def print_fsck(self, caches, dvc_files, files_and_stages): + for file in dvc_files: + print('File {}:'.format(file)) + + full_path = os.path.join(self.project.root_dir, file) + if self.args.physical: + md5 = file_md5(full_path)[0] + print(u' Actual checksum: {}'.format(md5)) + else: + md5 = None + + hardlink_md5 = caches.get(file) + hardlink_msg = hardlink_md5 if caches else 'Error - cache file not found!' + hardlink_error = '!!!' if hardlink_md5 and md5 and md5 <> hardlink_md5 else '' + print(u' Hardlink to cache file: {} {}'.format(hardlink_msg, hardlink_error)) + + state = self.project.state.get(file) + local_state_error = '!!!' if state.md5 and hardlink_md5 and state.md5 <> hardlink_md5 else '' + print(u' Local state checksum: {} {}'.format(state.md5 if state else '', local_state_error)) + print(u' Local state mtime: {}'.format(state.mtime if state else '')) + mtime = int(os.path.getmtime(full_path)) if os.path.exists(full_path) else 'None' + print(u' Actual mtime: {}'.format(mtime)) + + for stage, dep in files_and_stages.get(file, []): + stage_error = '!!!' if dep.md5 and hardlink_md5 and dep.md5 <> hardlink_md5 else '' + print(u' Stage file: {}'.format(stage.dvc_path)) + print(u' Type: {}'.format(dep.direction)) + print(u' Checksum: {} {}'.format(dep.md5, stage_error)) + print(u' Use cache: {}'.format(str(dep.use_cache).lower())) + pass + + @staticmethod + def directions_by_datafile(directions): + result = {} + + for stage, dep in directions: + if dep.dvc_path not in result: + result[dep.dvc_path] = [] + result[dep.dvc_path].append((stage, dep)) + + return result diff --git a/dvc/project.py b/dvc/project.py index 62f620f25..a5c8274c3 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -232,6 +232,13 @@ class Project(object): return G + def all_directions(self): + result = [] + for stage in self.stages(): + for dep in stage.deps + stage.outs: + result.append((stage, dep)) + return result + def stages(self): stages = [] for root, dirs, files in os.walk(self.root_dir): diff --git a/dvc/stage.py b/dvc/stage.py index e0fdfa883..ed9ae6508 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -38,7 +38,12 @@ class OutputAlreadyTrackedError(OutputError): super(OutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)') -class Output(object): +class Direction(object): + DIRECTION_TYPE_OUT = 'Output' + DIRECTION_TYPE_DEP = 'Dependency' + + +class Output(Direction): PARAM_PATH = 'path' PARAM_MD5 = 'md5' PARAM_CACHE = 'cache' @@ -52,8 +57,16 @@ class Output(object): self.md5 = md5 self.use_cache = use_cache + self.direction = self.DIRECTION_TYPE_OUT + + @property + def dvc_path(self): + return os.path.relpath(self.path, self.project.root_dir) def _changed_md5(self): + if not os.path.exists(self.path): + return True + state = self.project.state.get(self.path) if state and state.mtime == self.mtime(): md5 = state.md5 @@ -179,7 +192,9 @@ class Output(object): class Dependency(Output): - pass + def __init__(self, project, path, md5=None, use_cache=False): + super(Dependency, self).__init__(project, path, md5, use_cache) + self.direction = self.DIRECTION_TYPE_DEP class Stage(object): @@ -200,6 +215,10 @@ class Stage(object): self.deps = deps self.locked = locked + @property + def dvc_path(self): + return os.path.relpath(self.path, self.project.root_dir) + @staticmethod def is_stage_file(path): if not os.path.isfile(path): @@ -223,7 +242,8 @@ class Stage(object): self.project.logger.debug("Removing '{}'".format(out.path)) os.chmod(out.path, stat.S_IWUSR) os.unlink(out.path) - os.chmod(out.cache, stat.S_IREAD) + if os.path.exists(out.cache): + os.chmod(out.cache, stat.S_IREAD) def remove(self): self.remove_outs() diff --git a/dvc/system.py b/dvc/system.py index 99c662e9a..b8b0c8098 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -26,7 +26,11 @@ class System(object): return winlink.create(source, link_name) @staticmethod - def samefile(path1, path2): + def samefile(path1, path2, check_exist=False): + if check_exist: + if not os.path.exists(path1) or not os.path.exists(path2): + return False + if System.is_unix(): return os.path.samefile(path1, path2)