dvc: remove fsck

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-03-16 11:23:54 +02:00
parent 80a53f6d77
commit b38b12962d
4 changed files with 0 additions and 337 deletions
-24
View File
@@ -5,7 +5,6 @@ import sys
import argparse
from multiprocessing import cpu_count
from dvc.command.fsck import CmdFsck
from dvc.command.init import CmdInit
from dvc.command.remove import CmdRemove
from dvc.command.run import CmdRun
@@ -228,27 +227,4 @@ def parse_args(argv=None):
help='Option value')
config_parser.set_defaults(func=CmdConfig)
# Fsck
fsck_parser = subparsers.add_parser(
'fsck',
parents=[parent_parser],
help='Data file consistency check')
fsck_parser.add_argument(
'targets',
nargs='*',
help='Data files to check')
fsck_parser.add_argument(
'-p',
'--physical',
action='store_true',
default=False,
help='Compute actual md5')
fsck_parser.add_argument(
'-a',
'--all',
action='store_true',
default=False,
help='Show all data files including correct ones')
fsck_parser.set_defaults(func=CmdFsck)
return parser.parse_args(argv)
-172
View File
@@ -1,172 +0,0 @@
import os
from dvc import stage
from dvc.command.common.base import CmdBase
from dvc.utils import file_md5
class FsckFileDep(object):
def __init__(self, dvc_file_name, dep):
self.dvc_file_name = dvc_file_name
self.md5 = dep.md5
self.type_name = type(dep).__name__
self.use_cache = dep.use_cache if hasattr(dep, 'use_cache') else False
@property
def is_output(self):
return self.type_name == stage.Output.__name__
class FsckFile(object):
ERR_STATUS_NO_CACHE_FILE = 'No cache file found'
ERR_STATUS_CHECKSUM_MISMATCH = 'Checksum missmatch'
def __init__(self, dvc_path, full_path, md5, hardlink_md5, state, fsck_deps):
self.dvc_path = dvc_path
self.full_path = full_path
self.md5 = md5
self.hardlink_md5 = hardlink_md5
self.state = state
self.fsck_deps = fsck_deps
self.mtime = os.path.getmtime(full_path) if os.path.exists(full_path) else None
def hardlink_msg(self):
if not self.is_data_file():
return 'Not data file - no cache needed'
elif self.hardlink_md5:
return self.hardlink_md5
return 'none'
def local_state_checksum_msg(self):
return u'{}'.format(self.state.md5 if self.state else 'none')
def state_mtime_msg(self):
return self.state.mtime if self.state else 'none'
def mtime_msg(self):
return os.path.getmtime(self.full_path) if os.path.exists(self.full_path) else 'none'
def is_data_file(self):
return any([d.is_output and d.use_cache for d in self.fsck_deps])
def checksums_mismatch(self):
md5_set = {self.md5, self.hardlink_md5}
md5_deps = set([d.md5 for d in self.fsck_deps])
md5_set = md5_set.union(md5_deps)
if None in md5_set:
md5_set.remove(None)
return len(md5_set) != 1
@property
def error_status(self):
if self.checksums_mismatch():
return self.ERR_STATUS_CHECKSUM_MISMATCH
if self.is_data_file() and self.hardlink_md5 is None:
return self.ERR_STATUS_NO_CACHE_FILE
return None
def print_info(self):
print(u'File {}:'.format(self.dvc_path))
if self.error_status:
print(u' Error status: {}!!!'.format(self.error_status))
if self.md5:
print(u' Actual checksum: {}'.format(self.md5))
print(u' Cache file name: {}'.format(self.hardlink_msg()))
print(u' Local state checksum: {}'.format(self.local_state_checksum_msg()))
print(u' Local state mtime: {}'.format(self.state_mtime_msg()))
print(u' Actual mtime: {}'.format(self.mtime_msg()))
for fsck_deps in self.fsck_deps:
print(u' DVC file: {}'.format(fsck_deps.dvc_file_name))
print(u' Checksum: {}'.format(fsck_deps.md5))
print(u' Type: {}'.format(fsck_deps.type_name))
if fsck_deps.use_cache:
print(u' Use cache: {}'.format(str(fsck_deps.use_cache).lower()))
pass
class Fsck(object):
def __init__(self, project, targets=None, physical=False, all=False):
self.project = project
self.targets = targets or []
self.physical = physical
self.all = all
self.files_and_stages = self.directions_by_datafile(self.all_directions())
self.caches = self.project.cache.find_cache(self.files_and_stages)
self.fsck_objs = self.dvc_files_to_fsck_objs()
@property
def is_target_defined(self):
return len(self.targets) > 0
def file_list(self, files_and_stages, targets):
if targets:
dvc_files = [os.path.relpath(os.path.abspath(f), self.project.root_dir)
for f in targets]
else:
dvc_files = files_and_stages.keys()
return dvc_files
def all_directions(self):
result = []
for stage in self.project.stages():
for dep in stage.deps + stage.outs:
result.append((stage, dep))
return result
def create_file_fsck_state(self, dvc_path):
full_path = os.path.join(self.project.root_dir, dvc_path)
if self.physical:
md5 = file_md5(full_path)[0]
else:
md5 = None
hardlink_md5 = self.caches.get(dvc_path)
state = self.project.state.get(dvc_path)
fsck_deps = [FsckFileDep(stage.dvc_path, dep)
for stage, dep in self.files_and_stages.get(dvc_path, [])]
return FsckFile(dvc_path, full_path, md5, hardlink_md5, state, fsck_deps)
def dvc_files_to_fsck_objs(self):
if not self.is_target_defined or self.all:
files = self.files_and_stages.keys()
else:
files = map(lambda t: os.path.relpath(os.path.abspath(t), self.project.root_dir),
self.targets)
fsck_objs = map(lambda file: self.create_file_fsck_state(file), files)
if not self.is_target_defined and not self.all:
fsck_objs = filter(lambda o: o.error_status is not None, fsck_objs)
return list(fsck_objs)
@staticmethod
def directions_by_datafile(directions):
result = {}
for stage, dep in directions:
if dep.dvc_path not in result:
result[dep.dvc_path] = []
result[dep.dvc_path].append((stage, dep))
return result
class CmdFsck(CmdBase):
def run(self):
return self.fsck(self.args.targets, self.args.physical, self.args.all)
def fsck(self, targets, physical, all):
checker = Fsck(self.project, targets, physical, all)
for file_fsck in checker.fsck_objs:
file_fsck.print_info()
return 0
-16
View File
@@ -11,7 +11,6 @@ from dvc.command.remove import CmdRemove
from dvc.command.gc import CmdGC
from dvc.command.config import CmdConfig
from dvc.command.checkout import CmdCheckout
from dvc.command.fsck import CmdFsck
from tests.basic_env import TestDvc
@@ -147,18 +146,3 @@ class TestCheckout(TestDvc):
def test(self):
args = parse_args(['checkout'])
self.assertIsInstance(args.func(args), CmdCheckout)
class TestFsck(TestDvc):
def test(self):
target1 = 'target1'
target2 = 'target2'
args = parse_args(['fsck',
'-p', '--physical',
target1,
target2])
self.assertIsInstance(args.func(args), CmdFsck)
self.assertTrue(args.physical)
self.assertEqual(args.targets, [target1, target2])
-125
View File
@@ -1,125 +0,0 @@
import os
from dvc import cli
from tests.basic_env import TestDvc
from dvc.command.fsck import Fsck, FsckFile, CmdFsck
class TestFsck(TestDvc):
def no_file_corruptions_test(self):
fsck_objs = Fsck(self.dvc).fsck_objs
self.assertEqual(len(fsck_objs), 0)
def get_all_test(self):
self.dvc.add(self.FOO)
self.dvc.add(self.BAR)
fsck_objs = Fsck(self.dvc, all=True).fsck_objs
self.assertEqual(len(fsck_objs), 2)
for obj in fsck_objs:
self.assertIsNone(obj.error_status)
def get_one_of_many_test(self):
self.dvc.add(self.FOO)
self.dvc.add(self.BAR)
fsck_objs = Fsck(self.dvc, [self.FOO]).fsck_objs
self.assertEqual(len(fsck_objs), 1)
self.assertEqual(fsck_objs[0].dvc_path, self.FOO)
def by_file_test(self):
self.dvc.add(self.FOO)
self.dvc.add(self.BAR)
fsck_objs = Fsck(self.dvc, [self.FOO, self.BAR]).fsck_objs
self.assertEqual(len(fsck_objs), 2)
for obj in fsck_objs:
self.assertIsNone(obj.error_status)
def defined_target_with_mismatch_test(self):
stage_foo = self.dvc.add(self.FOO)
self.dvc.add(self.BAR)
for out in stage_foo.outs:
if out.dvc_path == self.FOO:
out.md5 = out.md5 + '1234fe'
stage_foo.dump()
fsck_objs = Fsck(self.dvc, [self.FOO, self.BAR]).fsck_objs
self.assertEqual(len(fsck_objs), 2)
def checksum_mismatch_test(self):
stage_foo = self.dvc.add(self.FOO)
self.dvc.add(self.BAR)
for out in stage_foo.outs:
if out.dvc_path == self.FOO:
out.md5 = out.md5 + '1234fe'
stage_foo.dump()
fsck_objs = Fsck(self.dvc).fsck_objs
self.assertEqual(len(fsck_objs), 1)
self.assertEqual(fsck_objs[0].dvc_path, self.FOO)
self.assertEqual(fsck_objs[0].error_status, FsckFile.ERR_STATUS_CHECKSUM_MISMATCH)
def no_cache_file_test(self):
stage_foo = self.dvc.add(self.FOO)
self.dvc.add(self.BAR)
md5 = ''
for out in stage_foo.outs:
if out.dvc_path == self.FOO:
md5 = out.md5
foo_cache_file = os.path.join(self.dvc.cache.cache_dir, md5[0:2], md5[2:])
self.assertTrue(os.path.exists(foo_cache_file))
os.remove(foo_cache_file)
self.assertFalse(os.path.exists(foo_cache_file))
fsck_objs = Fsck(self.dvc).fsck_objs
self.assertEqual(len(fsck_objs), 1)
self.assertEqual(fsck_objs[0].dvc_path, self.FOO)
self.assertEqual(fsck_objs[0].error_status, FsckFile.ERR_STATUS_NO_CACHE_FILE)
def physical_test(self):
self.dvc.add(self.FOO)
self.dvc.add(self.BAR)
os.remove(self.FOO)
with open(self.FOO, 'w') as fd:
fd.write('randome stuff s9dfj')
fsck_objs = Fsck(self.dvc, physical=True).fsck_objs
self.assertEqual(len(fsck_objs), 1)
self.assertEqual(fsck_objs[0].dvc_path, self.FOO)
def by_timestemp_physical_test(self):
self.dvc.add(self.FOO)
self.dvc.add(self.BAR)
os.remove(self.FOO)
with open(self.FOO, 'w') as fd:
fd.write('randome stuff 39re2fwecsb nj')
fsck_objs = Fsck(self.dvc, physical=False).fsck_objs
self.assertEqual(len(fsck_objs), 1)
self.assertEqual(fsck_objs[0].dvc_path, self.FOO)
def end_to_end_test(self):
self.dvc.add(self.FOO)
self.dvc.add(self.BAR)
os.remove(self.FOO)
no_data_file = 'baz'
open(no_data_file, 'w').write('asd')
with open(self.FOO, 'w') as fd:
fd.write('randome stuff s9dfj')
parser = cli.parse_args(['fsck', self.FOO, self.BAR, no_data_file])
cmd = CmdFsck(args=parser)
cmd.run()