mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
+17
-11
@@ -41,13 +41,10 @@ class OutputLOCAL(DependencyLOCAL):
|
||||
return ret
|
||||
|
||||
def changed(self):
|
||||
if super(OutputLOCAL, self).changed():
|
||||
return True
|
||||
if not self.use_cache:
|
||||
return super(OutputLOCAL, self).changed()
|
||||
|
||||
if self.use_cache and self.info != self.project.cache.local.save_info(self.path_info):
|
||||
return True
|
||||
|
||||
return False
|
||||
return self.info != self.project.cache.local.save_info(self.path_info)
|
||||
|
||||
def checkout(self):
|
||||
if not self.use_cache:
|
||||
@@ -61,15 +58,24 @@ class OutputLOCAL(DependencyLOCAL):
|
||||
self.project.cache.local.checkout(self.path_info, self.info)
|
||||
|
||||
def save(self):
|
||||
if not self.changed():
|
||||
msg = 'Output \'{}\' didn\'t change. Skipping saving.'
|
||||
if not self.use_cache:
|
||||
super(OutputLOCAL, self).save()
|
||||
msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.'
|
||||
self.project.logger.debug(msg.format(self.rel_path))
|
||||
return
|
||||
|
||||
super(OutputLOCAL, self).save()
|
||||
if not os.path.exists(self.path):
|
||||
raise self.DoesNotExistError(self.rel_path)
|
||||
|
||||
if not self.use_cache:
|
||||
msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.'
|
||||
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
|
||||
raise self.NotFileOrDirError(self.rel_path)
|
||||
|
||||
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
|
||||
(os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
|
||||
self.project.logger.warn("File/directory '{}' is empty.".format(self.rel_path))
|
||||
|
||||
if not self.changed():
|
||||
msg = 'Output \'{}\' didn\'t change. Skipping saving.'
|
||||
self.project.logger.debug(msg.format(self.rel_path))
|
||||
return
|
||||
|
||||
|
||||
+17
-4
@@ -10,7 +10,7 @@ from dvc.logger import Logger
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.stage import Stage
|
||||
from dvc.config import Config
|
||||
from dvc.state import LinkState
|
||||
from dvc.state import LinkState, State
|
||||
from dvc.lock import Lock
|
||||
from dvc.scm import SCM, Base
|
||||
from dvc.cache import Cache
|
||||
@@ -50,7 +50,10 @@ class Project(object):
|
||||
self.config = Config(self.dvc_dir)
|
||||
self.scm = SCM(self.root_dir)
|
||||
self.lock = Lock(self.dvc_dir)
|
||||
self.link_state = LinkState(self.root_dir, self.dvc_dir)
|
||||
# NOTE: storing state and link_state in the repository itself to avoid
|
||||
# any possible state corruption in 'shared cache dir' scenario.
|
||||
self.state = State(self)
|
||||
self.link_state = LinkState(self)
|
||||
self.logger = Logger(self.config._config[Config.SECTION_CORE].get(Config.SECTION_CORE_LOGLEVEL, None))
|
||||
self.cache = Cache(self)
|
||||
self.cloud = DataCloud(self, config=self.config._config)
|
||||
@@ -241,14 +244,24 @@ class Project(object):
|
||||
raise ReproductionError(stages[n].relpath, ex)
|
||||
return result
|
||||
|
||||
def _cleanup_unused_links(self, all_stages):
|
||||
used = []
|
||||
for stage in all_stages:
|
||||
for out in stage.outs:
|
||||
used.append(out.path)
|
||||
self.link_state.remove_unused(used)
|
||||
|
||||
def checkout(self, target=None):
|
||||
all_stages = self.stages()
|
||||
|
||||
if target:
|
||||
if not Stage.is_stage_file(target):
|
||||
raise NotDvcFileError(target)
|
||||
stages = [Stage.load(self, target)]
|
||||
else:
|
||||
self.link_state.remove_all()
|
||||
stages = self.stages()
|
||||
stages = all_stages
|
||||
|
||||
self._cleanup_unused_links(all_stages)
|
||||
|
||||
for stage in stages:
|
||||
stage.checkout()
|
||||
|
||||
+66
-48
@@ -31,6 +31,7 @@ class RemoteLOCAL(RemoteBase):
|
||||
|
||||
def __init__(self, project, config):
|
||||
self.project = project
|
||||
self.state = self.project.state
|
||||
self.link_state = project.link_state
|
||||
storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
|
||||
self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
|
||||
@@ -46,8 +47,6 @@ class RemoteLOCAL(RemoteBase):
|
||||
if self.cache_dir != None and not os.path.exists(self.cache_dir):
|
||||
os.mkdir(self.cache_dir)
|
||||
|
||||
self.state = State(self.cache_dir)
|
||||
|
||||
@property
|
||||
def prefix(self):
|
||||
return self.cache_dir
|
||||
@@ -85,36 +84,32 @@ class RemoteLOCAL(RemoteBase):
|
||||
|
||||
return False
|
||||
|
||||
def link(self, md5, path, dump=True):
|
||||
cache = self.get(md5)
|
||||
if not cache or not os.path.exists(cache) or self.changed(md5):
|
||||
if cache:
|
||||
msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.'
|
||||
Logger.warn(msg.format(md5, os.path.relpath(path)))
|
||||
return
|
||||
def link(self, cache, path):
|
||||
assert os.path.isfile(cache)
|
||||
|
||||
dname = os.path.dirname(path)
|
||||
if not os.path.exists(dname):
|
||||
os.makedirs(dname)
|
||||
|
||||
if self.cache_types != None:
|
||||
types = self.cache_types
|
||||
else:
|
||||
types = self.CACHE_TYPES
|
||||
|
||||
for typ in types:
|
||||
i = 0
|
||||
N = len(self.cache_types)
|
||||
while i < N:
|
||||
try:
|
||||
self.CACHE_TYPE_MAP[typ](cache, path)
|
||||
self.link_state.update(path, dump=dump)
|
||||
self.CACHE_TYPE_MAP[self.cache_types[i]](cache, path)
|
||||
return
|
||||
except Exception as exc:
|
||||
msg = 'Cache type \'{}\' is not supported'.format(typ)
|
||||
msg = 'Cache type \'{}\' is not supported'.format(self.cache_types[i])
|
||||
Logger.debug(msg)
|
||||
if typ == types[-1]:
|
||||
raise DvcException(msg, cause=exc)
|
||||
del self.cache_types[i]
|
||||
i += 1
|
||||
|
||||
raise DvcException('No possible cache types left to try out.')
|
||||
|
||||
def load_dir_cache(self, md5):
|
||||
path = self.get(md5)
|
||||
|
||||
assert self.is_dir_cache(path)
|
||||
|
||||
@staticmethod
|
||||
def load_dir_cache(path):
|
||||
try:
|
||||
with open(path, 'r') as fd:
|
||||
d = json.load(fd)
|
||||
@@ -130,6 +125,23 @@ class RemoteLOCAL(RemoteBase):
|
||||
|
||||
return d
|
||||
|
||||
def dump_dir_cache(self, md5, dir_info):
|
||||
path = self.get(md5)
|
||||
dname = os.path.dirname(path)
|
||||
|
||||
assert self.is_dir_cache(path)
|
||||
assert isinstance(dir_info, list)
|
||||
|
||||
if not os.path.isdir(dname):
|
||||
os.makedirs(dname)
|
||||
|
||||
# NOTE: Writing first and renaming after that
|
||||
# to make sure that the operation is atomic.
|
||||
tmp = '{}.{}'.format(path, str(uuid.uuid4()))
|
||||
with open(tmp, 'w+') as fd:
|
||||
json.dump(dir_info, fd, sort_keys=True)
|
||||
move(tmp, path)
|
||||
|
||||
@staticmethod
|
||||
def is_dir_cache(cache):
|
||||
return cache.endswith(State.MD5_DIR_SUFFIX)
|
||||
@@ -143,6 +155,12 @@ class RemoteLOCAL(RemoteBase):
|
||||
Logger.warn('No cache info for \'{}\'. Skipping checkout.'.format(os.path.relpath(path)))
|
||||
return
|
||||
|
||||
if self.changed(md5):
|
||||
msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.'
|
||||
Logger.warn(msg.format(md5, os.path.relpath(path)))
|
||||
remove(path)
|
||||
return
|
||||
|
||||
if os.path.exists(path):
|
||||
msg = u'Data \'{}\' exists. Removing before checkout'
|
||||
Logger.debug(msg.format(os.path.relpath(path)))
|
||||
@@ -152,7 +170,8 @@ class RemoteLOCAL(RemoteBase):
|
||||
Logger.debug(msg.format(os.path.relpath(path), md5))
|
||||
|
||||
if not self.is_dir_cache(cache):
|
||||
self.link(md5, path, dump=True)
|
||||
self.link(cache, path)
|
||||
self.link_state.update(path)
|
||||
return
|
||||
|
||||
# Create dir separately so that dir is created
|
||||
@@ -160,12 +179,13 @@ class RemoteLOCAL(RemoteBase):
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
for entry in self.load_dir_cache(cache):
|
||||
for entry in self.load_dir_cache(md5):
|
||||
md5 = entry[self.PARAM_MD5]
|
||||
c = self.get(md5)
|
||||
relpath = entry[self.PARAM_RELPATH]
|
||||
p = os.path.join(path, relpath)
|
||||
self.link(md5, p, dump=False)
|
||||
self.link_state.dump()
|
||||
self.link(c, p)
|
||||
self.link_state.update(path)
|
||||
|
||||
def _move(self, inp, outp):
|
||||
# moving in two stages to make last the move atomic in
|
||||
@@ -177,36 +197,38 @@ class RemoteLOCAL(RemoteBase):
|
||||
def _save_file(self, path_info):
|
||||
path = path_info['path']
|
||||
md5 = self.state.update(path)
|
||||
assert md5 != None
|
||||
|
||||
cache = self.get(md5)
|
||||
|
||||
if self.changed(md5):
|
||||
Logger.debug(u'Saving \'{}\' to \'{}\''.format(os.path.relpath(path),
|
||||
os.path.relpath(cache)))
|
||||
self._move(path, cache)
|
||||
self.state.update(cache)
|
||||
else:
|
||||
remove(path)
|
||||
|
||||
self.link(cache, path)
|
||||
self.link_state.update(path)
|
||||
|
||||
return {self.PARAM_MD5: md5}
|
||||
|
||||
def _save_dir(self, path_info):
|
||||
path = path_info['path']
|
||||
md5 = self.state.update(path)
|
||||
cache = self.get(md5)
|
||||
dname = os.path.dirname(cache)
|
||||
dir_info = self.state.collect_dir(path)
|
||||
md5, dir_info = self.state.update_info(path)
|
||||
|
||||
for entry in dir_info:
|
||||
relpath = entry[State.PARAM_RELPATH]
|
||||
m = entry[State.PARAM_MD5]
|
||||
p = os.path.join(path, relpath)
|
||||
c = self.get(m)
|
||||
|
||||
self._save_file({'scheme': 'local', 'path': p})
|
||||
if self.changed(m):
|
||||
self._move(p, c)
|
||||
else:
|
||||
remove(p)
|
||||
|
||||
if not os.path.isdir(dname):
|
||||
os.makedirs(dname)
|
||||
self.link(c, p)
|
||||
|
||||
Logger.debug(u'Saving directory \'{}\' to \'{}\''.format(os.path.relpath(path),
|
||||
os.path.relpath(cache)))
|
||||
|
||||
with open(cache, 'w+') as fd:
|
||||
json.dump(dir_info, fd, sort_keys=True)
|
||||
self.link_state.update(path)
|
||||
|
||||
return {self.PARAM_MD5: md5}
|
||||
|
||||
@@ -217,13 +239,9 @@ class RemoteLOCAL(RemoteBase):
|
||||
path = path_info['path']
|
||||
|
||||
if os.path.isdir(path):
|
||||
checksum_info = self._save_dir(path_info)
|
||||
return self._save_dir(path_info)
|
||||
else:
|
||||
checksum_info = self._save_file(path_info)
|
||||
|
||||
self.checkout(path_info, checksum_info)
|
||||
|
||||
return checksum_info
|
||||
return self._save_file(path_info)
|
||||
|
||||
def save_info(self, path_info):
|
||||
if path_info['scheme'] != 'local':
|
||||
@@ -318,7 +336,7 @@ class RemoteLOCAL(RemoteBase):
|
||||
continue
|
||||
if not os.path.exists(cache):
|
||||
missing.append(info)
|
||||
collected.extend(self.load_dir_cache(cache))
|
||||
collected.extend(self.load_dir_cache(md5))
|
||||
collected.extend(checksum_infos)
|
||||
return collected, missing
|
||||
|
||||
|
||||
+49
-29
@@ -48,22 +48,23 @@ class State(object):
|
||||
PARAM_MD5 = 'md5'
|
||||
MD5_DIR_SUFFIX = '.dir'
|
||||
|
||||
def __init__(self, dvc_dir):
|
||||
self.dvc_dir = dvc_dir
|
||||
def __init__(self, project):
|
||||
self.project = project
|
||||
self.dvc_dir = project.dvc_dir
|
||||
self._lock = threading.Lock()
|
||||
if dvc_dir:
|
||||
self.state_file = os.path.join(dvc_dir, self.STATE_FILE)
|
||||
self._lock_file = Lock(dvc_dir, self.STATE_LOCK_FILE)
|
||||
if self.dvc_dir:
|
||||
self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
|
||||
self._lock_file = Lock(self.dvc_dir, self.STATE_LOCK_FILE)
|
||||
else:
|
||||
self.state_file = None
|
||||
self._lock_file = threading.Lock()
|
||||
self._db = self.load()
|
||||
|
||||
@staticmethod
|
||||
def init(dvc_dir):
|
||||
return State(dvc_dir)
|
||||
def init(project):
|
||||
return State(project)
|
||||
|
||||
def collect_dir(self, dname):
|
||||
def _collect_dir(self, dname):
|
||||
dir_info = []
|
||||
|
||||
for root, dirs, files in os.walk(dname):
|
||||
@@ -71,19 +72,25 @@ class State(object):
|
||||
path = os.path.join(root, fname)
|
||||
relpath = os.path.relpath(path, dname)
|
||||
|
||||
md5 = self.update(path, dump=False)
|
||||
# FIXME: we could've used `md5 = self.update(path, dump=False)` here,
|
||||
# but it is around twice as slow(on ssd, don't know about hdd) for a
|
||||
# directory with small files. What we could do here is introduce some
|
||||
# kind of a limit for file size, after which we would actually register
|
||||
# it in our state file.
|
||||
md5 = file_md5(path)[0]
|
||||
dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5})
|
||||
|
||||
self.dump()
|
||||
md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
|
||||
if self.project.cache.local.changed(md5):
|
||||
self.project.cache.local.dump_dir_cache(md5, dir_info)
|
||||
|
||||
return dir_info
|
||||
return (md5, dir_info)
|
||||
|
||||
def compute_md5(self, path):
|
||||
def _collect(self, path):
|
||||
if os.path.isdir(path):
|
||||
dir_info = self.collect_dir(path)
|
||||
return dict_md5(dir_info) + self.MD5_DIR_SUFFIX
|
||||
return self._collect_dir(path)
|
||||
else:
|
||||
return file_md5(path)[0]
|
||||
return (file_md5(path)[0], None)
|
||||
|
||||
def changed(self, path, md5):
|
||||
actual = self.update(path)
|
||||
@@ -120,18 +127,18 @@ class State(object):
|
||||
def inode(path):
|
||||
return str(System.inode(path))
|
||||
|
||||
def update(self, path, dump=True):
|
||||
def _do_update(self, path, dump=True):
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
return (None, None)
|
||||
|
||||
mtime = self.mtime(path)
|
||||
inode = self.inode(path)
|
||||
|
||||
md5 = self._get(inode, mtime)
|
||||
if md5:
|
||||
return md5
|
||||
return (md5, None)
|
||||
|
||||
md5 = self.compute_md5(path)
|
||||
md5, info = self._collect(path)
|
||||
state = StateEntry(md5, mtime)
|
||||
d = state.dumpd()
|
||||
|
||||
@@ -142,7 +149,16 @@ class State(object):
|
||||
if dump:
|
||||
self.dump()
|
||||
|
||||
return md5
|
||||
return (md5, info)
|
||||
|
||||
def update(self, path, dump=True):
|
||||
return self._do_update(path, dump=dump)[0]
|
||||
|
||||
def update_info(self, path, dump=True):
|
||||
md5, info = self._do_update(path, dump=dump)
|
||||
if not info:
|
||||
info = self.project.cache.local.load_dir_cache(md5)
|
||||
return (md5, info)
|
||||
|
||||
def _get(self, inode, mtime):
|
||||
with self._lock:
|
||||
@@ -190,9 +206,9 @@ class LinkState(State):
|
||||
STATE_FILE = 'link.state'
|
||||
STATE_LOCK_FILE = STATE_FILE + '.lock'
|
||||
|
||||
def __init__(self, root_dir, dvc_dir):
|
||||
super(LinkState, self).__init__(dvc_dir)
|
||||
self.root_dir = root_dir
|
||||
def __init__(self, project):
|
||||
super(LinkState, self).__init__(project)
|
||||
self.root_dir = project.root_dir
|
||||
|
||||
def update(self, path, dump=True):
|
||||
if not os.path.exists(path):
|
||||
@@ -209,11 +225,15 @@ class LinkState(State):
|
||||
with self._lock_file:
|
||||
self.dump()
|
||||
|
||||
def _do_remove_all(self):
|
||||
for p, s in self._db.items():
|
||||
def _do_remove_unused(self, used):
|
||||
items = self._db.copy().items()
|
||||
for p, s in items:
|
||||
path = os.path.join(self.root_dir, p)
|
||||
state = LinkStateEntry.loadd(s)
|
||||
|
||||
if path in used:
|
||||
continue
|
||||
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
|
||||
@@ -221,12 +241,12 @@ class LinkState(State):
|
||||
mtime = self.mtime(path)
|
||||
|
||||
if inode == state.inode and mtime == state.mtime:
|
||||
Logger.debug('Removing \'{}\' as unused link.'.format(path))
|
||||
remove(path)
|
||||
del self._db[p]
|
||||
|
||||
self._db = {}
|
||||
|
||||
def remove_all(self):
|
||||
def remove_unused(self, used):
|
||||
with self._lock:
|
||||
with self._lock_file:
|
||||
self._do_remove_all()
|
||||
self._do_remove_unused(used)
|
||||
self.dump()
|
||||
|
||||
+4
-4
@@ -41,13 +41,13 @@ class TestCacheLoadBadDirCache(TestDvc):
|
||||
self.assertEqual(len(ret), 0)
|
||||
|
||||
def test(self):
|
||||
fname = 'not-json'
|
||||
fname = self.dvc.cache.local.get('123.dir')
|
||||
self.create(fname, '<clearly>not,json')
|
||||
self._do_test(RemoteLOCAL.load_dir_cache(fname))
|
||||
self._do_test(self.dvc.cache.local.load_dir_cache(fname))
|
||||
|
||||
fname = 'not-list'
|
||||
fname = self.dvc.cache.local.get('234.dir')
|
||||
self.create(fname, '{"a": "b"}')
|
||||
self._do_test(RemoteLOCAL.load_dir_cache(fname))
|
||||
self._do_test(self.dvc.cache.local.load_dir_cache(fname))
|
||||
|
||||
|
||||
class TestExternalCacheDir(TestDvc):
|
||||
|
||||
+2
-2
@@ -16,7 +16,7 @@ class TestState(TestDvc):
|
||||
mtime = os.path.getmtime(path)
|
||||
inode = System.inode(path)
|
||||
|
||||
state = State(self.dvc.dvc_dir)
|
||||
state = State(self.dvc)
|
||||
|
||||
entry_md5 = state.update(path)
|
||||
self.assertEqual(entry_md5, md5)
|
||||
@@ -41,6 +41,6 @@ class TestState(TestDvc):
|
||||
|
||||
# Reload state db to make sure that it stays the same
|
||||
old_db = state._db
|
||||
state = State(self.dvc.dvc_dir)
|
||||
state = State(self.dvc)
|
||||
new_db = state._db
|
||||
self.assertEqual(old_db, new_db)
|
||||
|
||||
Reference in New Issue
Block a user