Files
dvc/dvc/state.py
T
Ruslan Kuprieiev 0a049c232b cache: configurable cache dir and cache type
From now on we allow specifying standalone cache dirs as well as
specifying linking method(copy, symlink, hardlink, reflink) for it.
If 'cache type' is not specified in the config, dvc will try all of
them starting from reflink all the way down to copy.

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-04-28 21:50:49 +03:00

152 lines
3.4 KiB
Python

import os
import json
import nanotime
import threading
from dvc.lock import Lock
from dvc.system import System
from dvc.utils import file_md5, dict_md5
from dvc.exceptions import DvcException
class StateEntry(object):
PARAM_MTIME = 'mtime'
PARAM_MD5 = 'md5'
def __init__(self, md5, mtime):
self.mtime = mtime
self.md5 = md5
def update(self, md5, mtime):
self.mtime = mtime
self.md5 = md5
@staticmethod
def loadd(d):
mtime = d[StateEntry.PARAM_MTIME]
md5 = d[StateEntry.PARAM_MD5]
return StateEntry(md5, mtime)
def dumpd(self):
return {
self.PARAM_MD5: self.md5,
self.PARAM_MTIME: self.mtime,
}
class StateDuplicateError(DvcException):
pass
class State(object):
STATE_FILE = 'state'
STATE_LOCK_FILE = 'state.lock'
PARAM_RELPATH = 'relpath'
PARAM_MD5 = 'md5'
MD5_DIR_SUFFIX = '.dir'
def __init__(self, dvc_dir):
self.dvc_dir = dvc_dir
self.state_file = os.path.join(dvc_dir, self.STATE_FILE)
self._db = self.load()
self._lock = threading.Lock()
self._lock_file = Lock(dvc_dir, self.STATE_LOCK_FILE)
@staticmethod
def init(dvc_dir):
return State(dvc_dir)
def collect_dir(self, dname):
dir_info = []
for root, dirs, files in os.walk(dname):
for fname in files:
path = os.path.join(root, fname)
relpath = os.path.relpath(path, dname)
md5 = self.update(path, dump=False)
dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5})
self.dump()
return dir_info
def compute_md5(self, path):
if os.path.isdir(path):
dir_info = self.collect_dir(path)
return dict_md5(dir_info) + self.MD5_DIR_SUFFIX
else:
return file_md5(path)[0]
def changed(self, path, md5):
actual = self.update(path)
if not md5 or not actual:
return True
return actual.split('.')[0] != md5.split('.')[0]
def load(self):
if not os.path.isfile(self.state_file):
return {}
with open(self.state_file, 'r') as fd:
return json.load(fd)
def dump(self):
with open(self.state_file, 'w+') as fd:
json.dump(self._db, fd)
@staticmethod
def mtime(path):
return str(int(nanotime.timestamp(os.path.getmtime(path))))
@staticmethod
def inode(path):
return str(System.inode(path))
def update(self, path, dump=True):
if not os.path.exists(path):
return None
mtime = self.mtime(path)
inode = self.inode(path)
md5 = self._get(inode, mtime)
if md5:
return md5
md5 = self.compute_md5(path)
state = StateEntry(md5, mtime)
d = state.dumpd()
with self._lock:
with self._lock_file:
self._db[inode] = d
if dump:
self.dump()
return md5
def _get(self, inode, mtime):
with self._lock:
with self._lock_file:
d = self._db.get(inode, None)
if not d:
return None
state = StateEntry.loadd(d)
if mtime == state.mtime:
return state.md5
return None
def get(self, path):
mtime = self.mtime(path)
inode = self.inode(path)
return self._get(inode, mtime)