Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-10 07:18:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #499 from efiop/master

Browse Source 
dvc: use two-level hash for cache
This commit is contained in:
Ruslan Kuprieiev
2018-03-02 03:15:32 +02:00
committed by GitHub
parent 45e302a92a 539ac41b70
commit 13892e3428
10 changed files with 40 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cache.py
+16 -5
View File
@@ -11,29 +11,40 @@ class Cache(object):
if not os.path.exists(self.cache_dir):
os.mkdir(self.cache_dir)
for subdir in self._cache_subdirs:
if not os.path.exists(subdir):
os.mkdir(subdir)
@staticmethod
def init(dvc_dir):
cache_dir = os.path.join(dvc_dir, Cache.CACHE_DIR)
os.mkdir(cache_dir)
return Cache(dvc_dir)
@property
def _cache_subdirs(self):
dirs = []
for x in range(0x00, 0xFF):
dirs.append(os.path.join(self.cache_dir, format(x, '02x')))
return dirs
def all(self):
clist = []
for cache in os.listdir(self.cache_dir):
path = os.path.join(self.cache_dir, cache)
if os.path.isfile(path):
for subdir in self._cache_subdirs:
for cache in os.listdir(subdir):
path = os.path.join(subdir, cache)
clist.append(path)
return clist
def get(self, md5):
return os.path.join(self.cache_dir, md5)
return os.path.join(self.cache_dir, md5[0:2], md5[2:])
def find_cache(self, files):
file_set = set(files)
cached = {}
for cache_file in self.all():
cached_files = list(filter(lambda f: System.samefile(cache_file, f), file_set))
cached.update(dict((f, os.path.basename(cache_file)) for f in cached_files))
cached.update(dict((f, os.path.basename(os.path.dirname(cache_file)) + os.path.basename(cache_file)) for f in cached_files))
file_set = file_set - set(cached_files)
return cached
dvc/cloud/base.py
+4 -4
View File
@@ -75,7 +75,7 @@ class DataCloudBase(object):
def cache_file_key(self, fname):
""" Key of a file within the bucket """
relpath = os.path.relpath(fname, self._cloud_settings.cache_dir)
relpath = os.path.relpath(fname, self._cloud_settings.cache.cache_dir)
relpath.replace('\\', '/')
return '{}/{}'.format(self.storage_prefix, relpath).strip('/')
@@ -107,7 +107,7 @@ class DataCloudBase(object):
with open(p, 'r') as fd:
d = yaml.safe_load(fd)
md5 = d[Output.PARAM_MD5]
cache = os.path.join(self._cloud_settings.cache_dir, md5)
cache = self._cloud_settings.cache.get(md5)
res.append(self._push(cache))
return res
@@ -159,7 +159,7 @@ class DataCloudBase(object):
d = yaml.safe_load(fd)
md5 = d[Output.PARAM_MD5]
cache = os.path.join(self._cloud_settings.cache_dir, md5)
cache = self._cloud_settings.cache.get(md5)
cache_key = self._get_key(cache)
if not cache_key:
Logger.error("File '{}' does not exist in the cloud".format(path))
@@ -202,7 +202,7 @@ class DataCloudBase(object):
d = yaml.safe_load(fd)
md5 = d[Output.PARAM_MD5]
cache = os.path.join(self._cloud_settings.cache_dir, md5)
cache = self._cloud_settings.cache.get(md5)
cache_key = self._get_key(cache)
res.append(self._status(cache_key, cache))
dvc/cloud/data_cloud.py
+4 -4
View File
@@ -24,7 +24,7 @@ class DataCloud(object):
'' : 'LOCAL',
}
def __init__(self, cache_dir, config):
def __init__(self, cache, config):
self._config = config
cloud_type = self._config['Global'].get('Cloud', '').strip().upper()
@@ -34,7 +34,7 @@ class DataCloud(object):
if cloud_type not in self._config.keys():
raise ConfigError('Can\'t find cloud section \'[%s]\' in config' % cloud_type)
cloud_settings = self.get_cloud_settings(cache_dir,
cloud_settings = self.get_cloud_settings(cache,
self._config,
cloud_type)
@@ -44,7 +44,7 @@ class DataCloud(object):
self.sanity_check()
@staticmethod
def get_cloud_settings(cache_dir, config, cloud_type):
def get_cloud_settings(cache, config, cloud_type):
"""
Obtain cloud settings from config.
"""
@@ -53,7 +53,7 @@ class DataCloud(object):
else:
cloud_config = config[cloud_type]
global_storage_path = config['Global'].get('StoragePath', None)
cloud_settings = CloudSettings(cache_dir, global_storage_path, cloud_config)
cloud_settings = CloudSettings(cache, global_storage_path, cloud_config)
return cloud_settings
def sanity_check(self):
dvc/cloud/instance_manager.py
+2 -2
View File
@@ -1,5 +1,5 @@
class CloudSettings(object):
def __init__(self, cache_dir, global_storage_path, cloud_config):
self.cache_dir = cache_dir
def __init__(self, cache, global_storage_path, cloud_config):
self.cache = cache
self.cloud_config = cloud_config
self.global_storage_path = global_storage_path
dvc/cloud/local.py
+1 -1
View File
@@ -24,7 +24,7 @@ class DataCloudLOCAL(DataCloudBase):
Driver for local storage.
"""
def cache_file_key(self, path):
return os.path.relpath(os.path.abspath(path), self._cloud_settings.cache_dir)
return os.path.relpath(os.path.abspath(path), self._cloud_settings.cache.cache_dir)
def _cmp_checksum(self, key, path):
return filecmp.cmp(path, key.path)
dvc/project.py
+1 -1
View File
@@ -38,7 +38,7 @@ class Project(object):
self.state = State(self.root_dir, self.dvc_dir)
self.config = Config(self.dvc_dir)
self.logger = Logger(self.config._config)
self.cloud = DataCloud(self.cache.cache_dir, self.config._config)
self.cloud = DataCloud(self.cache, self.config._config)
@staticmethod
def init(root_dir=os.curdir):
tests/test_cache.py
+7 -5
View File
@@ -9,8 +9,10 @@ from tests.basic_env import TestDvc
class TestCache(TestDvc):
def setUp(self):
super(TestCache, self).setUp()
self.cache1 = os.path.join(self.dvc.cache.cache_dir, '1')
self.cache2 = os.path.join(self.dvc.cache.cache_dir, '2')
self.cache1_md5 = '123'
self.cache2_md5 = '234'
self.cache1 = os.path.join(self.dvc.cache.cache_dir, self.cache1_md5[0:2], self.cache1_md5[2:])
self.cache2 = os.path.join(self.dvc.cache.cache_dir, self.cache2_md5[0:2], self.cache2_md5[2:])
self.create(self.cache1, '1')
self.create(self.cache2, '2')
@@ -21,14 +23,14 @@ class TestCache(TestDvc):
self.assertTrue(self.cache2 in flist)
def test_get(self):
cache = Cache(self.dvc.dvc_dir).get(os.path.basename(self.cache1))
cache = Cache(self.dvc.dvc_dir).get(self.cache1_md5)
self.assertEquals(cache, self.cache1)
def test_find_cache(self):
fname1 = os.path.basename(self.cache1)
fname1_md5 = fname1
fname1_md5 = self.cache1_md5
fname2 = os.path.basename(self.cache2)
fname2_md5 = fname2
fname2_md5 = self.cache2_md5
fname3 = 'non_existing'
System.hardlink(self.cache1, fname1)
tests/test_data_cloud.py
+2 -2
View File
@@ -154,7 +154,7 @@ class TestDataCloudGCP(TestDataCloudBase):
# Setup cloud
config = {'StoragePath': repo,
'Region': self.TEST_REPO_GCP_PROJECT}
cloud_settings = CloudSettings(self.dvc.cache.cache_dir, None, config)
cloud_settings = CloudSettings(self.dvc.cache, None, config)
self.cloud = DataCloudGCP(cloud_settings)
def test(self):
@@ -172,7 +172,7 @@ class TestDataCloudLOCAL(TestDataCloudBase):
self.dname = 'cloud'
os.mkdir(self.dname)
config = {'StoragePath': self.dname}
cloud_settings = CloudSettings(self.dvc.cache.cache_dir, None, config)
cloud_settings = CloudSettings(self.dvc.cache, None, config)
self.cloud = DataCloudLOCAL(cloud_settings)
def test(self):
tests/test_fsck.py
+1 -1
View File
@@ -71,7 +71,7 @@ class TestFsck(TestDvc):
if out.dvc_path == self.FOO:
md5 = out.md5
foo_cache_file = os.path.join(self.dvc.cache.cache_dir, md5)
foo_cache_file = os.path.join(self.dvc.cache.cache_dir, md5[0:2], md5[2:])
self.assertTrue(os.path.exists(foo_cache_file))
os.chmod(foo_cache_file, 0o777)
tests/test_gc.py
+2 -2
View File
@@ -21,8 +21,8 @@ class TestGC(TestDvc):
self.good_cache = self.dvc.cache.all()
self.bad_cache = []
for i in ['1', '2', '3']:
path = os.path.join(self.dvc.cache.cache_dir, i)
for i in ['123', '234', '345']:
path = os.path.join(self.dvc.cache.cache_dir, i[0:2], i[2:])
self.create(path, i)
self.bad_cache.append(path)