Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-13 23:10:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

state: optimize operations

Browse Source 
Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-09-17 10:47:16 +03:00
parent 23d00a15b9
commit 59dcdb02ea
5 changed files with 239 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_data_cloud.py
+38 -43
View File
@@ -221,55 +221,56 @@ class TestDataCloudBase(TestDvc):
info_dir = stage_dir.outs[0].dumpd()
md5_dir = info_dir['md5']
# Check status
sleep()
status = self.cloud.status([info], show_checksums=True)
self.assertEqual([(md5, STATUS_NEW)], status)
with self.cloud.project.state:
# Check status
sleep()
status = self.cloud.status([info], show_checksums=True)
self.assertEqual([(md5, STATUS_NEW)], status)
status_dir = self.cloud.status([info_dir], show_checksums=True)
self.assertTrue((md5_dir, STATUS_NEW) in status_dir)
status_dir = self.cloud.status([info_dir], show_checksums=True)
self.assertTrue((md5_dir, STATUS_NEW) in status_dir)
# Push and check status
self.cloud.push([info])
self.assertTrue(os.path.exists(cache))
self.assertTrue(os.path.isfile(cache))
# Push and check status
self.cloud.push([info])
self.assertTrue(os.path.exists(cache))
self.assertTrue(os.path.isfile(cache))
self.cloud.push([info_dir])
self.assertTrue(os.path.isfile(cache_dir))
self.cloud.push([info_dir])
self.assertTrue(os.path.isfile(cache_dir))
sleep()
status = self.cloud.status([info], show_checksums=True)
self.assertEqual([(md5, STATUS_OK)], status)
sleep()
status = self.cloud.status([info], show_checksums=True)
self.assertEqual([(md5, STATUS_OK)], status)
status_dir = self.cloud.status([info_dir], show_checksums=True)
self.assertTrue((md5_dir, STATUS_OK) in status_dir)
status_dir = self.cloud.status([info_dir], show_checksums=True)
self.assertTrue((md5_dir, STATUS_OK) in status_dir)
# Remove and check status
sleep()
shutil.rmtree(self.dvc.cache.local.cache_dir)
# Remove and check status
sleep()
shutil.rmtree(self.dvc.cache.local.cache_dir)
status = self.cloud.status([info], show_checksums=True)
self.assertEqual([(md5, STATUS_DELETED)], status)
status = self.cloud.status([info], show_checksums=True)
self.assertEqual([(md5, STATUS_DELETED)], status)
status_dir = self.cloud.status([info_dir], show_checksums=True)
self.assertEqual([(md5_dir, STATUS_DELETED)], status_dir)
status_dir = self.cloud.status([info_dir], show_checksums=True)
self.assertEqual([(md5_dir, STATUS_DELETED)], status_dir)
# Pull and check status
self.cloud.pull([info])
self.assertTrue(os.path.exists(cache))
self.assertTrue(os.path.isfile(cache))
with open(cache, 'r') as fd:
self.assertEqual(fd.read(), self.FOO_CONTENTS)
# Pull and check status
self.cloud.pull([info])
self.assertTrue(os.path.exists(cache))
self.assertTrue(os.path.isfile(cache))
with open(cache, 'r') as fd:
self.assertEqual(fd.read(), self.FOO_CONTENTS)
self.cloud.pull([info_dir])
self.assertTrue(os.path.isfile(cache_dir))
self.cloud.pull([info_dir])
self.assertTrue(os.path.isfile(cache_dir))
sleep()
status = self.cloud.status([info], show_checksums=True)
self.assertEqual([(md5, STATUS_OK)], status)
sleep()
status = self.cloud.status([info], show_checksums=True)
self.assertEqual([(md5, STATUS_OK)], status)
status_dir = self.cloud.status([info_dir], show_checksums=True)
self.assertTrue((md5_dir, STATUS_OK) in status_dir)
status_dir = self.cloud.status([info_dir], show_checksums=True)
self.assertTrue((md5_dir, STATUS_OK) in status_dir)
def test(self):
self._ensure_should_run()
@@ -335,9 +336,6 @@ class TestRemoteSSH(TestDataCloudBase):
def _get_cloud_class(self):
return RemoteSSH
def test(self):
super(TestRemoteSSH, self).test()
class TestRemoteHDFS(TestDataCloudBase):
def _should_test(self):
@@ -349,9 +347,6 @@ class TestRemoteHDFS(TestDataCloudBase):
def _get_cloud_class(self):
return RemoteHDFS
def test(self):
super(TestRemoteHDFS, self).test()
class TestDataCloudCLIBase(TestDvc):
def main(self, args):
tests/test_state.py
+17 -16
View File
@@ -18,23 +18,24 @@ class TestState(TestDvc):
state = State(self.dvc)
entry_md5 = state.update(path)
self.assertEqual(entry_md5, md5)
with state:
entry_md5 = state.update(path)
self.assertEqual(entry_md5, md5)
# Sleep some time to simulate realistic behavior.
# Some filesystems have a bad date resolution for
# mtime(i.e. 1sec for HFS) that cause problems with
# our 'state' system not being able to distinguish
# files that were modified within that delta.
time.sleep(1)
# Sleep some time to simulate realistic behavior.
# Some filesystems have a bad date resolution for
# mtime(i.e. 1sec for HFS) that cause problems with
# our 'state' system not being able to distinguish
# files that were modified within that delta.
time.sleep(1)
os.unlink(path)
with open(path, 'w+') as fd:
fd.write('1')
os.unlink(path)
with open(path, 'w+') as fd:
fd.write('1')
md5 = file_md5(path)[0]
mtime = os.path.getmtime(path)
inode = System.inode(path)
md5 = file_md5(path)[0]
mtime = os.path.getmtime(path)
inode = System.inode(path)
entry_md5 = state.update(path)
self.assertEqual(entry_md5, md5)
entry_md5 = state.update(path)
self.assertEqual(entry_md5, md5)