diff --git a/dvc/cloud/local.py b/dvc/cloud/local.py index 5a606a7bb..880937129 100644 --- a/dvc/cloud/local.py +++ b/dvc/cloud/local.py @@ -15,9 +15,6 @@ class LocalKey(object): def path(self): return os.path.join(self.bucket, self.name) - def delete(self): - os.unlink(self.path) - class DataCloudLOCAL(DataCloudBase): """ @@ -52,7 +49,7 @@ class DataCloudLOCAL(DataCloudBase): tmp_file = self.tmp_file(path) try: - copyfile(key.path, tmp_file) + copyfile(key.path, tmp_file, no_progress_bar=no_progress_bar) except Exception as exc: Logger.error('Failed to copy "{}": {}'.format(key.path, exc)) return None diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index cab8201ef..b29b77eda 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -8,25 +8,5 @@ from dvc.logger import Logger class CmdCheckout(CmdBase): def run(self): - self.remove_untracked_hardlinks() self.project.checkout() return 0 - - def remove_untracked_hardlinks(self): - untracked = self.project.scm.untracked_files() - cache = dict((System.inode(c), c) for c in self.project.cache.all()) - for file in untracked: - inode = System.inode(file) - if inode not in cache.keys(): - continue - - Logger.info(u'Remove \'{}\''.format(file)) - os.chmod(file, stat.S_IWRITE) - os.remove(file) - os.chmod(cache[inode], stat.S_IREAD) - - dir = os.path.dirname(file) - if len(dir) != 0 and not os.listdir(dir): - Logger.info(u'Remove empty directory \'{}\''.format(dir)) - os.removedirs(dir) - pass diff --git a/dvc/progress.py b/dvc/progress.py index a6652911e..64ea7ed95 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -13,6 +13,7 @@ class Progress(object): def set_n_total(self, total): self._n_total = total + self._n_finished = 0 def _clearln(self): print('\r\x1b[K', end='') diff --git a/dvc/project.py b/dvc/project.py index dc5478849..7028e5634 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -11,6 +11,7 @@ from dvc.lock import Lock from dvc.scm import SCM from dvc.cache import Cache from dvc.cloud.data_cloud import DataCloud +from dvc.system import System class StageNotFoundError(DvcException): @@ -149,7 +150,26 @@ class Project(object): raise ReproductionError(stages[n].relpath, ex) return result + def _remove_untracked_hardlinks(self): + untracked = self.scm.untracked_files() + cache = dict((System.inode(c), c) for c in self.cache.all()) + for file in untracked: + inode = System.inode(file) + if inode not in cache.keys(): + continue + + Logger.info(u'Remove \'{}\''.format(file)) + os.chmod(file, stat.S_IWRITE) + os.remove(file) + os.chmod(cache[inode], stat.S_IREAD) + + dir = os.path.dirname(file) + if len(dir) != 0 and not os.listdir(dir): + Logger.info(u'Remove empty directory \'{}\''.format(dir)) + os.removedirs(dir) + def checkout(self): + self._remove_untracked_hardlinks() for stage in self.stages(): stage.checkout() @@ -173,29 +193,15 @@ class Project(object): return list(cache_set) def _remove_cache_file(self, cache): - os.chmod(cache, stat.S_IWRITE) + os.chmod(cache, stat.S_IWRITE | stat.S_IREAD) os.unlink(cache) - def _remove_cache(self, cache): - if os.path.isfile(cache): - self._remove_cache_file(cache) - return - - for root, dirs, files in os.walk(cache, topdown=False): - for dname in dirs: - path = os.path.join(root, dname) - os.rmdir(path) - for fname in files: - path = os.path.join(root, fname) - self._remove_cache_file(path) - os.rmdir(cache) - def gc(self): clist = self._used_cache() for cache in self.cache.all(): if cache in clist: continue - self._remove_cache(cache) + self._remove_cache_file(cache) self.logger.info(u'\'{}\' was removed'.format(self.to_dvc_path(cache))) def push(self, target=None, jobs=1): diff --git a/dvc/utils.py b/dvc/utils.py index 19e022e23..b9aaf62a1 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -45,7 +45,7 @@ def cached_property(func): return property(get) -def copyfile(src, dest): +def copyfile(src, dest, no_progress_bar=False): '''Copy file with progress bar''' copied = 0 name = os.path.basename(src) @@ -64,9 +64,11 @@ def copyfile(src, dest): break fdest.write(buf) copied += len(buf) - progress.update_target(name, copied, total) + if not no_progress_bar: + progress.update_target(name, copied, total) - progress.finish_target(name) + if not no_progress_bar: + progress.finish_target(name) fsrc.close() fdest.close() diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index 9b5826d9f..722e1ab5b 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -71,11 +71,14 @@ class TestDataCloudBase(TestDvc): self.assertEqual(status, STATUS_MODIFIED) # Remove and check status - os.unlink(cache) + for root, dirs, files in os.walk(self.dvc.cache.cache_dir): + for f in files: + path = os.path.join(root, f) + self.dvc._remove_cache_file(path) + status = self.cloud.status(cache) self.assertEqual(status, STATUS_DELETED) - self.dvc._remove_cache(cache_dir) status_dir = self.cloud.status(cache_dir) self.assertEqual(status_dir, STATUS_DELETED) @@ -164,8 +167,6 @@ class TestDataCloudGCP(TestDataCloudBase): class TestDataCloudLOCAL(TestDataCloudBase): def _should_test(self): - if os.name == 'nt': - return False return True def _setup_cloud(self): @@ -187,10 +188,6 @@ class TestDataCloudLocalCli(TestDvc): self.assertEqual(ret, 0) def test(self): - #FIXME enable this test for windows - if os.name == 'nt': - return - dname = 'cloud' os.mkdir(dname) @@ -213,9 +210,10 @@ class TestDataCloudLocalCli(TestDvc): self.main(['status']) - os.chmod(cache, 0o777) - os.unlink(cache) - self.dvc._remove_cache(cache_dir) + for root, dirs, files in os.walk(self.dvc.cache.cache_dir): + for f in files: + path = os.path.join(root, f) + self.dvc._remove_cache_file(path) self.main(['status']) diff --git a/tests/test_gc.py b/tests/test_gc.py index e9de7ad65..2bd29ea58 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -27,10 +27,6 @@ class TestGC(TestDvc): self.create(path, i) self.bad_cache.append(path) - path = os.path.join(self.dvc.cache.cache_dir, '45', '6', 'data') - self.create(path, 'md5: "123"') - self.bad_cache.append(path) - def test_api(self): self.dvc.gc() self._test_gc()