diff --git a/dvc/command/base.py b/dvc/command/base.py index 722c77492..167bae691 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -1,11 +1,9 @@ import argparse -import os import fasteners from dvc.config import ConfigError from dvc.exceptions import DvcException from dvc.logger import Logger -from dvc.system import System class CmdBaseError(DvcException): diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 820aaad20..4474b2594 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -1,14 +1,7 @@ -import base64 -import hashlib import os - from multiprocessing.pool import ThreadPool -from boto.s3.connection import S3Connection -from google.cloud import storage as gc - from dvc.command.base import CmdBase, DvcLock -from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.runtime import Runtime from dvc.system import System diff --git a/dvc/command/gc.py b/dvc/command/gc.py new file mode 100644 index 000000000..0f160b544 --- /dev/null +++ b/dvc/command/gc.py @@ -0,0 +1,59 @@ +import os + +from dvc.command.traverse import Traverse +from dvc.logger import Logger +from dvc.path.data_item import DataItemError +from dvc.runtime import Runtime + + +class CmdGC(Traverse): + def __init__(self, settings): + super(CmdGC, self).__init__(settings, "garbage collect", do_not_start_from_root=False) + + def define_args(self, parser): + super(CmdGC, self).define_args(parser) + parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively.') + parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False, + help='Do not remove data from cache.') + pass + + def process_file(self, target): + Logger.debug(u'[Cmd-GC] GC file {}.'.format(target)) + + try: + data_item = self.settings.path_factory.existing_data_item(target) + except DataItemError: + Logger.warn(u'[Cmd-GC] Data item {} is not a valid symbolic link'.format(target)) + data_item = self.settings.path_factory.data_item(target) + + print('DATA ITEM {} {}'.format(data_item.data.relative, data_item.cache.relative)) + + for cache_data_item in data_item.get_all_caches(): + if cache_data_item.cache.relative != data_item.cache.relative: + os.remove(cache_data_item.cache.relative) + Logger.info(u'GC cache file {}'.format(cache_data_item.cache.relative)) + self._remove_cloud_cache(cache_data_item) + + Logger.debug(u'[Cmd-GC] GC data item {}. Success.'.format(data_item.data.relative)) + pass + + def _remove_cloud_cache(self, data_item): + if not self.parsed_args.keep_in_cloud: + self.cloud.remove_from_cloud(data_item) + + def is_recursive(self): + return self.parsed_args.recursive + + # Renaming + def gc_dir(self, target): + return self._traverse_dir(target) + + def gc_file(self, target): + self.process_file(target) + + def gc_all(self): + return self._traverse_all() + + +if __name__ == '__main__': + Runtime.run(CmdGC) diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index 9a0efb054..a97911364 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -2,7 +2,6 @@ import os from shutil import copyfile import re import requests - from multiprocessing.pool import ThreadPool from dvc.command.base import CmdBase, DvcLock diff --git a/dvc/command/remove.py b/dvc/command/remove.py index 34b8d06f5..75d859f61 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -1,90 +1,30 @@ import os -from boto.s3.connection import S3Connection - -from dvc.command.base import CmdBase, DvcLock -from dvc.exceptions import DvcException +from dvc.command.traverse import Traverse from dvc.logger import Logger +from dvc.path.data_item import DataItemError from dvc.runtime import Runtime -from dvc.data_cloud import DataCloud - -class DataRemoveError(DvcException): - def __init__(self, msg): - DvcException.__init__(self, 'Data remove error: {}'.format(msg)) -class CmdDataRemove(CmdBase): +class CmdDataRemove(Traverse): def __init__(self, settings): - super(CmdDataRemove, self).__init__(settings) + super(CmdDataRemove, self).__init__(settings, "remove") def define_args(self, parser): - self.set_no_git_actions(parser) - - parser.add_argument('target', metavar='', help='Target to remove - file or directory.', nargs='*') + super(CmdDataRemove, self).define_args(parser) parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively.') - parser.add_argument('-l', '--keep-in-cloud', action='store_true', default=False, - help='Do not remove data from cloud.') parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False, help='Do not remove data from cache.') pass - def run(self): - with DvcLock(self.is_locker, self.git): - self.cloud = DataCloud(self.settings) - - if not self.remove_all_targets(): - return 1 - - return 0 - - def remove_all_targets(self): - if not self.no_git_actions and not self.git.is_ready_to_go(): - return False - - error = False - for target in self.parsed_args.target: - if not self.remove_target(target): - error = True - - message = 'DVC data remove: {}'.format(' '.join(self.parsed_args.target)) - self.commit_if_needed(message, error) - - return error == 0 - - def remove_target(self, target): - try: - if os.path.isdir(target): - self.remove_dir(target) - else: - self.remove_file(target) - return True - except DvcException as ex: - Logger.error('[Cmd-Remove] Unable to remove data item "{}": {}'.format(target, ex)) - return False - - def remove_dir(self, target): - if not self.parsed_args.recursive: - raise DataRemoveError('[Cmd-Remove] Directory "%s" cannot be removed. Use --recurcive flag.' % target) - - data_item = self.settings.path_factory.data_item(target) - if data_item.data_dvc_short == '': - raise DataRemoveError('[Cmd-Remove] Data directory "%s" cannot be removed' % target) - - return self.remove_dir_file_by_file(target) - - @staticmethod - def remove_dir_if_empty(file): - dir = os.path.dirname(file) - if dir != '' and not os.listdir(dir): - Logger.debug(u'[Cmd-Remove] Empty directory was removed {}.'.format(dir)) - os.rmdir(dir) - pass - - def remove_file(self, target): - # it raises exception if not a symlink is provided + def process_file(self, target): Logger.debug(u'[Cmd-Remove] Remove file {}.'.format(target)) - data_item = self.settings.path_factory.existing_data_item(target) + try: + data_item = self.settings.path_factory.existing_data_item(target) + except DataItemError: + Logger.warn(u'[Cmd-Remove] Data file {} is not valid symbolic link'.format(target)) + data_item = self.settings.path_factory.data_item(target) self._remove_cache_file(data_item) self._remove_state_file(data_item) @@ -120,17 +60,30 @@ class CmdDataRemove(CmdBase): self.remove_dir_if_empty(dvc_path.relative) Logger.debug(u'[Cmd-Remove] Remove {}. Success.'.format(name)) - def remove_dir_file_by_file(self, target): - for f in os.listdir(target): - file = os.path.join(target, f) - if os.path.isdir(file): - self.remove_dir_file_by_file(file) - else: - self.remove_file(file) - - os.rmdir(target) + @staticmethod + def remove_dir_if_empty(file): + dir = os.path.dirname(file) + if dir != '' and not os.listdir(dir): + Logger.debug(u'[Cmd-Remove] Empty directory was removed {}.'.format(dir)) + os.rmdir(dir) pass + def traverse_dir_finalize(self, target): + os.rmdir(target) + + def is_recursive(self): + return self.parsed_args.recursive + + # Renaming + def remove_dir(self, target): + return self._traverse_dir(target) + + def remove_file(self, target): + self.process_file(target) + + def remove_all(self): + return self._traverse_all() + if __name__ == '__main__': Runtime.run(CmdDataRemove) diff --git a/dvc/command/traverse.py b/dvc/command/traverse.py new file mode 100644 index 000000000..686a0643e --- /dev/null +++ b/dvc/command/traverse.py @@ -0,0 +1,90 @@ +import os + +from dvc.command.base import CmdBase, DvcLock +from dvc.exceptions import DvcException +from dvc.logger import Logger +from dvc.data_cloud import DataCloud + + +class TraverseError(DvcException): + def __init__(self, msg): + DvcException.__init__(self, 'Traverse file tree error: {}'.format(msg)) + + +class Traverse(CmdBase): + def __init__(self, settings, cmd_name, do_not_start_from_root=True): + super(Traverse, self).__init__(settings) + self._cmd_name = cmd_name + self.cloud = DataCloud(self.settings) + self._do_not_start_from_root = do_not_start_from_root + + def define_args(self, parser): + self.set_no_git_actions(parser) + + parser.add_argument('target', metavar='', help='Target to remove - file or directory.', nargs='*') + # parser.add_argument('-r', '--recursive', action='store_true', help='CmdGarbage collect directory recursively.') + parser.add_argument('-l', '--keep-in-cloud', action='store_true', default=False, + help='Do not remove data from cloud.') + pass + + def run(self): + with DvcLock(self.is_locker, self.git): + if not self._traverse_all(): + return 1 + + return 0 + + def _traverse_all(self): + if not self.no_git_actions and not self.git.is_ready_to_go(): + return False + + error = False + print('TARGET={}'.format(self.parsed_args.target)) + for target in self.parsed_args.target: + if not self._traverse(target): + error = True + + message = 'DVC {}: {}'.format(self._cmd_name, ' '.join(self.parsed_args.target)) + self.commit_if_needed(message, error) + + return error == 0 + + def _traverse(self, target): + try: + if os.path.isdir(target): + if not self.is_recursive: + msg = '[TraverseFileTree] Directory "%s" cannot be traversed. Use --recurcive flag.' + raise TraverseError(msg % target) + + if self._do_not_start_from_root and self.settings.path_factory.data_item(target).data_dvc_short == '': + # The entire data directory + raise TraverseError('[Traverse] Root data directory "%s" cannot be traversed' % target) + + self._traverse_dir(target) + else: + self.process_file(target) + return True + except DvcException as ex: + Logger.error('[TraverseFileTree] Unable to {} data item "{}": {}'.format(self._cmd_name, target, ex)) + return False + + def _traverse_dir(self, target): + for f in os.listdir(target): + file = os.path.join(target, f) + if os.path.isdir(file): + self._traverse_dir(file) + else: + self.process_file(file) + + self.traverse_dir_finalize(target) + pass + + ## API: + def is_recursive(self): + return True + + def process_file(self, target): + pass + + def traverse_dir_finalize(self, target): + pass diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py index a8a6a9699..7295ee106 100644 --- a/dvc/git_wrapper.py +++ b/dvc/git_wrapper.py @@ -7,6 +7,8 @@ from dvc.system import System class GitWrapperI(object): + COMMIT_LEN = 7 + def __init__(self, git_dir=None, commit=None): self._git_dir = git_dir self._commit = commit diff --git a/dvc/path/data_item.py b/dvc/path/data_item.py index e892e6843..80e630825 100644 --- a/dvc/path/data_item.py +++ b/dvc/path/data_item.py @@ -1,5 +1,7 @@ import os +import re import shutil +import fnmatch from dvc.path.path import Path from dvc.exceptions import DvcException @@ -67,6 +69,11 @@ class DataItem(object): raise NotInDataDirError.build(data_file, self._config.data_dir) pass + def copy(self, cache_file=None): + if not cache_file: + cache_file = self._cache_file + return DataItem(self._data.abs, self._git, self._config, cache_file) + def __hash__(self): return self.data.dvc.__hash__() @@ -99,6 +106,19 @@ class DataItem(object): cache_file = os.path.join(cache_dir, file_name) return Path(cache_file, self._git) + def get_all_caches(self): + result = [] + + suffix_len = self._git.COMMIT_LEN + len(self.CACHE_FILE_SEP) + cache_prefix = os.path.basename(self.cache.relative[:-suffix_len]) + + print('DATA={} DIRNAME={}'.format(self._data.relative, self.cache.dirname)) + for cache_file in os.listdir(self.cache.dirname): + if cache_file[:-suffix_len] == cache_prefix: + result.append(self.copy(cache_file)) + + return result + @cached_property def resolved_cache(self): resolved_cache = os.path.realpath(self._data.relative) diff --git a/dvc/path/path.py b/dvc/path/path.py index 08e405ee5..de225eac1 100644 --- a/dvc/path/path.py +++ b/dvc/path/path.py @@ -13,6 +13,7 @@ class Path(object): self._dvc = os.path.relpath(self.abs, git.git_dir_abs) self._relative = os.path.relpath(self._abs, git.curr_dir_abs) self._filename = os.path.basename(self._abs) + self._dirname = os.path.dirname(self._abs) @staticmethod def from_dvc_path(dvc_path, git): @@ -33,3 +34,7 @@ class Path(object): @property def filename(self): return self._filename + + @property + def dirname(self): + return self._dirname \ No newline at end of file diff --git a/dvc/system.py b/dvc/system.py index 43c52053a..752545d31 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -82,7 +82,9 @@ class System(object): def get_long_path(path): """Convert short path to a full path. It is needed for Windows.""" if System.is_unix(): + print('======================11111111111111111111') return path + print('======================222222222222222222222') buffer = create_unicode_buffer(System.LONG_PATH_BUFFER_SIZE) get_long_path_name = windll.kernel32.GetLongPathNameW