GC command

This commit is contained in:
Dmitry Petrov
2017-06-14 23:48:58 -07:00
parent 5e4c11e8b5
commit f2bac9aac1
10 changed files with 211 additions and 90 deletions
-2
View File
@@ -1,11 +1,9 @@
import argparse
import os
import fasteners
from dvc.config import ConfigError
from dvc.exceptions import DvcException
from dvc.logger import Logger
from dvc.system import System
class CmdBaseError(DvcException):
-7
View File
@@ -1,14 +1,7 @@
import base64
import hashlib
import os
from multiprocessing.pool import ThreadPool
from boto.s3.connection import S3Connection
from google.cloud import storage as gc
from dvc.command.base import CmdBase, DvcLock
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.runtime import Runtime
from dvc.system import System
+59
View File
@@ -0,0 +1,59 @@
import os
from dvc.command.traverse import Traverse
from dvc.logger import Logger
from dvc.path.data_item import DataItemError
from dvc.runtime import Runtime
class CmdGC(Traverse):
def __init__(self, settings):
super(CmdGC, self).__init__(settings, "garbage collect", do_not_start_from_root=False)
def define_args(self, parser):
super(CmdGC, self).define_args(parser)
parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively.')
parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False,
help='Do not remove data from cache.')
pass
def process_file(self, target):
Logger.debug(u'[Cmd-GC] GC file {}.'.format(target))
try:
data_item = self.settings.path_factory.existing_data_item(target)
except DataItemError:
Logger.warn(u'[Cmd-GC] Data item {} is not a valid symbolic link'.format(target))
data_item = self.settings.path_factory.data_item(target)
print('DATA ITEM {} {}'.format(data_item.data.relative, data_item.cache.relative))
for cache_data_item in data_item.get_all_caches():
if cache_data_item.cache.relative != data_item.cache.relative:
os.remove(cache_data_item.cache.relative)
Logger.info(u'GC cache file {}'.format(cache_data_item.cache.relative))
self._remove_cloud_cache(cache_data_item)
Logger.debug(u'[Cmd-GC] GC data item {}. Success.'.format(data_item.data.relative))
pass
def _remove_cloud_cache(self, data_item):
if not self.parsed_args.keep_in_cloud:
self.cloud.remove_from_cloud(data_item)
def is_recursive(self):
return self.parsed_args.recursive
# Renaming
def gc_dir(self, target):
return self._traverse_dir(target)
def gc_file(self, target):
self.process_file(target)
def gc_all(self):
return self._traverse_all()
if __name__ == '__main__':
Runtime.run(CmdGC)
-1
View File
@@ -2,7 +2,6 @@ import os
from shutil import copyfile
import re
import requests
from multiprocessing.pool import ThreadPool
from dvc.command.base import CmdBase, DvcLock
+33 -80
View File
@@ -1,90 +1,30 @@
import os
from boto.s3.connection import S3Connection
from dvc.command.base import CmdBase, DvcLock
from dvc.exceptions import DvcException
from dvc.command.traverse import Traverse
from dvc.logger import Logger
from dvc.path.data_item import DataItemError
from dvc.runtime import Runtime
from dvc.data_cloud import DataCloud
class DataRemoveError(DvcException):
def __init__(self, msg):
DvcException.__init__(self, 'Data remove error: {}'.format(msg))
class CmdDataRemove(CmdBase):
class CmdDataRemove(Traverse):
def __init__(self, settings):
super(CmdDataRemove, self).__init__(settings)
super(CmdDataRemove, self).__init__(settings, "remove")
def define_args(self, parser):
self.set_no_git_actions(parser)
parser.add_argument('target', metavar='', help='Target to remove - file or directory.', nargs='*')
super(CmdDataRemove, self).define_args(parser)
parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively.')
parser.add_argument('-l', '--keep-in-cloud', action='store_true', default=False,
help='Do not remove data from cloud.')
parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False,
help='Do not remove data from cache.')
pass
def run(self):
with DvcLock(self.is_locker, self.git):
self.cloud = DataCloud(self.settings)
if not self.remove_all_targets():
return 1
return 0
def remove_all_targets(self):
if not self.no_git_actions and not self.git.is_ready_to_go():
return False
error = False
for target in self.parsed_args.target:
if not self.remove_target(target):
error = True
message = 'DVC data remove: {}'.format(' '.join(self.parsed_args.target))
self.commit_if_needed(message, error)
return error == 0
def remove_target(self, target):
try:
if os.path.isdir(target):
self.remove_dir(target)
else:
self.remove_file(target)
return True
except DvcException as ex:
Logger.error('[Cmd-Remove] Unable to remove data item "{}": {}'.format(target, ex))
return False
def remove_dir(self, target):
if not self.parsed_args.recursive:
raise DataRemoveError('[Cmd-Remove] Directory "%s" cannot be removed. Use --recurcive flag.' % target)
data_item = self.settings.path_factory.data_item(target)
if data_item.data_dvc_short == '':
raise DataRemoveError('[Cmd-Remove] Data directory "%s" cannot be removed' % target)
return self.remove_dir_file_by_file(target)
@staticmethod
def remove_dir_if_empty(file):
dir = os.path.dirname(file)
if dir != '' and not os.listdir(dir):
Logger.debug(u'[Cmd-Remove] Empty directory was removed {}.'.format(dir))
os.rmdir(dir)
pass
def remove_file(self, target):
# it raises exception if not a symlink is provided
def process_file(self, target):
Logger.debug(u'[Cmd-Remove] Remove file {}.'.format(target))
data_item = self.settings.path_factory.existing_data_item(target)
try:
data_item = self.settings.path_factory.existing_data_item(target)
except DataItemError:
Logger.warn(u'[Cmd-Remove] Data file {} is not valid symbolic link'.format(target))
data_item = self.settings.path_factory.data_item(target)
self._remove_cache_file(data_item)
self._remove_state_file(data_item)
@@ -120,17 +60,30 @@ class CmdDataRemove(CmdBase):
self.remove_dir_if_empty(dvc_path.relative)
Logger.debug(u'[Cmd-Remove] Remove {}. Success.'.format(name))
def remove_dir_file_by_file(self, target):
for f in os.listdir(target):
file = os.path.join(target, f)
if os.path.isdir(file):
self.remove_dir_file_by_file(file)
else:
self.remove_file(file)
os.rmdir(target)
@staticmethod
def remove_dir_if_empty(file):
dir = os.path.dirname(file)
if dir != '' and not os.listdir(dir):
Logger.debug(u'[Cmd-Remove] Empty directory was removed {}.'.format(dir))
os.rmdir(dir)
pass
def traverse_dir_finalize(self, target):
os.rmdir(target)
def is_recursive(self):
return self.parsed_args.recursive
# Renaming
def remove_dir(self, target):
return self._traverse_dir(target)
def remove_file(self, target):
self.process_file(target)
def remove_all(self):
return self._traverse_all()
if __name__ == '__main__':
Runtime.run(CmdDataRemove)
+90
View File
@@ -0,0 +1,90 @@
import os
from dvc.command.base import CmdBase, DvcLock
from dvc.exceptions import DvcException
from dvc.logger import Logger
from dvc.data_cloud import DataCloud
class TraverseError(DvcException):
def __init__(self, msg):
DvcException.__init__(self, 'Traverse file tree error: {}'.format(msg))
class Traverse(CmdBase):
def __init__(self, settings, cmd_name, do_not_start_from_root=True):
super(Traverse, self).__init__(settings)
self._cmd_name = cmd_name
self.cloud = DataCloud(self.settings)
self._do_not_start_from_root = do_not_start_from_root
def define_args(self, parser):
self.set_no_git_actions(parser)
parser.add_argument('target', metavar='', help='Target to remove - file or directory.', nargs='*')
# parser.add_argument('-r', '--recursive', action='store_true', help='CmdGarbage collect directory recursively.')
parser.add_argument('-l', '--keep-in-cloud', action='store_true', default=False,
help='Do not remove data from cloud.')
pass
def run(self):
with DvcLock(self.is_locker, self.git):
if not self._traverse_all():
return 1
return 0
def _traverse_all(self):
if not self.no_git_actions and not self.git.is_ready_to_go():
return False
error = False
print('TARGET={}'.format(self.parsed_args.target))
for target in self.parsed_args.target:
if not self._traverse(target):
error = True
message = 'DVC {}: {}'.format(self._cmd_name, ' '.join(self.parsed_args.target))
self.commit_if_needed(message, error)
return error == 0
def _traverse(self, target):
try:
if os.path.isdir(target):
if not self.is_recursive:
msg = '[TraverseFileTree] Directory "%s" cannot be traversed. Use --recurcive flag.'
raise TraverseError(msg % target)
if self._do_not_start_from_root and self.settings.path_factory.data_item(target).data_dvc_short == '':
# The entire data directory
raise TraverseError('[Traverse] Root data directory "%s" cannot be traversed' % target)
self._traverse_dir(target)
else:
self.process_file(target)
return True
except DvcException as ex:
Logger.error('[TraverseFileTree] Unable to {} data item "{}": {}'.format(self._cmd_name, target, ex))
return False
def _traverse_dir(self, target):
for f in os.listdir(target):
file = os.path.join(target, f)
if os.path.isdir(file):
self._traverse_dir(file)
else:
self.process_file(file)
self.traverse_dir_finalize(target)
pass
## API:
def is_recursive(self):
return True
def process_file(self, target):
pass
def traverse_dir_finalize(self, target):
pass
+2
View File
@@ -7,6 +7,8 @@ from dvc.system import System
class GitWrapperI(object):
COMMIT_LEN = 7
def __init__(self, git_dir=None, commit=None):
self._git_dir = git_dir
self._commit = commit
+20
View File
@@ -1,5 +1,7 @@
import os
import re
import shutil
import fnmatch
from dvc.path.path import Path
from dvc.exceptions import DvcException
@@ -67,6 +69,11 @@ class DataItem(object):
raise NotInDataDirError.build(data_file, self._config.data_dir)
pass
def copy(self, cache_file=None):
if not cache_file:
cache_file = self._cache_file
return DataItem(self._data.abs, self._git, self._config, cache_file)
def __hash__(self):
return self.data.dvc.__hash__()
@@ -99,6 +106,19 @@ class DataItem(object):
cache_file = os.path.join(cache_dir, file_name)
return Path(cache_file, self._git)
def get_all_caches(self):
result = []
suffix_len = self._git.COMMIT_LEN + len(self.CACHE_FILE_SEP)
cache_prefix = os.path.basename(self.cache.relative[:-suffix_len])
print('DATA={} DIRNAME={}'.format(self._data.relative, self.cache.dirname))
for cache_file in os.listdir(self.cache.dirname):
if cache_file[:-suffix_len] == cache_prefix:
result.append(self.copy(cache_file))
return result
@cached_property
def resolved_cache(self):
resolved_cache = os.path.realpath(self._data.relative)
+5
View File
@@ -13,6 +13,7 @@ class Path(object):
self._dvc = os.path.relpath(self.abs, git.git_dir_abs)
self._relative = os.path.relpath(self._abs, git.curr_dir_abs)
self._filename = os.path.basename(self._abs)
self._dirname = os.path.dirname(self._abs)
@staticmethod
def from_dvc_path(dvc_path, git):
@@ -33,3 +34,7 @@ class Path(object):
@property
def filename(self):
return self._filename
@property
def dirname(self):
return self._dirname
+2
View File
@@ -82,7 +82,9 @@ class System(object):
def get_long_path(path):
"""Convert short path to a full path. It is needed for Windows."""
if System.is_unix():
print('======================11111111111111111111')
return path
print('======================222222222222222222222')
buffer = create_unicode_buffer(System.LONG_PATH_BUFFER_SIZE)
get_long_path_name = windll.kernel32.GetLongPathNameW