mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
GC command
This commit is contained in:
@@ -1,11 +1,9 @@
|
||||
import argparse
|
||||
import os
|
||||
import fasteners
|
||||
|
||||
from dvc.config import ConfigError
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
from dvc.system import System
|
||||
|
||||
|
||||
class CmdBaseError(DvcException):
|
||||
|
||||
@@ -1,14 +1,7 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
from multiprocessing.pool import ThreadPool
|
||||
|
||||
from boto.s3.connection import S3Connection
|
||||
from google.cloud import storage as gc
|
||||
|
||||
from dvc.command.base import CmdBase, DvcLock
|
||||
from dvc.logger import Logger
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.runtime import Runtime
|
||||
from dvc.system import System
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import os
|
||||
|
||||
from dvc.command.traverse import Traverse
|
||||
from dvc.logger import Logger
|
||||
from dvc.path.data_item import DataItemError
|
||||
from dvc.runtime import Runtime
|
||||
|
||||
|
||||
class CmdGC(Traverse):
|
||||
def __init__(self, settings):
|
||||
super(CmdGC, self).__init__(settings, "garbage collect", do_not_start_from_root=False)
|
||||
|
||||
def define_args(self, parser):
|
||||
super(CmdGC, self).define_args(parser)
|
||||
parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively.')
|
||||
parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False,
|
||||
help='Do not remove data from cache.')
|
||||
pass
|
||||
|
||||
def process_file(self, target):
|
||||
Logger.debug(u'[Cmd-GC] GC file {}.'.format(target))
|
||||
|
||||
try:
|
||||
data_item = self.settings.path_factory.existing_data_item(target)
|
||||
except DataItemError:
|
||||
Logger.warn(u'[Cmd-GC] Data item {} is not a valid symbolic link'.format(target))
|
||||
data_item = self.settings.path_factory.data_item(target)
|
||||
|
||||
print('DATA ITEM {} {}'.format(data_item.data.relative, data_item.cache.relative))
|
||||
|
||||
for cache_data_item in data_item.get_all_caches():
|
||||
if cache_data_item.cache.relative != data_item.cache.relative:
|
||||
os.remove(cache_data_item.cache.relative)
|
||||
Logger.info(u'GC cache file {}'.format(cache_data_item.cache.relative))
|
||||
self._remove_cloud_cache(cache_data_item)
|
||||
|
||||
Logger.debug(u'[Cmd-GC] GC data item {}. Success.'.format(data_item.data.relative))
|
||||
pass
|
||||
|
||||
def _remove_cloud_cache(self, data_item):
|
||||
if not self.parsed_args.keep_in_cloud:
|
||||
self.cloud.remove_from_cloud(data_item)
|
||||
|
||||
def is_recursive(self):
|
||||
return self.parsed_args.recursive
|
||||
|
||||
# Renaming
|
||||
def gc_dir(self, target):
|
||||
return self._traverse_dir(target)
|
||||
|
||||
def gc_file(self, target):
|
||||
self.process_file(target)
|
||||
|
||||
def gc_all(self):
|
||||
return self._traverse_all()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
Runtime.run(CmdGC)
|
||||
@@ -2,7 +2,6 @@ import os
|
||||
from shutil import copyfile
|
||||
import re
|
||||
import requests
|
||||
|
||||
from multiprocessing.pool import ThreadPool
|
||||
|
||||
from dvc.command.base import CmdBase, DvcLock
|
||||
|
||||
+33
-80
@@ -1,90 +1,30 @@
|
||||
import os
|
||||
|
||||
from boto.s3.connection import S3Connection
|
||||
|
||||
from dvc.command.base import CmdBase, DvcLock
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.command.traverse import Traverse
|
||||
from dvc.logger import Logger
|
||||
from dvc.path.data_item import DataItemError
|
||||
from dvc.runtime import Runtime
|
||||
from dvc.data_cloud import DataCloud
|
||||
|
||||
class DataRemoveError(DvcException):
|
||||
def __init__(self, msg):
|
||||
DvcException.__init__(self, 'Data remove error: {}'.format(msg))
|
||||
|
||||
|
||||
class CmdDataRemove(CmdBase):
|
||||
class CmdDataRemove(Traverse):
|
||||
def __init__(self, settings):
|
||||
super(CmdDataRemove, self).__init__(settings)
|
||||
super(CmdDataRemove, self).__init__(settings, "remove")
|
||||
|
||||
def define_args(self, parser):
|
||||
self.set_no_git_actions(parser)
|
||||
|
||||
parser.add_argument('target', metavar='', help='Target to remove - file or directory.', nargs='*')
|
||||
super(CmdDataRemove, self).define_args(parser)
|
||||
parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively.')
|
||||
parser.add_argument('-l', '--keep-in-cloud', action='store_true', default=False,
|
||||
help='Do not remove data from cloud.')
|
||||
parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False,
|
||||
help='Do not remove data from cache.')
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
with DvcLock(self.is_locker, self.git):
|
||||
self.cloud = DataCloud(self.settings)
|
||||
|
||||
if not self.remove_all_targets():
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
def remove_all_targets(self):
|
||||
if not self.no_git_actions and not self.git.is_ready_to_go():
|
||||
return False
|
||||
|
||||
error = False
|
||||
for target in self.parsed_args.target:
|
||||
if not self.remove_target(target):
|
||||
error = True
|
||||
|
||||
message = 'DVC data remove: {}'.format(' '.join(self.parsed_args.target))
|
||||
self.commit_if_needed(message, error)
|
||||
|
||||
return error == 0
|
||||
|
||||
def remove_target(self, target):
|
||||
try:
|
||||
if os.path.isdir(target):
|
||||
self.remove_dir(target)
|
||||
else:
|
||||
self.remove_file(target)
|
||||
return True
|
||||
except DvcException as ex:
|
||||
Logger.error('[Cmd-Remove] Unable to remove data item "{}": {}'.format(target, ex))
|
||||
return False
|
||||
|
||||
def remove_dir(self, target):
|
||||
if not self.parsed_args.recursive:
|
||||
raise DataRemoveError('[Cmd-Remove] Directory "%s" cannot be removed. Use --recurcive flag.' % target)
|
||||
|
||||
data_item = self.settings.path_factory.data_item(target)
|
||||
if data_item.data_dvc_short == '':
|
||||
raise DataRemoveError('[Cmd-Remove] Data directory "%s" cannot be removed' % target)
|
||||
|
||||
return self.remove_dir_file_by_file(target)
|
||||
|
||||
@staticmethod
|
||||
def remove_dir_if_empty(file):
|
||||
dir = os.path.dirname(file)
|
||||
if dir != '' and not os.listdir(dir):
|
||||
Logger.debug(u'[Cmd-Remove] Empty directory was removed {}.'.format(dir))
|
||||
os.rmdir(dir)
|
||||
pass
|
||||
|
||||
def remove_file(self, target):
|
||||
# it raises exception if not a symlink is provided
|
||||
def process_file(self, target):
|
||||
Logger.debug(u'[Cmd-Remove] Remove file {}.'.format(target))
|
||||
|
||||
data_item = self.settings.path_factory.existing_data_item(target)
|
||||
try:
|
||||
data_item = self.settings.path_factory.existing_data_item(target)
|
||||
except DataItemError:
|
||||
Logger.warn(u'[Cmd-Remove] Data file {} is not valid symbolic link'.format(target))
|
||||
data_item = self.settings.path_factory.data_item(target)
|
||||
|
||||
self._remove_cache_file(data_item)
|
||||
self._remove_state_file(data_item)
|
||||
@@ -120,17 +60,30 @@ class CmdDataRemove(CmdBase):
|
||||
self.remove_dir_if_empty(dvc_path.relative)
|
||||
Logger.debug(u'[Cmd-Remove] Remove {}. Success.'.format(name))
|
||||
|
||||
def remove_dir_file_by_file(self, target):
|
||||
for f in os.listdir(target):
|
||||
file = os.path.join(target, f)
|
||||
if os.path.isdir(file):
|
||||
self.remove_dir_file_by_file(file)
|
||||
else:
|
||||
self.remove_file(file)
|
||||
|
||||
os.rmdir(target)
|
||||
@staticmethod
|
||||
def remove_dir_if_empty(file):
|
||||
dir = os.path.dirname(file)
|
||||
if dir != '' and not os.listdir(dir):
|
||||
Logger.debug(u'[Cmd-Remove] Empty directory was removed {}.'.format(dir))
|
||||
os.rmdir(dir)
|
||||
pass
|
||||
|
||||
def traverse_dir_finalize(self, target):
|
||||
os.rmdir(target)
|
||||
|
||||
def is_recursive(self):
|
||||
return self.parsed_args.recursive
|
||||
|
||||
# Renaming
|
||||
def remove_dir(self, target):
|
||||
return self._traverse_dir(target)
|
||||
|
||||
def remove_file(self, target):
|
||||
self.process_file(target)
|
||||
|
||||
def remove_all(self):
|
||||
return self._traverse_all()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
Runtime.run(CmdDataRemove)
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import os
|
||||
|
||||
from dvc.command.base import CmdBase, DvcLock
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
from dvc.data_cloud import DataCloud
|
||||
|
||||
|
||||
class TraverseError(DvcException):
|
||||
def __init__(self, msg):
|
||||
DvcException.__init__(self, 'Traverse file tree error: {}'.format(msg))
|
||||
|
||||
|
||||
class Traverse(CmdBase):
|
||||
def __init__(self, settings, cmd_name, do_not_start_from_root=True):
|
||||
super(Traverse, self).__init__(settings)
|
||||
self._cmd_name = cmd_name
|
||||
self.cloud = DataCloud(self.settings)
|
||||
self._do_not_start_from_root = do_not_start_from_root
|
||||
|
||||
def define_args(self, parser):
|
||||
self.set_no_git_actions(parser)
|
||||
|
||||
parser.add_argument('target', metavar='', help='Target to remove - file or directory.', nargs='*')
|
||||
# parser.add_argument('-r', '--recursive', action='store_true', help='CmdGarbage collect directory recursively.')
|
||||
parser.add_argument('-l', '--keep-in-cloud', action='store_true', default=False,
|
||||
help='Do not remove data from cloud.')
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
with DvcLock(self.is_locker, self.git):
|
||||
if not self._traverse_all():
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
def _traverse_all(self):
|
||||
if not self.no_git_actions and not self.git.is_ready_to_go():
|
||||
return False
|
||||
|
||||
error = False
|
||||
print('TARGET={}'.format(self.parsed_args.target))
|
||||
for target in self.parsed_args.target:
|
||||
if not self._traverse(target):
|
||||
error = True
|
||||
|
||||
message = 'DVC {}: {}'.format(self._cmd_name, ' '.join(self.parsed_args.target))
|
||||
self.commit_if_needed(message, error)
|
||||
|
||||
return error == 0
|
||||
|
||||
def _traverse(self, target):
|
||||
try:
|
||||
if os.path.isdir(target):
|
||||
if not self.is_recursive:
|
||||
msg = '[TraverseFileTree] Directory "%s" cannot be traversed. Use --recurcive flag.'
|
||||
raise TraverseError(msg % target)
|
||||
|
||||
if self._do_not_start_from_root and self.settings.path_factory.data_item(target).data_dvc_short == '':
|
||||
# The entire data directory
|
||||
raise TraverseError('[Traverse] Root data directory "%s" cannot be traversed' % target)
|
||||
|
||||
self._traverse_dir(target)
|
||||
else:
|
||||
self.process_file(target)
|
||||
return True
|
||||
except DvcException as ex:
|
||||
Logger.error('[TraverseFileTree] Unable to {} data item "{}": {}'.format(self._cmd_name, target, ex))
|
||||
return False
|
||||
|
||||
def _traverse_dir(self, target):
|
||||
for f in os.listdir(target):
|
||||
file = os.path.join(target, f)
|
||||
if os.path.isdir(file):
|
||||
self._traverse_dir(file)
|
||||
else:
|
||||
self.process_file(file)
|
||||
|
||||
self.traverse_dir_finalize(target)
|
||||
pass
|
||||
|
||||
## API:
|
||||
def is_recursive(self):
|
||||
return True
|
||||
|
||||
def process_file(self, target):
|
||||
pass
|
||||
|
||||
def traverse_dir_finalize(self, target):
|
||||
pass
|
||||
@@ -7,6 +7,8 @@ from dvc.system import System
|
||||
|
||||
|
||||
class GitWrapperI(object):
|
||||
COMMIT_LEN = 7
|
||||
|
||||
def __init__(self, git_dir=None, commit=None):
|
||||
self._git_dir = git_dir
|
||||
self._commit = commit
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import fnmatch
|
||||
|
||||
from dvc.path.path import Path
|
||||
from dvc.exceptions import DvcException
|
||||
@@ -67,6 +69,11 @@ class DataItem(object):
|
||||
raise NotInDataDirError.build(data_file, self._config.data_dir)
|
||||
pass
|
||||
|
||||
def copy(self, cache_file=None):
|
||||
if not cache_file:
|
||||
cache_file = self._cache_file
|
||||
return DataItem(self._data.abs, self._git, self._config, cache_file)
|
||||
|
||||
def __hash__(self):
|
||||
return self.data.dvc.__hash__()
|
||||
|
||||
@@ -99,6 +106,19 @@ class DataItem(object):
|
||||
cache_file = os.path.join(cache_dir, file_name)
|
||||
return Path(cache_file, self._git)
|
||||
|
||||
def get_all_caches(self):
|
||||
result = []
|
||||
|
||||
suffix_len = self._git.COMMIT_LEN + len(self.CACHE_FILE_SEP)
|
||||
cache_prefix = os.path.basename(self.cache.relative[:-suffix_len])
|
||||
|
||||
print('DATA={} DIRNAME={}'.format(self._data.relative, self.cache.dirname))
|
||||
for cache_file in os.listdir(self.cache.dirname):
|
||||
if cache_file[:-suffix_len] == cache_prefix:
|
||||
result.append(self.copy(cache_file))
|
||||
|
||||
return result
|
||||
|
||||
@cached_property
|
||||
def resolved_cache(self):
|
||||
resolved_cache = os.path.realpath(self._data.relative)
|
||||
|
||||
@@ -13,6 +13,7 @@ class Path(object):
|
||||
self._dvc = os.path.relpath(self.abs, git.git_dir_abs)
|
||||
self._relative = os.path.relpath(self._abs, git.curr_dir_abs)
|
||||
self._filename = os.path.basename(self._abs)
|
||||
self._dirname = os.path.dirname(self._abs)
|
||||
|
||||
@staticmethod
|
||||
def from_dvc_path(dvc_path, git):
|
||||
@@ -33,3 +34,7 @@ class Path(object):
|
||||
@property
|
||||
def filename(self):
|
||||
return self._filename
|
||||
|
||||
@property
|
||||
def dirname(self):
|
||||
return self._dirname
|
||||
@@ -82,7 +82,9 @@ class System(object):
|
||||
def get_long_path(path):
|
||||
"""Convert short path to a full path. It is needed for Windows."""
|
||||
if System.is_unix():
|
||||
print('======================11111111111111111111')
|
||||
return path
|
||||
print('======================222222222222222222222')
|
||||
|
||||
buffer = create_unicode_buffer(System.LONG_PATH_BUFFER_SIZE)
|
||||
get_long_path_name = windll.kernel32.GetLongPathNameW
|
||||
|
||||
Reference in New Issue
Block a user