Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-09 23:10:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

Extract cache_dir\state_dir as a CongigI class variable

Browse Source
This commit is contained in:
Dmitry
2017-11-13 00:38:44 -08:00
parent 62e8b751ba
commit 8d6c134780
7 changed files with 34 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/gc.py
+4 -2
View File
@@ -4,6 +4,8 @@ from dvc.command.base import DvcLock
from dvc.command.traverse import Traverse
from dvc.logger import Logger
from dvc.path.data_item import DataItemError
from dvc.config import ConfigI
class CmdGC(Traverse):
def __init__(self, settings):
@@ -16,8 +18,8 @@ class CmdGC(Traverse):
Logger.error('Failed to collect used cache')
return 1
for cache in os.listdir(self.settings.config.cache_dir):
fname = os.path.join(self.settings.config.cache_dir, cache)
for cache in os.listdir(ConfigI.CACHE_DIR):
fname = os.path.join(ConfigI.CACHE_DIR, cache)
if fname in self.clist:
continue
os.remove(fname)
dvc/command/import_file.py
+2 -2
View File
@@ -107,8 +107,8 @@ class CmdImportFile(CmdBase):
raise ImportFileError('Output file directory "{}" does not exists'.format(
os.path.dirname(data_item.data.relative)))
if not os.path.exists(data_item.cache_dir):
os.makedirs(data_item.cache_dir)
if not os.path.exists(data_item.cache_dir_abs):
os.makedirs(data_item.cache_dir_abs)
data_targets.append((input, data_item))
dvc/command/merge.py
+3 -2
View File
@@ -2,6 +2,7 @@ import os
import json
from dvc.command.base import CmdBase, DvcLock
from dvc.config import ConfigI
from dvc.logger import Logger
from dvc.state_file import StateFile
from dvc.path.data_item import DataItem
@@ -20,8 +21,8 @@ class CmdMerge(CmdBase):
dlist = []
flist = self.git.get_last_merge_changed_files()
for fname in flist:
if fname.startswith(self.settings.config.state_dir) and fname.endswith(DataItem.STATE_FILE_SUFFIX):
data = os.path.relpath(fname, self.settings.config.state_dir)[:-len(DataItem.STATE_FILE_SUFFIX)]
if fname.startswith(ConfigI.STATE_DIR) and fname.endswith(DataItem.STATE_FILE_SUFFIX):
data = os.path.relpath(fname, ConfigI.STATE_DIR)[:-len(DataItem.STATE_FILE_SUFFIX)]
dlist.append(data)
return dlist
dvc/command/run.py
+1 -1
View File
@@ -117,7 +117,7 @@ class CmdRun(CmdBase):
result = []
for data_item in repo_change.changed_data_items:
Logger.debug('Move output file "{}" to cache dir "{}" and create a hardlink'.format(
data_item.data.relative, data_item.cache_dir))
data_item.data.relative, data_item.cache_dir_abs))
data_item.move_data_to_cache()
Logger.debug('Create state file "{}"'.format(data_item.state.relative))
dvc/config.py
+12 -9
View File
@@ -22,6 +22,9 @@ class ConfigI(object):
STATE_DIR_NAME = 'state'
CACHE_DIR_NAME = 'cache'
CACHE_DIR = os.path.join(CONFIG_DIR, CACHE_DIR_NAME)
STATE_DIR = os.path.join(CONFIG_DIR, STATE_DIR_NAME)
def __init__(self, cloud=None, conf_parser=None):
self._conf_parser = None
self._cloud = None
@@ -32,15 +35,15 @@ class ConfigI(object):
self._cloud = cloud
self._conf_parser = conf_parser
@property
def cache_dir(self):
""" Directory with cached data files """
return os.path.join(self.CONFIG_DIR, self.CACHE_DIR_NAME)
@property
def state_dir(self):
""" Directory with state files """
return os.path.join(self.CONFIG_DIR, self.STATE_DIR_NAME)
# @classmethod
# def CACHE_DIR(self):
# """ Directory with cached data files """
# return os.path.join(self.CONFIG_DIR, self.CACHE_DIR_NAME)
#
# @classmethod
# def STATE_DIR(self):
# """ Directory with state files """
# return os.path.join(self.CONFIG_DIR, self.STATE_DIR_NAME)
@property
def cloud(self):
dvc/path/data_item.py
+7 -7
View File
@@ -1,7 +1,7 @@
import os
import stat
import shutil
from dvc.config import ConfigI
from dvc.path.path import Path
from dvc.exceptions import DvcException
from dvc.system import System
@@ -71,17 +71,17 @@ class DataItem(object):
@cached_property
def state(self):
state_dir = os.path.join(self._git.git_dir_abs, self._config.state_dir)
state_dir = os.path.join(self._git.git_dir_abs, ConfigI.STATE_DIR)
state_file = os.path.join(state_dir, self.data.dvc + self.STATE_FILE_SUFFIX)
return Path(state_file, self._git)
@cached_property
def cache_dir(self):
return os.path.join(self._git.git_dir_abs, self._config.cache_dir)
def cache_dir_abs(self):
return os.path.join(self._git.git_dir_abs, ConfigI.CACHE_DIR)
@cached_property
def cache(self):
cache_dir = self.cache_dir
cache_dir = self.cache_dir_abs
if self._cache_file:
file_name = os.path.relpath(os.path.realpath(self._cache_file), cache_dir)
@@ -94,11 +94,11 @@ class DataItem(object):
@cached_property
def state_dir_abs(self):
return os.path.join(self._git.git_dir_abs, self._config.state_dir)
return os.path.join(self._git.git_dir_abs, ConfigI.STATE_DIR)
def move_data_to_cache(self):
md5 = file_md5(self.data.relative)[0]
self._cache_file = os.path.join(self.cache_dir, md5)
self._cache_file = os.path.join(self.cache_dir_abs, md5)
self._git.modify_gitignore([self.data.relative])
if not os.path.isfile(self.cache.relative):
System.hardlink(self.data.relative, self.cache.relative)
dvc/path/factory.py
+5 -3
View File
@@ -1,5 +1,6 @@
import os
from dvc.config import ConfigI
from dvc.path.data_item import DataItem, DataItemError, DataDirError
from dvc.path.path import Path
from dvc.path.stated_data_item import StatedDataItem
@@ -30,7 +31,7 @@ class PathFactory(object):
def is_data_item(self, fname):
data = os.path.relpath(os.path.realpath(fname), self._git.git_dir_abs)
state = os.path.join(self._config.state_dir, data + DataItem.STATE_FILE_SUFFIX)
state = os.path.join(ConfigI.STATE_DIR, data + DataItem.STATE_FILE_SUFFIX)
return os.path.isfile(state)
def existing_data_item(self, fname):
@@ -55,7 +56,7 @@ class PathFactory(object):
def all_existing_data_items(self, subdir='.'):
states = []
for root, dirs, files in os.walk(os.path.join(self._config.state_dir, subdir)):
for root, dirs, files in os.walk(os.path.join(ConfigI.STATE_DIR, subdir)):
for fname in files:
path = os.path.join(root, fname)
@@ -71,5 +72,6 @@ class PathFactory(object):
return map(lambda s: self.data_item_from_dvc_path(self.state_path_to_dvc_path(s), existing),
states)
@staticmethod
def state_path_to_dvc_path(self, state):
return os.path.relpath(state, self._config.state_dir)[:-len(DataItem.STATE_FILE_SUFFIX)]
return os.path.relpath(state, ConfigI.STATE_DIR)[:-len(DataItem.STATE_FILE_SUFFIX)]