Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-06 15:28:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #295 from dataversioncontrol/dvc_checkout_witout_git_checkout

Browse Source 
Dvc checkout without git checkout and STATE_DIR refactoring
This commit is contained in:
Dmitry Petrov
2017-11-13 02:23:46 -08:00
committed by GitHub
parent 0cb7ec6179 b4bf6ffbd7
commit fabae6d1ce
14 changed files with 106 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/checkout.py
+14 -13
View File
@@ -35,16 +35,12 @@ class CmdCheckout(CmdBase):
def run(self):
with DvcLock(self.is_locker, self.git):
curr_commit = self.git.curr_branch_or_commit
prev_items = []
try:
self.git.checkout_previous()
prev_items = self.settings.path_factory.all_existing_data_items()
states = self.git.state_files_for_previous_commit()
prev_items = self.settings.path_factory.data_items_from_states(states, existing=False)
except Exception as ex:
Logger.error(u'Unable to get '.format(ex))
Logger.error(u'Unable to get data files from previous commit: {}'.format(ex))
return 1
finally:
self.git.checkout(curr_commit)
curr_items = self.settings.path_factory.all_existing_data_items()
self.checkout(curr_items)
@@ -55,10 +51,15 @@ class CmdCheckout(CmdBase):
@staticmethod
def remove_files(removed_items_set):
for item in removed_items_set:
Logger.info(u'Remove \'{}\''.format(item.data.relative))
os.remove(item.data.relative)
dir = os.path.dirname(item.data.relative)
if not os.listdir(dir):
Logger.info(u'Remove directory \'{}\''.format(dir))
os.removedirs(dir)
path = item.data.relative
if not os.path.exists(path):
print(u'Remove \'{}\'.'.format(path))
Logger.warn(u'File \'{}\' does not exist'.format(path))
else:
Logger.info(u'Remove \'{}\''.format(path))
os.remove(path)
dir = os.path.dirname(path)
if not os.listdir(dir):
Logger.info(u'Remove directory \'{}\''.format(dir))
os.removedirs(dir)
pass
dvc/command/gc.py
+4 -2
View File
@@ -4,6 +4,8 @@ from dvc.command.base import DvcLock
from dvc.command.traverse import Traverse
from dvc.logger import Logger
from dvc.path.data_item import DataItemError
from dvc.config import ConfigI
class CmdGC(Traverse):
def __init__(self, settings):
@@ -16,8 +18,8 @@ class CmdGC(Traverse):
Logger.error('Failed to collect used cache')
return 1
for cache in os.listdir(self.settings.config.cache_dir):
fname = os.path.join(self.settings.config.cache_dir, cache)
for cache in os.listdir(ConfigI.CACHE_DIR):
fname = os.path.join(ConfigI.CACHE_DIR, cache)
if fname in self.clist:
continue
os.remove(fname)
dvc/command/import_file.py
+2 -2
View File
@@ -107,8 +107,8 @@ class CmdImportFile(CmdBase):
raise ImportFileError('Output file directory "{}" does not exists'.format(
os.path.dirname(data_item.data.relative)))
if not os.path.exists(data_item.cache_dir):
os.makedirs(data_item.cache_dir)
if not os.path.exists(data_item.cache_dir_abs):
os.makedirs(data_item.cache_dir_abs)
data_targets.append((input, data_item))
dvc/command/init.py
+2 -2
View File
@@ -85,8 +85,8 @@ ProjectName =
return 1
config_dir_path = self.get_not_existing_path(Config.CONFIG_DIR)
cache_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.CACHE_DIR)
state_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.STATE_DIR)
cache_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.CACHE_DIR_NAME)
state_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.STATE_DIR_NAME)
conf_file_name = self.get_not_existing_conf_file_name()
dvc/command/merge.py
+3 -2
View File
@@ -2,6 +2,7 @@ import os
import json
from dvc.command.base import CmdBase, DvcLock
from dvc.config import ConfigI
from dvc.logger import Logger
from dvc.state_file import StateFile
from dvc.path.data_item import DataItem
@@ -20,8 +21,8 @@ class CmdMerge(CmdBase):
dlist = []
flist = self.git.get_last_merge_changed_files()
for fname in flist:
if fname.startswith(self.settings.config.state_dir) and fname.endswith(DataItem.STATE_FILE_SUFFIX):
data = os.path.relpath(fname, self.settings.config.state_dir)[:-len(DataItem.STATE_FILE_SUFFIX)]
if fname.startswith(ConfigI.STATE_DIR) and fname.endswith(DataItem.STATE_FILE_SUFFIX):
data = os.path.relpath(fname, ConfigI.STATE_DIR)[:-len(DataItem.STATE_FILE_SUFFIX)]
dlist.append(data)
return dlist
dvc/command/repro.py
+1 -1
View File
@@ -156,7 +156,7 @@ class ReproChange(object):
Logger.debug('Removing output file {} before reproduction.'.format(output_dvc))
try:
data_item = self.cmd_obj.settings.path_factory.existing_data_item_from_dvc_path(output_dvc)
data_item = self.cmd_obj.settings.path_factory.data_item_from_dvc_path(output_dvc)
os.remove(data_item.data.relative)
except Exception as ex:
msg = 'Data item {} cannot be removed before reproduction: {}'
dvc/command/run.py
+1 -1
View File
@@ -117,7 +117,7 @@ class CmdRun(CmdBase):
result = []
for data_item in repo_change.changed_data_items:
Logger.debug('Move output file "{}" to cache dir "{}" and create a hardlink'.format(
data_item.data.relative, data_item.cache_dir))
data_item.data.relative, data_item.cache_dir_abs))
data_item.move_data_to_cache()
Logger.debug('Create state file "{}"'.format(data_item.state.relative))
dvc/config.py
+14 -11
View File
@@ -19,8 +19,11 @@ class ConfigI(object):
CONFIG_DIR = '.dvc'
TARGET_FILE_DEFAULT = 'target'
CONFIG = 'config'
STATE_DIR = 'state'
CACHE_DIR = 'cache'
STATE_DIR_NAME = 'state'
CACHE_DIR_NAME = 'cache'
CACHE_DIR = os.path.join(CONFIG_DIR, CACHE_DIR_NAME)
STATE_DIR = os.path.join(CONFIG_DIR, STATE_DIR_NAME)
def __init__(self, cloud=None, conf_parser=None):
self._conf_parser = None
@@ -32,15 +35,15 @@ class ConfigI(object):
self._cloud = cloud
self._conf_parser = conf_parser
@property
def cache_dir(self):
""" Directory with cached data files """
return os.path.join(self.CONFIG_DIR, self.CACHE_DIR)
@property
def state_dir(self):
""" Directory with state files """
return os.path.join(self.CONFIG_DIR, self.STATE_DIR)
# @classmethod
# def CACHE_DIR(self):
# """ Directory with cached data files """
# return os.path.join(self.CONFIG_DIR, self.CACHE_DIR_NAME)
#
# @classmethod
# def STATE_DIR(self):
# """ Directory with state files """
# return os.path.join(self.CONFIG_DIR, self.STATE_DIR_NAME)
@property
def cloud(self):
dvc/git_wrapper.py
+22 -1
View File
@@ -4,7 +4,7 @@ from collections import defaultdict
from dvc.exceptions import DvcException
from dvc.logger import Logger
from dvc.config import Config
from dvc.config import Config, ConfigI
from dvc.executor import Executor, ExecutorError
from dvc.path.data_item import DataItemError
from dvc.state_file import StateFile
@@ -423,3 +423,24 @@ class GitWrapper(GitWrapperI):
for (hash, branch) in items:
result[hash].append(branch)
return result
@staticmethod
def git_prev_commit():
cmd = 'git check-ref-format --branch @{-1}'
code, out, err = Executor.exec_cmd(cmd.split())
if code != 0:
return ''
return out
@staticmethod
def state_files_for_previous_commit():
commit = GitWrapper.git_prev_commit()
Logger.debug(u'[dvc-git] Previous commit "{}"'.format(commit))
if not commit:
Logger.debug(u'[dvc-git] Previous commit does not exist')
return []
git_cmd = u'git ls-tree --name-only -r {}'.format(commit)
lines = Executor.exec_cmd_only_success(git_cmd.split()).split('\n')
return filter(lambda s: s.startswith(ConfigI.STATE_DIR), lines)
dvc/path/data_item.py
+7 -7
View File
@@ -1,7 +1,7 @@
import os
import stat
import shutil
from dvc.config import ConfigI
from dvc.path.path import Path
from dvc.exceptions import DvcException
from dvc.system import System
@@ -71,17 +71,17 @@ class DataItem(object):
@cached_property
def state(self):
state_dir = os.path.join(self._git.git_dir_abs, self._config.state_dir)
state_dir = os.path.join(self._git.git_dir_abs, ConfigI.STATE_DIR)
state_file = os.path.join(state_dir, self.data.dvc + self.STATE_FILE_SUFFIX)
return Path(state_file, self._git)
@cached_property
def cache_dir(self):
return os.path.join(self._git.git_dir_abs, self._config.cache_dir)
def cache_dir_abs(self):
return os.path.join(self._git.git_dir_abs, ConfigI.CACHE_DIR)
@cached_property
def cache(self):
cache_dir = self.cache_dir
cache_dir = self.cache_dir_abs
if self._cache_file:
file_name = os.path.relpath(os.path.realpath(self._cache_file), cache_dir)
@@ -94,11 +94,11 @@ class DataItem(object):
@cached_property
def state_dir_abs(self):
return os.path.join(self._git.git_dir_abs, self._config.state_dir)
return os.path.join(self._git.git_dir_abs, ConfigI.STATE_DIR)
def move_data_to_cache(self):
md5 = file_md5(self.data.relative)[0]
self._cache_file = os.path.join(self.cache_dir, md5)
self._cache_file = os.path.join(self.cache_dir_abs, md5)
self._git.modify_gitignore([self.data.relative])
if not os.path.isfile(self.cache.relative):
System.hardlink(self.data.relative, self.cache.relative)
dvc/path/factory.py
+16 -9
View File
@@ -1,5 +1,6 @@
import os
from dvc.config import ConfigI
from dvc.path.data_item import DataItem, DataItemError, DataDirError
from dvc.path.path import Path
from dvc.path.stated_data_item import StatedDataItem
@@ -21,13 +22,16 @@ class PathFactory(object):
def stated_data_item(self, state, data_file):
return StatedDataItem(state, data_file, self._git, self._config)
def existing_data_item_from_dvc_path(self, dvc_path):
def data_item_from_dvc_path(self, dvc_path, existing=True):
path = Path.from_dvc_path(dvc_path, self._git)
return self.existing_data_item(path.relative)
if existing:
return self.existing_data_item(path.relative)
else:
return self.data_item(path.relative)
def is_data_item(self, fname):
data = os.path.relpath(os.path.realpath(fname), self._git.git_dir_abs)
state = os.path.join(self._config.state_dir, data + DataItem.STATE_FILE_SUFFIX)
state = os.path.join(ConfigI.STATE_DIR, data + DataItem.STATE_FILE_SUFFIX)
return os.path.isfile(state)
def existing_data_item(self, fname):
@@ -50,10 +54,9 @@ class PathFactory(object):
return result, externally_created_files
def all_existing_data_items(self, subdir='.'):
items = []
states = []
for root, dirs, files in os.walk(os.path.join(self._config.state_dir, subdir)):
for root, dirs, files in os.walk(os.path.join(ConfigI.STATE_DIR, subdir)):
for fname in files:
path = os.path.join(root, fname)
@@ -63,8 +66,12 @@ class PathFactory(object):
states.append(path)
for state in states:
data = os.path.relpath(state, self._config.state_dir)[:-len(DataItem.STATE_FILE_SUFFIX)]
items.append(self.existing_data_item_from_dvc_path(data))
return self.data_items_from_states(states)
return items
def data_items_from_states(self, states, existing=True):
return map(lambda s: self.data_item_from_dvc_path(self.state_path_to_dvc_path(s), existing),
states)
@staticmethod
def state_path_to_dvc_path(state):
return os.path.relpath(state, ConfigI.STATE_DIR)[:-len(DataItem.STATE_FILE_SUFFIX)]
tests/basic_env.py
+5 -5
View File
@@ -34,8 +34,8 @@ class BasicEnvironment(TestCase):
os.makedirs(self._test_git_dir)
data_dir = os.path.join(self._test_git_dir, 'data')
cache_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR)
state_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data')
cache_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME)
state_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data')
os.makedirs(data_dir)
os.makedirs(cache_dir)
@@ -103,13 +103,13 @@ class DirHierarchyEnvironment(BasicEnvironment):
def crate_data_item(self, data_file, cache_file=True, content='random text'):
file_result = os.path.join('data', data_file)
state_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, file_result + DataItem.STATE_FILE_SUFFIX)
state_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, file_result + DataItem.STATE_FILE_SUFFIX)
dummy_md5 = os.path.basename(data_file) + DataItem.CACHE_FILE_SEP + self._commit
self.create_content_file(state_result, '{"Md5" : "' + dummy_md5 + '", "Command" : "run", "Cwd" : "dir"}')
if cache_file:
cache_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dummy_md5)
cache_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, dummy_md5)
self.create_content_file(cache_result, content)
System.hardlink(cache_result, file_result)
@@ -121,4 +121,4 @@ class DirHierarchyEnvironment(BasicEnvironment):
def create_dirs(self, dir):
os.makedirs(os.path.join('data', dir))
os.makedirs(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data', dir))
os.makedirs(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', dir))
tests/test_cmd_run.py
+5 -5
View File
@@ -23,8 +23,8 @@ class RunBasicTest(TestCase):
os.chdir(self.test_dir)
os.mkdir('data')
os.mkdir(ConfigI.CONFIG_DIR)
os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR))
os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR))
os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME))
os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME))
self.init_git_repo()
self.git = GitWrapper()
@@ -37,7 +37,7 @@ class RunBasicTest(TestCase):
def init_git_repo(self):
Executor.exec_cmd_only_success(['git', 'init'])
msg = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR)
msg = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME)
msg += '\n'
msg += os.path.join(ConfigI.CONFIG_DIR, '.' + ConfigI.CONFIG + '.lock')
self.create_file('.gitignore', msg)
@@ -72,8 +72,8 @@ class RunTwoFilesBase(RunBasicTest):
stdout=self.file_name1,
stderr=self.file_name2)
self.state_file_name1 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, self.file_name1 + DataItem.STATE_FILE_SUFFIX)
self.state_file_name2 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, self.file_name2 + DataItem.STATE_FILE_SUFFIX)
self.state_file_name1 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, self.file_name1 + DataItem.STATE_FILE_SUFFIX)
self.state_file_name2 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, self.file_name2 + DataItem.STATE_FILE_SUFFIX)
self.state_file1 = None
self.state_file2 = None
tests/test_data_file_obj.py
+10 -10
View File
@@ -16,7 +16,7 @@ class TestDataFileObjBasic(BasicEnvironment):
config = ConfigI()
file = os.path.join('data', 'file.txt')
self.cache_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'file.txt_ad45ba8')
self.cache_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file.txt_ad45ba8')
self._data_path = DataItem(file, git, config, cache_file=self.cache_file_full_path)
pass
@@ -28,7 +28,7 @@ class TestDataFileObjBasic(BasicEnvironment):
self.assertEqual(self._data_path.cache.abs, self.cache_file_full_path)
def test_state_file(self):
state_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data', 'file.txt.state')
state_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file.txt.state')
self.assertEqual(self._data_path.state.abs, state_file_full_path)
@@ -40,7 +40,7 @@ class TestDataItemWithGivenCache(BasicEnvironment):
self._config = ConfigI('data')
self._file = os.path.join('data', 'file.txt')
self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'file.txt_abcd')
self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file.txt_abcd')
self._data_item = DataItem(self._file, self._git, self._config, self._cache)
pass
@@ -58,7 +58,7 @@ class TestDeepDataItemWithGivenCache(BasicEnvironment):
self._config = ConfigI('data')
self._file = os.path.join('data', 'dir1', 'file.txt')
self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'dir1', 'file.txt_abcd')
self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'dir1', 'file.txt_abcd')
self._data_item = DataItem(self._file, self._git, self._config, self._cache)
pass
@@ -78,8 +78,8 @@ class TestPathFactory(BasicEnvironment):
self.data_file = os.path.join('data', 'file.txt')
dummy_md5 = 'fsymlinc.txt'
self.cache_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dummy_md5)
self.state_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data', 'file.txt' + DataItem.STATE_FILE_SUFFIX)
self.cache_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, dummy_md5)
self.state_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file.txt' + DataItem.STATE_FILE_SUFFIX)
with open(self.cache_file, 'w+') as fd:
fd.write('some text')
@@ -134,7 +134,7 @@ class TestDataPathInDataDir(BasicEnvironment):
self.path_factory = PathFactory(git, config)
deep_path = os.path.join(self.data_dir, 'dir1', 'd2', 'file.txt')
self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'dir1', 'd2', 'file.txt_eeeff8f')
self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'dir1', 'd2', 'file.txt_eeeff8f')
self.data_path = self.path_factory.data_item(deep_path, cache_file=self.cache_file)
pass
@@ -146,7 +146,7 @@ class TestDataPathInDataDir(BasicEnvironment):
self.assertEqual(self.data_path.cache.abs, self.cache_file)
def test_state_file(self):
target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'da', 'dir1', 'd2', 'file.txt.state')
target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'da', 'dir1', 'd2', 'file.txt.state')
self.assertEqual(self.data_path.state.abs, target)
@@ -162,7 +162,7 @@ class TestDataFileObjLongPath(BasicEnvironment):
self._config = ConfigI('data')
self.path_factory = PathFactory(self._git, self._config)
self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'file1.txt_123ed8')
self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file1.txt_123ed8')
self.data_path = self.path_factory.data_item(os.path.join('..', '..', 'data', 'file1.txt'), cache_file=self.cache_file)
pass
@@ -174,7 +174,7 @@ class TestDataFileObjLongPath(BasicEnvironment):
self.assertEqual(self.data_path.cache.abs, self.cache_file)
def test_state_file(self):
target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data', 'file1.txt.state')
target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file1.txt.state')
self.assertEqual(self.data_path.state.abs, target)