mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-09 23:10:24 +01:00
Merge pull request #388 from efiop/master
Misc tests, fixes and error handling
This commit is contained in:
Ruslan Kuprieiev
@@ -0,0 +1,7 @@
|
||||
[run]
|
||||
omit =
|
||||
*__main__.py
|
||||
|
||||
[report]
|
||||
exclude_lines =
|
||||
if __name__ == .__main__.:
|
||||
@@ -1,4 +1,6 @@
|
||||
import sys
|
||||
|
||||
from dvc.main import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main(sys.argv[1:])
|
||||
|
||||
+3
-1
@@ -1,4 +1,6 @@
|
||||
import sys
|
||||
|
||||
from dvc.main import main
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
main(sys.argv[1:])
|
||||
|
||||
+6
-28
@@ -39,12 +39,6 @@ def parse_args(argv=None):
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Be verbose.')
|
||||
parent_parser.add_argument(
|
||||
'-G',
|
||||
'--no-git-actions',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Skip all git actions including reproducibility check and commits.')
|
||||
|
||||
parent_parser.add_argument(
|
||||
'-b',
|
||||
@@ -221,20 +215,6 @@ def parse_args(argv=None):
|
||||
'gc',
|
||||
parents=[parent_parser],
|
||||
help='Collect garbage')
|
||||
gc_parser.add_argument('-l',
|
||||
'--keep-in-cloud',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Do not remove data from cloud.')
|
||||
gc_parser.add_argument('-r',
|
||||
'--recursive',
|
||||
action='store_true',
|
||||
help='Remove directory recursively.')
|
||||
gc_parser.add_argument('-c',
|
||||
'--keep-in-cache',
|
||||
action='store_false',
|
||||
default=False,
|
||||
help='Do not remove data from cache.')
|
||||
gc_parser.set_defaults(func=CmdGC)
|
||||
|
||||
# Cloud
|
||||
@@ -339,12 +319,13 @@ def parse_args(argv=None):
|
||||
parents=[parent_parser],
|
||||
help='Show graphs')
|
||||
show_subparsers = show_parser.add_subparsers(
|
||||
dest='cmd',
|
||||
help='Use `dvc show CMD` --help for command-specific help')
|
||||
dest='cmd',
|
||||
help='Use `dvc show CMD` --help for command-specific help')
|
||||
|
||||
pipeline_parser = show_subparsers.add_parser(
|
||||
'pipeline',
|
||||
parents=[parent_parser],
|
||||
help='Show pipeline image')
|
||||
'pipeline',
|
||||
parents=[parent_parser],
|
||||
help='Show pipeline image')
|
||||
pipeline_parser.add_argument(
|
||||
'target',
|
||||
nargs='*',
|
||||
@@ -431,7 +412,4 @@ def parse_args(argv=None):
|
||||
help='Compute actual md5')
|
||||
fsck_parser.set_defaults(func=CmdFsck)
|
||||
|
||||
if isinstance(argv, str):
|
||||
argv = argv.split()
|
||||
|
||||
return parser.parse_args(argv)
|
||||
|
||||
+6
-1
@@ -1,8 +1,13 @@
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.command.common.base import CmdBase
|
||||
|
||||
|
||||
class CmdAdd(CmdBase):
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
self.project.add(target)
|
||||
try:
|
||||
self.project.add(target)
|
||||
except DvcException as ex:
|
||||
self.project.logger.error('Failed to add {}: {}'.format(target, str(ex)))
|
||||
return 1
|
||||
return 0
|
||||
|
||||
@@ -21,7 +21,7 @@ class CmdBase(object):
|
||||
return root
|
||||
root = os.path.dirname(root)
|
||||
msg = "Not a dvc repository (or any parent up to mount point {})"
|
||||
Logger.error(msg.format(root))
|
||||
self.project.logger.error(msg.format(root))
|
||||
|
||||
def run_cmd(self):
|
||||
with self.project.lock:
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.command.common.base import CmdBase
|
||||
|
||||
|
||||
class CmdRemove(CmdBase):
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
self.project.remove(target)
|
||||
try:
|
||||
self.project.remove(target)
|
||||
except DvcException as ex:
|
||||
self.project.logger.error('Failed to remove {}: {}'.format(target, str(ex)))
|
||||
return 1
|
||||
return 0
|
||||
|
||||
@@ -26,8 +26,7 @@ class CmdShowPipeline(CmdBase):
|
||||
|
||||
def draw_targets(self, target):
|
||||
if not target:
|
||||
target = '.'
|
||||
return self.draw(self.g, target, target)
|
||||
return self.draw(self.g, '.', 'all')
|
||||
|
||||
for t in self.args.target:
|
||||
fname_suffix = os.path.basename(os.path.normpath(t))
|
||||
@@ -59,7 +58,6 @@ class CmdShowPipeline(CmdBase):
|
||||
|
||||
def collect_stage(self, stage):
|
||||
name = os.path.relpath(stage.path, self.project.root_dir)
|
||||
state = StateFile.load(data_item, self.git)
|
||||
|
||||
self.g.add_node(name)
|
||||
|
||||
@@ -68,7 +66,7 @@ class CmdShowPipeline(CmdBase):
|
||||
self.g.add_node(i)
|
||||
self.g.add_edge(i, name)
|
||||
|
||||
for out in state.outs:
|
||||
for out in stage.outs:
|
||||
o = os.path.relpath(out.path, self.project.root_dir)
|
||||
self.g.add_node(o)
|
||||
self.g.add_edge(name, o)
|
||||
|
||||
@@ -1,16 +1,87 @@
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
from dvc.logger import Logger
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.git_wrapper import GitWrapper
|
||||
from dvc.graph.workflow import Workflow
|
||||
from dvc.graph.commit import Commit
|
||||
from dvc.executor import Executor, ExecutorError
|
||||
|
||||
|
||||
class CmdShowWorkflow(CmdBase):
|
||||
COMMIT_LEN = 7
|
||||
LOG_SEPARATOR = '|'
|
||||
LOG_FORMAT = ['%h', '%p', '%an', '%ai', '%s']
|
||||
|
||||
def get_all_commits(self, target):
|
||||
# git log --all --abbrev=7 --pretty=format:"%h|%p|%an|%ai|%s"
|
||||
try:
|
||||
merges_map = self.get_merges_map()
|
||||
|
||||
format_str = self.LOG_SEPARATOR.join(self.LOG_FORMAT)
|
||||
git_cmd = ['git', 'log', '--all', '--abbrev={}'.format(self.COMMIT_LEN),
|
||||
'--pretty=format:{}'.format(format_str)]
|
||||
lines = Executor.exec_cmd_only_success(git_cmd).split('\n')
|
||||
|
||||
branches_multimap = self.branches_multimap()
|
||||
|
||||
wf = Workflow(target, merges_map, branches_multimap)
|
||||
for line in lines:
|
||||
items = line.split(self.LOG_SEPARATOR, len(self.LOG_FORMAT))
|
||||
assert len(items) == 5, 'Git wrapper: git log format has {} items, 5 expected'.format(len(items))
|
||||
hash, parent_hash, name, date, comment = items
|
||||
|
||||
commit = Commit(hash, parent_hash, name, date, comment,
|
||||
*self.was_target_changed(hash, target),
|
||||
branch_tips=branches_multimap.get(hash))
|
||||
wf.add_commit(commit)
|
||||
|
||||
return wf
|
||||
except ExecutorError:
|
||||
raise
|
||||
|
||||
def was_target_changed(self, hash, target):
|
||||
git_cmd = ['git', 'show', '--pretty=', '--name-only', hash]
|
||||
changed_files = set(Executor.exec_cmd_only_success(git_cmd).split('\n'))
|
||||
|
||||
if target not in changed_files:
|
||||
return False, None
|
||||
|
||||
metric = utils.parse_target_metric_file(target)
|
||||
if metric is None:
|
||||
return False, None
|
||||
|
||||
return True, metric
|
||||
|
||||
def get_merges_map(self):
|
||||
# git log --merges --all --abbrev=7
|
||||
# {'a4b56f1': back_to_600_est}
|
||||
git_cmd = ['git', 'log', '--all', '--merges',
|
||||
'--abbrev={}'.format(self.COMMIT_LEN)]
|
||||
# lines = map(str.strip, Executor.exec_cmd_only_success(git_cmd).split('\n'))
|
||||
# lines.map
|
||||
return {}
|
||||
|
||||
def branches_multimap(self):
|
||||
git_cmd = ['git', 'show-ref', '--abbrev={}'.format(self.COMMIT_LEN)]
|
||||
lines = Executor.exec_cmd_only_success(git_cmd).split('\n')
|
||||
items_full = map(unicode.split, lines)
|
||||
items = map(lambda it: (it[0], re.sub(r'^refs/heads/', '', it[1])), items_full)
|
||||
|
||||
result = defaultdict(list)
|
||||
for (hash, branch) in items:
|
||||
result[hash].append(branch)
|
||||
return result
|
||||
|
||||
def run(self):
|
||||
target = self.args.target
|
||||
if not target:
|
||||
target = self.project.config._config['Global'].get('Target', '')
|
||||
self.project.logger.debug(u'Set show workflow target as {}'.format(target))
|
||||
|
||||
wf = GitWrapper.get_all_commits(target, self.settings)
|
||||
wf = self.get_all_commits(target)
|
||||
wf.build_graph(self.args.dvc_commits,
|
||||
self.args.all_commits,
|
||||
self.args.max_commits)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -62,7 +62,6 @@ ProjectName =
|
||||
|
||||
self._config = configparser.SafeConfigParser()
|
||||
self._config.read(self.config_file)
|
||||
Logger.set_level(self._config['Global']['LogLevel'])
|
||||
|
||||
@staticmethod
|
||||
def init(dvc_dir):
|
||||
|
||||
@@ -1,448 +0,0 @@
|
||||
import os
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
from dvc import utils
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
from dvc.config import Config
|
||||
from dvc.executor import Executor, ExecutorError
|
||||
from dvc.system import System
|
||||
from dvc.graph.workflow import Workflow
|
||||
from dvc.graph.commit import Commit
|
||||
|
||||
|
||||
class GitWrapperError(DvcException):
|
||||
def __init__(self, msg):
|
||||
DvcException.__init__(self, msg)
|
||||
|
||||
|
||||
class GitWrapperI(object):
|
||||
COMMIT_LEN = 7
|
||||
|
||||
def __init__(self, git_dir=None, commit=None):
|
||||
self._git_dir = git_dir
|
||||
self._commit = commit
|
||||
|
||||
@property
|
||||
def git_dir(self):
|
||||
return self._git_dir
|
||||
|
||||
@property
|
||||
def lock_file(self):
|
||||
return os.path.join(self.git_dir_abs, Config.CONFIG_DIR, '.' + Config.CONFIG + '.lock')
|
||||
|
||||
@property
|
||||
def git_dir_abs(self):
|
||||
return System.realpath(self.git_dir)
|
||||
|
||||
@property
|
||||
def curr_dir_abs(self):
|
||||
return os.path.abspath(os.curdir)
|
||||
|
||||
@property
|
||||
def curr_commit(self):
|
||||
return self._commit
|
||||
|
||||
def is_ready_to_go(self):
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def git_file_statuses():
|
||||
Logger.debug('[dvc-git] Getting file statuses. Command: git status --porcelain')
|
||||
code, out, err = Executor.exec_cmd(['git', 'status', '--porcelain'])
|
||||
if code != 0:
|
||||
raise ExecutorError('[dvc-git] File status command error - {}'.format(err))
|
||||
Logger.debug('[dvc-git] Getting file statuses. Success.')
|
||||
|
||||
return GitWrapper.parse_porcelain_files(out)
|
||||
|
||||
@staticmethod
|
||||
def git_config_get(name):
|
||||
code, out, err = Executor.exec_cmd(['git', 'config', '--get', name])
|
||||
Logger.debug('[dvc-git] "git config --get {}": code({}), out({}), err({})'.format(name,
|
||||
code,
|
||||
out,
|
||||
err))
|
||||
if code != 0:
|
||||
return None
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def git_path_to_system_path(path):
|
||||
if os.name == 'nt':
|
||||
return path.replace('/', '\\')
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def parse_porcelain_files(out):
|
||||
result = []
|
||||
if len(out) > 0:
|
||||
lines = out.split('\n')
|
||||
for line in lines:
|
||||
status = line[:2]
|
||||
file = GitWrapperI.git_path_to_system_path(line[3:])
|
||||
result.append((status, file))
|
||||
return result
|
||||
|
||||
def abs_paths_to_dvc(self, files):
|
||||
result = []
|
||||
for file in files:
|
||||
result.append(os.path.relpath(os.path.abspath(file), self.git_dir_abs))
|
||||
|
||||
return result
|
||||
|
||||
def commit_all_changes_and_log_status(self, message):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _file_contains(fname, s):
|
||||
if not os.path.isfile(fname):
|
||||
return False
|
||||
|
||||
with open(fname, 'r') as fd:
|
||||
if s in fd.read():
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def modify_gitignore(flist):
|
||||
for fname in flist:
|
||||
# We're modifying .gitignore in the same dir as the file itself, so
|
||||
# that if dir consists only of data files it will be still tracked by
|
||||
# git and we will not have any problems with --porcelain output for
|
||||
# specific changed files.
|
||||
gitignore = os.path.join(os.path.dirname(fname), '.gitignore')
|
||||
entry = os.path.basename(fname)
|
||||
|
||||
if GitWrapperI._file_contains(gitignore, entry + '\n'):
|
||||
return
|
||||
|
||||
with open(gitignore, 'a') as fd:
|
||||
fd.write('\n' + entry)
|
||||
|
||||
|
||||
class GitWrapper(GitWrapperI):
|
||||
FLOATS_FROM_STRING = re.compile(r'[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?')
|
||||
|
||||
def __init__(self):
|
||||
super(GitWrapper, self).__init__()
|
||||
|
||||
def is_inside_work_tree(self):
|
||||
code, out, err = Executor.exec_cmd(['git', 'rev-parse', '--is-inside-work-tree'])
|
||||
if code != 0:
|
||||
Logger.error('[dvc-git] Not inside git tree');
|
||||
return code == 0
|
||||
|
||||
def is_ready_to_go(self):
|
||||
if not self.is_inside_work_tree():
|
||||
return False
|
||||
|
||||
statuses = self.git_file_statuses()
|
||||
if len(statuses) > 0:
|
||||
Logger.error('[dvc-git] Commit all changed files before running '
|
||||
'reproducible command. Changed files:')
|
||||
for status, file in statuses:
|
||||
Logger.error("{} {}".format(status, file))
|
||||
return False
|
||||
|
||||
return self.check_config()
|
||||
|
||||
def check_config(self):
|
||||
# Sanity check to make sure we will be able to commit
|
||||
name = self.git_config_get('user.name')
|
||||
if name == None:
|
||||
Logger.error('[dvc-git] Please setup user.name in git config')
|
||||
return False
|
||||
|
||||
email = self.git_config_get('user.email')
|
||||
if email == None:
|
||||
Logger.error('[dvc-git] Please setup user.email in git config')
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@property
|
||||
def curr_dir_dvc(self):
|
||||
return os.path.relpath(os.path.abspath(os.curdir), self.git_dir_abs)
|
||||
|
||||
@property
|
||||
def git_dir(self):
|
||||
if self._git_dir:
|
||||
return self._git_dir
|
||||
|
||||
try:
|
||||
Logger.debug('[dvc-git] Getting git directory. Command: git rev-parse --show-toplevel')
|
||||
code, out, err = Executor.exec_cmd(['git', 'rev-parse', '--show-toplevel'])
|
||||
|
||||
if code != 0:
|
||||
raise ExecutorError('[dvc-git] Git directory command error - {}'.format(err))
|
||||
Logger.debug('[dvc-git] Getting git directory. Success.')
|
||||
|
||||
self._git_dir = out
|
||||
return self._git_dir
|
||||
except ExecutorError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise ExecutorError('Unable to run git command: {}'.format(e))
|
||||
pass
|
||||
|
||||
def curr_commit(self, branches=False):
|
||||
Logger.debug('[dvc-git] Getting current git commit. Command: git rev-parse --short HEAD')
|
||||
|
||||
if branches:
|
||||
cmd = 'git rev-parse --abbrev-ref HEAD'
|
||||
else:
|
||||
cmd = 'git rev-parse HEAD'
|
||||
|
||||
code, out, err = Executor.exec_cmd(cmd.split())
|
||||
if code != 0:
|
||||
raise ExecutorError('[dvc-git] Commit command error - {}'.format(err))
|
||||
Logger.debug('[dvc-git] Getting current git commit. Success.')
|
||||
return out
|
||||
|
||||
@property
|
||||
def curr_branch_or_commit(self):
|
||||
Logger.debug('[dvc-git] Getting current git branch or commit.')
|
||||
|
||||
branch = self.curr_commit(branches=True)
|
||||
if branch == '' or branch == 'HEAD':
|
||||
result = self.curr_commit(branches=False)
|
||||
else:
|
||||
result = branch
|
||||
|
||||
Logger.debug('[dvc-git] Getting current git branch or commit. Result {}. Success.'.format(result))
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def commit_all_changes(message):
|
||||
Logger.debug('[dvc-git] Commit all changes. Commands: {} && {} && {}'.format(
|
||||
'git add --all', 'git status --porcelain', 'git commit -m'))
|
||||
|
||||
Executor.exec_cmd_only_success(['git', 'add', '--all'])
|
||||
out_status = Executor.exec_cmd_only_success(['git', 'status', '--porcelain'])
|
||||
changes = GitWrapper.parse_porcelain_files(out_status)
|
||||
if len(changes) == 0:
|
||||
return []
|
||||
|
||||
Executor.exec_cmd_only_success(['git', 'commit', '-m', '\'{}\''.format(message)])
|
||||
Logger.debug('[dvc-git] Commit all changes. Success.')
|
||||
return changes
|
||||
|
||||
def commit_all_changes_and_log_status(self, message):
|
||||
statuses = self.commit_all_changes(message)
|
||||
Logger.debug('[dvc-git] A new commit {} was made in the current branch. '
|
||||
'Added files:'.format(self.curr_commit()))
|
||||
for status, file in statuses:
|
||||
Logger.debug('[dvc-git]\t{} {}'.format(status, file))
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def abs_paths_to_relative(files):
|
||||
cur_dir = System.realpath(os.curdir)
|
||||
|
||||
result = []
|
||||
for file in files:
|
||||
result.append(os.path.relpath(System.realpath(file), cur_dir))
|
||||
|
||||
return result
|
||||
|
||||
def dvc_paths_to_abs(self, files):
|
||||
results = []
|
||||
|
||||
for file in files:
|
||||
results.append(os.path.join(self.git_dir_abs, file))
|
||||
|
||||
return results
|
||||
|
||||
def were_files_changed(self, code_dependencies, path_factory, changed_files):
|
||||
code_files, code_dirs = self.separate_dependency_files_and_dirs(code_dependencies)
|
||||
code_files_set = set([path_factory.path(x).dvc for x in code_files])
|
||||
for changed_file in changed_files:
|
||||
if changed_file in code_files_set:
|
||||
return True
|
||||
|
||||
for dir in code_dirs:
|
||||
if changed_file.startswith(dir):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_changed_files(target_commit):
|
||||
Logger.debug('[dvc-git] Identify changes. Command: '
|
||||
'git diff --name-only HEAD {}'.format(target_commit))
|
||||
|
||||
changed_files_str = Executor.exec_cmd_only_success(['git',
|
||||
'diff',
|
||||
'--name-only',
|
||||
'HEAD',
|
||||
target_commit])
|
||||
changed_files = changed_files_str.strip('"').split('\n')
|
||||
|
||||
Logger.debug('[dvc-git] Identify changes. Success. Changed files: {}'.format(
|
||||
', '.join(changed_files)))
|
||||
return changed_files
|
||||
|
||||
@staticmethod
|
||||
def get_target_commit(fname):
|
||||
Logger.debug('[dvc-git] Get target commit. Command: '
|
||||
'git log -1 --pretty=format:"%h" {}'.format(fname))
|
||||
try:
|
||||
commit = Executor.exec_cmd_only_success(['git',
|
||||
'log',
|
||||
'-1',
|
||||
'--pretty=format:"%h"',
|
||||
fname])
|
||||
return commit.strip('"')
|
||||
except ExecutorError:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_last_merge_changed_files():
|
||||
cmd = 'git diff --name-only @{1}'
|
||||
Logger.debug('[dvc-git] Get last merge changed files. Command: {}'.format(cmd))
|
||||
try:
|
||||
fstr = Executor.exec_cmd_only_success(cmd.split())
|
||||
return fstr.split('\n')
|
||||
except ExecutorError:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def checkout_file_before_last_merge(fname):
|
||||
cmd = 'git checkout @{1} -- ' + fname
|
||||
Logger.debug('[dvc-git] Checkout file {} before last merge. Command: {}'.format(fname, cmd))
|
||||
Executor.exec_cmd_only_success(cmd.split())
|
||||
|
||||
@staticmethod
|
||||
def get_file_content_before_last_merge(fname):
|
||||
return GitWrapper.get_file_content(fname, '@{1}')
|
||||
|
||||
@staticmethod
|
||||
def checkout(commit_or_branch, create_new=False):
|
||||
prefix = '-b ' if create_new else ''
|
||||
Logger.debug('[dvc-git] Checkout {}{}'.format(prefix, commit_or_branch))
|
||||
cmd = 'git checkout {}{}'.format(prefix, commit_or_branch)
|
||||
return Executor.exec_cmd(cmd.split())
|
||||
|
||||
@staticmethod
|
||||
def checkout_previous():
|
||||
Logger.debug('[dvc-git] Checkout previous')
|
||||
cmd = 'git checkout -'
|
||||
return Executor.exec_cmd(cmd.split())
|
||||
|
||||
def separate_dependency_files_and_dirs(self, code_dependencies):
|
||||
code_files = []
|
||||
code_dirs = []
|
||||
|
||||
code_dependencies_abs = self.dvc_paths_to_abs(code_dependencies)
|
||||
for code in code_dependencies_abs:
|
||||
if os.path.isdir(code):
|
||||
code_dirs.append(code)
|
||||
else:
|
||||
code_files.append(code)
|
||||
|
||||
return code_files, code_dirs
|
||||
|
||||
LOG_SEPARATOR = '|'
|
||||
LOG_FORMAT = ['%h', '%p', '%an', '%ai', '%s']
|
||||
|
||||
def get_all_commits(self, target, settings):
|
||||
# git log --all --abbrev=7 --pretty=format:"%h|%p|%an|%ai|%s"
|
||||
try:
|
||||
merges_map = GitWrapper.get_merges_map()
|
||||
|
||||
format_str = GitWrapper.LOG_SEPARATOR.join(GitWrapper.LOG_FORMAT)
|
||||
git_cmd = ['git', 'log', '--all', '--abbrev={}'.format(GitWrapper.COMMIT_LEN),
|
||||
'--pretty=format:{}'.format(format_str)]
|
||||
lines = Executor.exec_cmd_only_success(git_cmd).split('\n')
|
||||
|
||||
branches_multimap = GitWrapper.branches_multimap()
|
||||
|
||||
wf = Workflow(target, merges_map, branches_multimap)
|
||||
for line in lines:
|
||||
items = line.split(GitWrapper.LOG_SEPARATOR, len(GitWrapper.LOG_FORMAT))
|
||||
assert len(items) == 5, 'Git wrapper: git log format has {} items, 5 expected'.format(len(items))
|
||||
hash, parent_hash, name, date, comment = items
|
||||
|
||||
commit = Commit(hash, parent_hash, name, date, comment,
|
||||
*self.was_target_changed(hash, target, settings),
|
||||
branch_tips=branches_multimap.get(hash))
|
||||
wf.add_commit(commit)
|
||||
|
||||
return wf
|
||||
except ExecutorError:
|
||||
raise
|
||||
|
||||
def was_target_changed(self, hash, target, settings):
|
||||
git_cmd = ['git', 'show', '--pretty=', '--name-only', hash]
|
||||
changed_files = set(Executor.exec_cmd_only_success(git_cmd).split('\n'))
|
||||
|
||||
if target not in changed_files:
|
||||
return False, None
|
||||
|
||||
metric = utils.parse_target_metric_file(target)
|
||||
if metric is None:
|
||||
return False, None
|
||||
|
||||
return True, metric
|
||||
|
||||
@staticmethod
|
||||
def get_merges_map():
|
||||
# git log --merges --all --abbrev=7
|
||||
# {'a4b56f1': back_to_600_est}
|
||||
git_cmd = ['git', 'log', '--all', '--merges',
|
||||
'--abbrev={}'.format(GitWrapper.COMMIT_LEN)]
|
||||
# lines = map(str.strip, Executor.exec_cmd_only_success(git_cmd).split('\n'))
|
||||
# lines.map
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def branches_multimap():
|
||||
git_cmd = ['git', 'show-ref', '--abbrev={}'.format(GitWrapper.COMMIT_LEN)]
|
||||
lines = Executor.exec_cmd_only_success(git_cmd).split('\n')
|
||||
items_full = map(unicode.split, lines)
|
||||
items = map(lambda it: (it[0], re.sub(r'^refs/heads/', '', it[1])), items_full)
|
||||
|
||||
result = defaultdict(list)
|
||||
for (hash, branch) in items:
|
||||
result[hash].append(branch)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def git_prev_commit():
|
||||
cmd = 'git check-ref-format --branch @{-1}'
|
||||
code, out, err = Executor.exec_cmd(cmd.split())
|
||||
if code != 0:
|
||||
return ''
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def branches(regexp):
|
||||
cmd = 'git branch --format "%(refname)"'
|
||||
lines = Executor.exec_cmd_only_success(cmd.split()).split('\n')
|
||||
lines = map(lambda l: l.strip('"'), lines)
|
||||
|
||||
if not regexp or '/' not in regexp:
|
||||
# remove remotes like 'refs/heads'
|
||||
lines = map(lambda s: s.split('/')[-1], lines)
|
||||
|
||||
if not regexp:
|
||||
return lines
|
||||
|
||||
reg = re.compile(regexp)
|
||||
return list(filter(reg.search, lines))
|
||||
|
||||
@staticmethod
|
||||
def get_file_content(fname, branch):
|
||||
cmd = 'git show {}:{}'.format(branch, fname)
|
||||
return Executor.exec_cmd_only_success(cmd.split())
|
||||
|
||||
@staticmethod
|
||||
def all_untracked_files():
|
||||
cmd = 'git ls-files --others --exclude-standard'
|
||||
res = Executor.exec_cmd_only_success(cmd.split())
|
||||
if not res.strip():
|
||||
return []
|
||||
return res.split('\n')
|
||||
+29
-10
@@ -3,9 +3,12 @@ import logging
|
||||
|
||||
import colorama
|
||||
|
||||
|
||||
colorama.init()
|
||||
|
||||
|
||||
class Logger(object):
|
||||
FMT = '%(message)s'
|
||||
DEFAULT_LEVEL = logging.INFO
|
||||
|
||||
LEVEL_MAP = {
|
||||
@@ -21,21 +24,36 @@ class Logger(object):
|
||||
'error': colorama.Fore.RED
|
||||
}
|
||||
|
||||
logging.basicConfig(stream=sys.stdout, format='%(message)s', level=DEFAULT_LEVEL)
|
||||
def __init__(self, config=None):
|
||||
sh = logging.StreamHandler(sys.stdout)
|
||||
sh.setFormatter(logging.Formatter(self.FMT))
|
||||
sh.setLevel(logging.DEBUG)
|
||||
|
||||
_logger = logging.getLogger('dvc')
|
||||
self.logger().addHandler(sh)
|
||||
level = None
|
||||
if config:
|
||||
level = config['Global'].get('LogLevel', None)
|
||||
self.set_level(level)
|
||||
|
||||
@staticmethod
|
||||
def set_level(level):
|
||||
Logger._logger.setLevel(Logger.LEVEL_MAP.get(level.lower(), logging.DEBUG))
|
||||
def logger():
|
||||
return logging.getLogger('dvc')
|
||||
|
||||
@staticmethod
|
||||
def set_level(level=None):
|
||||
if not level:
|
||||
lvl = Logger.DEFAULT_LEVEL
|
||||
else:
|
||||
lvl = Logger.LEVEL_MAP.get(level.lower(), Logger.DEFAULT_LEVEL)
|
||||
Logger.logger().setLevel(lvl)
|
||||
|
||||
@staticmethod
|
||||
def be_quiet():
|
||||
Logger._logger.setLevel(logging.CRITICAL)
|
||||
Logger.logger().setLevel(logging.CRITICAL)
|
||||
|
||||
@staticmethod
|
||||
def be_verbose():
|
||||
Logger._logger.setLevel(logging.DEBUG)
|
||||
Logger.logger().setLevel(logging.DEBUG)
|
||||
|
||||
@staticmethod
|
||||
def colorize(msg, typ):
|
||||
@@ -50,16 +68,17 @@ class Logger(object):
|
||||
|
||||
@staticmethod
|
||||
def error(msg, **kwargs):
|
||||
return Logger._logger.error(Logger.colorize(msg, 'error'), **kwargs)
|
||||
exc_info = Logger.logger().getEffectiveLevel() == logging.DEBUG
|
||||
return Logger.logger().error(Logger.colorize(msg, 'error'), exc_info=exc_info, **kwargs)
|
||||
|
||||
@staticmethod
|
||||
def warn(msg, **kwargs):
|
||||
return Logger._logger.warn(Logger.colorize(msg, 'warn'), **kwargs)
|
||||
return Logger.logger().warn(Logger.colorize(msg, 'warn'), **kwargs)
|
||||
|
||||
@staticmethod
|
||||
def debug(msg, **kwargs):
|
||||
return Logger._logger.debug(Logger.colorize(msg, 'debug'), **kwargs)
|
||||
return Logger.logger().debug(Logger.colorize(msg, 'debug'), **kwargs)
|
||||
|
||||
@staticmethod
|
||||
def info(msg, **kwargs):
|
||||
return Logger._logger.info(Logger.colorize(msg, 'info'), **kwargs)
|
||||
return Logger.logger().info(Logger.colorize(msg, 'info'), **kwargs)
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
from dvc.cli import parse_args
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
def main(argv=None):
|
||||
args = parse_args(argv)
|
||||
return args.func(args).run_cmd()
|
||||
|
||||
+1
-1
@@ -70,7 +70,7 @@ class Project(object):
|
||||
self.cache = Cache(self.dvc_dir)
|
||||
self.state = State(self.root_dir, self.dvc_dir)
|
||||
self.config = Config(self.dvc_dir)
|
||||
self.logger = Logger()
|
||||
self.logger = Logger(self.config._config)
|
||||
self.cloud = DataCloud(self.config._config)
|
||||
|
||||
@staticmethod
|
||||
|
||||
+1
-2
@@ -8,7 +8,6 @@ TESTS+=" test-init.sh"
|
||||
TESTS+=" test-add-remove.sh"
|
||||
TESTS+=" test-sync.sh"
|
||||
TESTS+=" test-repro-code.sh"
|
||||
#TESTS+=" test-merge.sh"
|
||||
TESTS+=" test-checkout.sh"
|
||||
TESTS+=" test-gc.sh"
|
||||
TESTS+=" test-lock.sh"
|
||||
@@ -19,4 +18,4 @@ for t in $TESTS; do
|
||||
./$t
|
||||
done
|
||||
|
||||
#rm -rf $TEST_REPO
|
||||
rm -rf $TEST_REPO
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
dvc_create_repo
|
||||
|
||||
# Adding changes to the first branch
|
||||
git checkout -b foo
|
||||
cp code/code.sh code/code1.sh
|
||||
chmod +x code/code1.sh
|
||||
cp code/code1.sh code/code2.sh
|
||||
cp code/code1.sh code/code3.sh
|
||||
cp code/code1.sh code/code4.sh
|
||||
git add code
|
||||
git commit -s -m"add code"
|
||||
|
||||
cp $DATA_CACHE/foo data/data
|
||||
dvc add data/data
|
||||
|
||||
ORIG_DATA=$(dvc_md5 data/data)
|
||||
|
||||
dvc run -f copy_data_data1.dvc -d code/code1.sh -d data/data -o data/data1 ./code/code1.sh data/data data/data1
|
||||
dvc run -f copy_data1_data2.dvc -d code/code2.sh -d data/data1 -o data/data2 ./code/code2.sh data/data1 data/data2
|
||||
dvc run -f copy_data2_data3.dvc -d code/code3.sh -d data/data2 -o data/data3 ./code/code3.sh data/data2 data/data3
|
||||
dvc run -f copy_data3_data4.dvc -d code/code4.sh -d data/data3 -o data/data4 ./code/code4.sh data/data3 data/data4
|
||||
|
||||
dvc config Global.Target data/data4
|
||||
git commit -am 'Set Target'
|
||||
|
||||
# Create new branch to work on small data set
|
||||
git checkout -b bar
|
||||
dvc remove -c -l data/data
|
||||
cp $DATA_CACHE/bar data/data
|
||||
dvc add data/data
|
||||
dvc repro
|
||||
|
||||
echo -e "\n\n\n\n" >> code/code1.sh
|
||||
git commit -am"change code1.sh"
|
||||
dvc repro
|
||||
|
||||
# Merge bar into foo
|
||||
git checkout foo
|
||||
git merge bar
|
||||
# Notice we're running 'dvc merge' on foo branch. This
|
||||
# way dvc knows to favor data files from current branch.
|
||||
dvc merge
|
||||
|
||||
# Verify that data and respective cache state file are restored properly
|
||||
if [ "$(dvc_md5 data/data)" != "$ORIG_DATA" ]; then
|
||||
dvc_fail
|
||||
fi
|
||||
|
||||
grep $(echo "$ORIG_DATA" | cut -d " " -f1) data/data.dvc || dvc_fail
|
||||
|
||||
dvc_pass
|
||||
@@ -0,0 +1,236 @@
|
||||
import os
|
||||
|
||||
from dvc.cli import parse_args
|
||||
from dvc.command.init import CmdInit
|
||||
from dvc.command.run import CmdRun
|
||||
from dvc.command.data_sync import CmdDataPull, CmdDataPush, CmdDataStatus
|
||||
from dvc.command.repro import CmdRepro
|
||||
from dvc.command.add import CmdAdd
|
||||
from dvc.command.remove import CmdRemove
|
||||
from dvc.command.lock import CmdLock
|
||||
from dvc.command.gc import CmdGC
|
||||
from dvc.command.config import CmdConfig
|
||||
from dvc.command.show_pipeline import CmdShowPipeline
|
||||
from dvc.command.show_workflow import CmdShowWorkflow
|
||||
from dvc.command.checkout import CmdCheckout
|
||||
from dvc.command.find import CmdFind
|
||||
from dvc.command.fsck import CmdFsck
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestArgParse(TestDvc):
|
||||
def test(self):
|
||||
args = parse_args(['init'])
|
||||
self.assertIsInstance(args.func(args), CmdInit)
|
||||
|
||||
|
||||
class TestRun(TestDvc):
|
||||
def test(self):
|
||||
dep1 = 'dep1'
|
||||
dep2 = 'dep2'
|
||||
|
||||
out1 = 'out1'
|
||||
out2 = 'out2'
|
||||
|
||||
out_no_cache1 = 'out_no_cache1'
|
||||
out_no_cache2 = 'out_no_cache2'
|
||||
|
||||
fname = 'dvc.dvc'
|
||||
cwd = os.curdir
|
||||
cmd = 'cmd'
|
||||
arg1 = 'arg1'
|
||||
arg2 = 'arg2'
|
||||
|
||||
args = parse_args(['run',
|
||||
'-d', dep1,
|
||||
'--deps', dep2,
|
||||
'-o', out1,
|
||||
'--outs', out2,
|
||||
'-O', out_no_cache1,
|
||||
'--outs-no-cache', out_no_cache2,
|
||||
'-l', '--lock',
|
||||
'-f', fname,
|
||||
'--file', fname,
|
||||
'-c', cwd,
|
||||
cmd, arg1, arg2])
|
||||
|
||||
self.assertIsInstance(args.func(args), CmdRun)
|
||||
self.assertEqual(args.deps, [dep1, dep2])
|
||||
self.assertEqual(args.outs, [out1, out2])
|
||||
self.assertEqual(args.outs_no_cache, [out_no_cache1, out_no_cache2])
|
||||
self.assertEqual(args.file, fname)
|
||||
self.assertEqual(args.cwd, cwd)
|
||||
self.assertEqual(args.command, [cmd, arg1, arg2])
|
||||
|
||||
|
||||
class TestPull(TestDvc):
|
||||
def test(self):
|
||||
args = parse_args(['pull'])
|
||||
self.assertIsInstance(args.func(args), CmdDataPull)
|
||||
|
||||
|
||||
class TestPush(TestDvc):
|
||||
def test(self):
|
||||
args = parse_args(['push'])
|
||||
self.assertIsInstance(args.func(args), CmdDataPush)
|
||||
|
||||
|
||||
class TestStatus(TestDvc):
|
||||
def test(self):
|
||||
args = parse_args(['status'])
|
||||
self.assertIsInstance(args.func(args), CmdDataStatus)
|
||||
|
||||
|
||||
class TestRepro(TestDvc):
|
||||
def test(self):
|
||||
target1 = '1'
|
||||
target2 = '2'
|
||||
|
||||
args = parse_args(['repro',
|
||||
target1,
|
||||
target2,
|
||||
'-f', '--force',
|
||||
'-s', '--single-item'])
|
||||
|
||||
self.assertIsInstance(args.func(args), CmdRepro)
|
||||
self.assertEqual(args.targets, [target1, target2])
|
||||
self.assertEqual(args.force, True)
|
||||
self.assertEqual(args.single_item, True)
|
||||
|
||||
|
||||
class TestRemove(TestDvc):
|
||||
def test(self):
|
||||
target1 = '1'
|
||||
target2 = '2'
|
||||
|
||||
args = parse_args(['remove',
|
||||
target1,
|
||||
target2])
|
||||
|
||||
self.assertIsInstance(args.func(args), CmdRemove)
|
||||
self.assertEqual(args.targets, [target1, target2])
|
||||
|
||||
|
||||
class TestAdd(TestDvc):
|
||||
def test(self):
|
||||
target1 = '1'
|
||||
target2 = '2'
|
||||
|
||||
args = parse_args(['add',
|
||||
target1,
|
||||
target2])
|
||||
|
||||
self.assertIsInstance(args.func(args), CmdAdd)
|
||||
self.assertEqual(args.targets, [target1, target2])
|
||||
|
||||
|
||||
class TestLock(TestDvc):
|
||||
def test(self):
|
||||
target1 = '1'
|
||||
target2 = '2'
|
||||
|
||||
args = parse_args(['lock',
|
||||
target1,
|
||||
target2,
|
||||
'-u', '--unlock'])
|
||||
|
||||
self.assertIsInstance(args.func(args), CmdLock)
|
||||
self.assertEqual(args.files, [target1, target2])
|
||||
self.assertEqual(args.unlock, True)
|
||||
|
||||
|
||||
class TestGC(TestDvc):
|
||||
def test(self):
|
||||
args = parse_args(['gc'])
|
||||
self.assertIsInstance(args.func(args), CmdGC)
|
||||
|
||||
|
||||
class TestConfig(TestDvc):
|
||||
def test(self):
|
||||
name = 'param'
|
||||
value = '1'
|
||||
|
||||
args = parse_args(['config',
|
||||
'-u', '--unset',
|
||||
name,
|
||||
value])
|
||||
|
||||
self.assertIsInstance(args.func(args), CmdConfig)
|
||||
self.assertEqual(args.unset, True)
|
||||
self.assertEqual(args.name, name)
|
||||
self.assertEqual(args.value, value)
|
||||
|
||||
|
||||
class TestShowPipeline(TestDvc):
|
||||
def test(self):
|
||||
target1 = '1'
|
||||
target2 = '2'
|
||||
|
||||
args = parse_args(['show',
|
||||
'pipeline',
|
||||
target1,
|
||||
target2])
|
||||
|
||||
self.assertIsInstance(args.func(args), CmdShowPipeline)
|
||||
self.assertEqual(args.target, [target1, target2])
|
||||
|
||||
|
||||
class TestShowWorkflow(TestDvc):
|
||||
def test(self):
|
||||
target = '1'
|
||||
|
||||
args = parse_args(['show',
|
||||
'workflow',
|
||||
'-d', '--dvc-commits',
|
||||
'-a', '--all-commits',
|
||||
'-m', '1',
|
||||
'--max-commits', '1',
|
||||
target])
|
||||
|
||||
self.assertIsInstance(args.func(args), CmdShowWorkflow)
|
||||
self.assertEqual(args.target, target)
|
||||
self.assertTrue(args.dvc_commits)
|
||||
self.assertTrue(args.all_commits)
|
||||
self.assertEqual(args.max_commits, 1)
|
||||
|
||||
|
||||
class TestCheckout(TestDvc):
|
||||
def test(self):
|
||||
args = parse_args(['checkout'])
|
||||
self.assertIsInstance(args.func(args), CmdCheckout)
|
||||
|
||||
|
||||
class TestFind(TestDvc):
|
||||
def test(self):
|
||||
branch = 'branch'
|
||||
target = 'target'
|
||||
criteria = 'max'
|
||||
|
||||
args = parse_args(['find',
|
||||
branch,
|
||||
target,
|
||||
'-c', criteria,
|
||||
'--criteria', criteria,
|
||||
'-s', '--show-value'])
|
||||
|
||||
self.assertIsInstance(args.func(args), CmdFind)
|
||||
self.assertEqual(args.branch_name, branch)
|
||||
self.assertEqual(args.target, target)
|
||||
self.assertEqual(args.criteria, criteria)
|
||||
self.assertTrue(args.show_value)
|
||||
|
||||
|
||||
class TestFsck(TestDvc):
|
||||
def test(self):
|
||||
target1 = 'target1'
|
||||
target2 = 'target2'
|
||||
|
||||
args = parse_args(['fsck',
|
||||
'-p', '--physical',
|
||||
target1,
|
||||
target2])
|
||||
|
||||
self.assertIsInstance(args.func(args), CmdFsck)
|
||||
self.assertTrue(args.physical)
|
||||
self.assertEqual(args.targets, [target1, target2])
|
||||
@@ -0,0 +1,35 @@
|
||||
import colorama
|
||||
import logging
|
||||
|
||||
from dvc.logger import Logger
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestLogger(TestDvc):
|
||||
def test_config(self):
|
||||
config1 = {'Global': {'LogLevel': 'debug'}}
|
||||
Logger(config1)
|
||||
self.assertEqual(Logger.logger().getEffectiveLevel(), logging.DEBUG)
|
||||
|
||||
config2 = {'Global': {'LogLevel': 'error'}}
|
||||
Logger(config2)
|
||||
self.assertEqual(Logger.logger().getEffectiveLevel(), logging.ERROR)
|
||||
|
||||
def test_set_level(self):
|
||||
Logger.set_level('debug')
|
||||
self.assertEqual(Logger.logger().getEffectiveLevel(), logging.DEBUG)
|
||||
|
||||
def test_be_quiet(self):
|
||||
Logger.be_quiet()
|
||||
self.assertEqual(Logger.logger().getEffectiveLevel(), logging.CRITICAL)
|
||||
|
||||
def test_be_verbose(self):
|
||||
Logger.be_verbose()
|
||||
self.assertEqual(Logger.logger().getEffectiveLevel(), logging.DEBUG)
|
||||
|
||||
def test_colorize(self):
|
||||
for name, color in Logger.COLOR_MAP.items():
|
||||
msg = Logger.colorize(name, name)
|
||||
# This is not a tty, so it should not colorize anything
|
||||
self.assertEqual(msg, name)
|
||||
@@ -1,6 +1,8 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
ulimit -n 2048
|
||||
|
||||
nosetests --cover-inclusive --cover-erase --cover-package=dvc --with-coverage
|
||||
|
||||
if [ "$1" = "report" ]; then
|
||||
|
||||
Reference in New Issue
Block a user