From 609586eb605db37fc19b6cd797b961bec9cf0095 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 7 Jan 2018 12:41:33 +0200 Subject: [PATCH 1/8] test: add basic tests for cli.py Fixes #384 Signed-off-by: Ruslan Kuprieiev --- .coveragerc | 7 ++ dvc.py | 4 +- dvc/__main__.py | 4 +- dvc/cli.py | 34 +----- dvc/command/common/base.py | 2 +- dvc/main.py | 4 +- tests/test_cli.py | 236 +++++++++++++++++++++++++++++++++++++ 7 files changed, 258 insertions(+), 33 deletions(-) create mode 100644 .coveragerc create mode 100644 tests/test_cli.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..1408127c3 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +[run] +omit = + *__main__.py + +[report] +exclude_lines = + if __name__ == .__main__.: diff --git a/dvc.py b/dvc.py index 78bdb4e24..0040e8419 100755 --- a/dvc.py +++ b/dvc.py @@ -1,4 +1,6 @@ +import sys + from dvc.main import main if __name__ == "__main__": - main() + main(sys.argv[1:]) diff --git a/dvc/__main__.py b/dvc/__main__.py index 74fb03c3d..964065a73 100644 --- a/dvc/__main__.py +++ b/dvc/__main__.py @@ -1,4 +1,6 @@ +import sys + from dvc.main import main if __name__ == '__main__': - main() + main(sys.argv[1:]) diff --git a/dvc/cli.py b/dvc/cli.py index cea296125..b771fe25e 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -39,12 +39,6 @@ def parse_args(argv=None): action='store_true', default=False, help='Be verbose.') - parent_parser.add_argument( - '-G', - '--no-git-actions', - action='store_true', - default=False, - help='Skip all git actions including reproducibility check and commits.') parent_parser.add_argument( '-b', @@ -221,20 +215,6 @@ def parse_args(argv=None): 'gc', parents=[parent_parser], help='Collect garbage') - gc_parser.add_argument('-l', - '--keep-in-cloud', - action='store_true', - default=False, - help='Do not remove data from cloud.') - gc_parser.add_argument('-r', - '--recursive', - action='store_true', - help='Remove directory recursively.') - gc_parser.add_argument('-c', - '--keep-in-cache', - action='store_false', - default=False, - help='Do not remove data from cache.') gc_parser.set_defaults(func=CmdGC) # Cloud @@ -339,12 +319,13 @@ def parse_args(argv=None): parents=[parent_parser], help='Show graphs') show_subparsers = show_parser.add_subparsers( - dest='cmd', - help='Use `dvc show CMD` --help for command-specific help') + dest='cmd', + help='Use `dvc show CMD` --help for command-specific help') + pipeline_parser = show_subparsers.add_parser( - 'pipeline', - parents=[parent_parser], - help='Show pipeline image') + 'pipeline', + parents=[parent_parser], + help='Show pipeline image') pipeline_parser.add_argument( 'target', nargs='*', @@ -431,7 +412,4 @@ def parse_args(argv=None): help='Compute actual md5') fsck_parser.set_defaults(func=CmdFsck) - if isinstance(argv, str): - argv = argv.split() - return parser.parse_args(argv) diff --git a/dvc/command/common/base.py b/dvc/command/common/base.py index 805474e4b..7d9316b2c 100644 --- a/dvc/command/common/base.py +++ b/dvc/command/common/base.py @@ -21,7 +21,7 @@ class CmdBase(object): return root root = os.path.dirname(root) msg = "Not a dvc repository (or any parent up to mount point {})" - Logger.error(msg.format(root)) + self.project.logger.error(msg.format(root)) def run_cmd(self): with self.project.lock: diff --git a/dvc/main.py b/dvc/main.py index 280055aba..a97dfdde8 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -1,5 +1,5 @@ from dvc.cli import parse_args -def main(): - args = parse_args() +def main(argv=None): + args = parse_args(argv) return args.func(args).run_cmd() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 000000000..4481b605c --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,236 @@ +import os + +from dvc.cli import parse_args +from dvc.command.init import CmdInit +from dvc.command.run import CmdRun +from dvc.command.data_sync import CmdDataPull, CmdDataPush, CmdDataStatus +from dvc.command.repro import CmdRepro +from dvc.command.add import CmdAdd +from dvc.command.remove import CmdRemove +from dvc.command.lock import CmdLock +from dvc.command.gc import CmdGC +from dvc.command.config import CmdConfig +from dvc.command.show_pipeline import CmdShowPipeline +from dvc.command.show_workflow import CmdShowWorkflow +from dvc.command.checkout import CmdCheckout +from dvc.command.find import CmdFind +from dvc.command.fsck import CmdFsck + +from tests.basic_env import TestDvc + + +class TestArgParse(TestDvc): + def test(self): + args = parse_args(['init']) + self.assertIsInstance(args.func(args), CmdInit) + + +class TestRun(TestDvc): + def test(self): + dep1 = 'dep1' + dep2 = 'dep2' + + out1 = 'out1' + out2 = 'out2' + + out_no_cache1 = 'out_no_cache1' + out_no_cache2 = 'out_no_cache2' + + fname = 'dvc.dvc' + cwd = os.curdir + cmd = 'cmd' + arg1 = 'arg1' + arg2 = 'arg2' + + args = parse_args(['run', + '-d', dep1, + '--deps', dep2, + '-o', out1, + '--outs', out2, + '-O', out_no_cache1, + '--outs-no-cache', out_no_cache2, + '-l', '--lock', + '-f', fname, + '--file', fname, + '-c', cwd, + cmd, arg1, arg2]) + + self.assertIsInstance(args.func(args), CmdRun) + self.assertEqual(args.deps, [dep1, dep2]) + self.assertEqual(args.outs, [out1, out2]) + self.assertEqual(args.outs_no_cache, [out_no_cache1, out_no_cache2]) + self.assertEqual(args.file, fname) + self.assertEqual(args.cwd, cwd) + self.assertEqual(args.command, [cmd, arg1, arg2]) + + +class TestPull(TestDvc): + def test(self): + args = parse_args(['pull']) + self.assertIsInstance(args.func(args), CmdDataPull) + + +class TestPush(TestDvc): + def test(self): + args = parse_args(['push']) + self.assertIsInstance(args.func(args), CmdDataPush) + + +class TestStatus(TestDvc): + def test(self): + args = parse_args(['status']) + self.assertIsInstance(args.func(args), CmdDataStatus) + + +class TestRepro(TestDvc): + def test(self): + target1 = '1' + target2 = '2' + + args = parse_args(['repro', + target1, + target2, + '-f', '--force', + '-s', '--single-item']) + + self.assertIsInstance(args.func(args), CmdRepro) + self.assertEqual(args.targets, [target1, target2]) + self.assertEqual(args.force, True) + self.assertEqual(args.single_item, True) + + +class TestRemove(TestDvc): + def test(self): + target1 = '1' + target2 = '2' + + args = parse_args(['remove', + target1, + target2]) + + self.assertIsInstance(args.func(args), CmdRemove) + self.assertEqual(args.targets, [target1, target2]) + + +class TestAdd(TestDvc): + def test(self): + target1 = '1' + target2 = '2' + + args = parse_args(['add', + target1, + target2]) + + self.assertIsInstance(args.func(args), CmdAdd) + self.assertEqual(args.targets, [target1, target2]) + + +class TestLock(TestDvc): + def test(self): + target1 = '1' + target2 = '2' + + args = parse_args(['lock', + target1, + target2, + '-u', '--unlock']) + + self.assertIsInstance(args.func(args), CmdLock) + self.assertEqual(args.files, [target1, target2]) + self.assertEqual(args.unlock, True) + + +class TestGC(TestDvc): + def test(self): + args = parse_args(['gc']) + self.assertIsInstance(args.func(args), CmdGC) + + +class TestConfig(TestDvc): + def test(self): + name = 'param' + value = '1' + + args = parse_args(['config', + '-u', '--unset', + name, + value]) + + self.assertIsInstance(args.func(args), CmdConfig) + self.assertEqual(args.unset, True) + self.assertEqual(args.name, name) + self.assertEqual(args.value, value) + + +class TestShowPipeline(TestDvc): + def test(self): + target1 = '1' + target2 = '2' + + args = parse_args(['show', + 'pipeline', + target1, + target2]) + + self.assertIsInstance(args.func(args), CmdShowPipeline) + self.assertEqual(args.target, [target1, target2]) + + +class TestShowWorkflow(TestDvc): + def test(self): + target = '1' + + args = parse_args(['show', + 'workflow', + '-d', '--dvc-commits', + '-a', '--all-commits', + '-m', '1', + '--max-commits', '1', + target]) + + self.assertIsInstance(args.func(args), CmdShowWorkflow) + self.assertEqual(args.target, target) + self.assertTrue(args.dvc_commits) + self.assertTrue(args.all_commits) + self.assertEqual(args.max_commits, 1) + + +class TestCheckout(TestDvc): + def test(self): + args = parse_args(['checkout']) + self.assertIsInstance(args.func(args), CmdCheckout) + + +class TestFind(TestDvc): + def test(self): + branch = 'branch' + target = 'target' + criteria = 'max' + + args = parse_args(['find', + branch, + target, + '-c', criteria, + '--criteria', criteria, + '-s', '--show-value']) + + self.assertIsInstance(args.func(args), CmdFind) + self.assertEqual(args.branch_name, branch) + self.assertEqual(args.target, target) + self.assertEqual(args.criteria, criteria) + self.assertTrue(args.show_value) + + +class TestFsck(TestDvc): + def test(self): + target1 = 'target1' + target2 = 'target2' + + args = parse_args(['fsck', + '-p', '--physical', + target1, + target2]) + + self.assertIsInstance(args.func(args), CmdFsck) + self.assertTrue(args.physical) + self.assertEqual(args.targets, [target1, target2]) From aa396d1b42e1b415c26e82d8eb64dbf110c64644 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 7 Jan 2018 13:33:26 +0200 Subject: [PATCH 2/8] add/remove: catch and report errors Signed-off-by: Ruslan Kuprieiev --- dvc/command/add.py | 7 ++++++- dvc/command/remove.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/dvc/command/add.py b/dvc/command/add.py index 911188772..2846b21d6 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -1,8 +1,13 @@ +from dvc.exceptions import DvcException from dvc.command.common.base import CmdBase class CmdAdd(CmdBase): def run(self): for target in self.args.targets: - self.project.add(target) + try: + self.project.add(target) + except DvcException as ex: + self.project.logger.error('Failed to add {}: {}'.format(target, str(ex))) + return 1 return 0 diff --git a/dvc/command/remove.py b/dvc/command/remove.py index 45dcce197..e2ea0d8fa 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -1,8 +1,13 @@ +from dvc.exceptions import DvcException from dvc.command.common.base import CmdBase class CmdRemove(CmdBase): def run(self): for target in self.args.targets: - self.project.remove(target) + try: + self.project.remove(target) + except DvcException as ex: + self.project.logger.error('Failed to remove {}: {}'.format(target, str(ex))) + return 1 return 0 From af0d5eeeca9e0b68976b2d6fc135a5939540cbff Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 7 Jan 2018 16:24:09 +0200 Subject: [PATCH 3/8] test: use ulimit Signed-off-by: Ruslan Kuprieiev --- unittests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unittests.sh b/unittests.sh index 3f14befad..31e89d2fc 100755 --- a/unittests.sh +++ b/unittests.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +ulimit -n 2048 + nosetests --cover-inclusive --cover-erase --cover-package=dvc --with-coverage if [ "$1" = "report" ]; then From bce3210a9890582277ba567580d7a9f9a838c0e4 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 7 Jan 2018 14:42:24 +0200 Subject: [PATCH 4/8] logger: rework and add test Signed-off-by: Ruslan Kuprieiev --- dvc/config.py | 1 - dvc/logger.py | 39 +++++++++++++++++++++++++++++---------- dvc/project.py | 2 +- tests/test_logger.py | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 12 deletions(-) create mode 100644 tests/test_logger.py diff --git a/dvc/config.py b/dvc/config.py index 105d94bcf..3f3323b85 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -62,7 +62,6 @@ ProjectName = self._config = configparser.SafeConfigParser() self._config.read(self.config_file) - Logger.set_level(self._config['Global']['LogLevel']) @staticmethod def init(dvc_dir): diff --git a/dvc/logger.py b/dvc/logger.py index 31a4a1bb4..d277aff97 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -3,9 +3,12 @@ import logging import colorama + colorama.init() + class Logger(object): + FMT = '%(message)s' DEFAULT_LEVEL = logging.INFO LEVEL_MAP = { @@ -21,21 +24,36 @@ class Logger(object): 'error': colorama.Fore.RED } - logging.basicConfig(stream=sys.stdout, format='%(message)s', level=DEFAULT_LEVEL) + def __init__(self, config=None): + sh = logging.StreamHandler(sys.stdout) + sh.setFormatter(logging.Formatter(self.FMT)) + sh.setLevel(logging.DEBUG) - _logger = logging.getLogger('dvc') + self.logger().addHandler(sh) + level = None + if config: + level = config['Global'].get('LogLevel', None) + self.set_level(level) @staticmethod - def set_level(level): - Logger._logger.setLevel(Logger.LEVEL_MAP.get(level.lower(), logging.DEBUG)) + def logger(): + return logging.getLogger('dvc') + + @staticmethod + def set_level(level=None): + if not level: + lvl = Logger.DEFAULT_LEVEL + else: + lvl = Logger.LEVEL_MAP.get(level.lower(), Logger.DEFAULT_LEVEL) + Logger.logger().setLevel(lvl) @staticmethod def be_quiet(): - Logger._logger.setLevel(logging.CRITICAL) + Logger.logger().setLevel(logging.CRITICAL) @staticmethod def be_verbose(): - Logger._logger.setLevel(logging.DEBUG) + Logger.logger().setLevel(logging.DEBUG) @staticmethod def colorize(msg, typ): @@ -50,16 +68,17 @@ class Logger(object): @staticmethod def error(msg, **kwargs): - return Logger._logger.error(Logger.colorize(msg, 'error'), **kwargs) + exc_info = Logger.logger().getEffectiveLevel() == logging.DEBUG + return Logger.logger().error(Logger.colorize(msg, 'error'), exc_info=exc_info, **kwargs) @staticmethod def warn(msg, **kwargs): - return Logger._logger.warn(Logger.colorize(msg, 'warn'), **kwargs) + return Logger.logger().warn(Logger.colorize(msg, 'warn'), **kwargs) @staticmethod def debug(msg, **kwargs): - return Logger._logger.debug(Logger.colorize(msg, 'debug'), **kwargs) + return Logger.logger().debug(Logger.colorize(msg, 'debug'), **kwargs) @staticmethod def info(msg, **kwargs): - return Logger._logger.info(Logger.colorize(msg, 'info'), **kwargs) + return Logger.logger().info(Logger.colorize(msg, 'info'), **kwargs) diff --git a/dvc/project.py b/dvc/project.py index e77541af2..8f2c4e7db 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -70,7 +70,7 @@ class Project(object): self.cache = Cache(self.dvc_dir) self.state = State(self.root_dir, self.dvc_dir) self.config = Config(self.dvc_dir) - self.logger = Logger() + self.logger = Logger(self.config._config) self.cloud = DataCloud(self.config._config) @staticmethod diff --git a/tests/test_logger.py b/tests/test_logger.py new file mode 100644 index 000000000..3c6e04d1d --- /dev/null +++ b/tests/test_logger.py @@ -0,0 +1,35 @@ +import colorama +import logging + +from dvc.logger import Logger + +from tests.basic_env import TestDvc + + +class TestLogger(TestDvc): + def test_config(self): + config1 = {'Global': {'LogLevel': 'debug'}} + Logger(config1) + self.assertEqual(Logger.logger().getEffectiveLevel(), logging.DEBUG) + + config2 = {'Global': {'LogLevel': 'error'}} + Logger(config2) + self.assertEqual(Logger.logger().getEffectiveLevel(), logging.ERROR) + + def test_set_level(self): + Logger.set_level('debug') + self.assertEqual(Logger.logger().getEffectiveLevel(), logging.DEBUG) + + def test_be_quiet(self): + Logger.be_quiet() + self.assertEqual(Logger.logger().getEffectiveLevel(), logging.CRITICAL) + + def test_be_verbose(self): + Logger.be_verbose() + self.assertEqual(Logger.logger().getEffectiveLevel(), logging.DEBUG) + + def test_colorize(self): + for name, color in Logger.COLOR_MAP.items(): + msg = Logger.colorize(name, name) + # This is not a tty, so it should not colorize anything + self.assertEqual(msg, name) From 8181ca4a9a8d6ade6974a7606939b5d1b1b5cf75 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 7 Jan 2018 17:05:58 +0200 Subject: [PATCH 5/8] show: fix CmdShowPipeline Signed-off-by: Ruslan Kuprieiev --- dvc/command/show_pipeline.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dvc/command/show_pipeline.py b/dvc/command/show_pipeline.py index 873378a45..09af857ae 100644 --- a/dvc/command/show_pipeline.py +++ b/dvc/command/show_pipeline.py @@ -26,8 +26,7 @@ class CmdShowPipeline(CmdBase): def draw_targets(self, target): if not target: - target = '.' - return self.draw(self.g, target, target) + return self.draw(self.g, '.', 'all') for t in self.args.target: fname_suffix = os.path.basename(os.path.normpath(t)) @@ -59,7 +58,6 @@ class CmdShowPipeline(CmdBase): def collect_stage(self, stage): name = os.path.relpath(stage.path, self.project.root_dir) - state = StateFile.load(data_item, self.git) self.g.add_node(name) @@ -68,7 +66,7 @@ class CmdShowPipeline(CmdBase): self.g.add_node(i) self.g.add_edge(i, name) - for out in state.outs: + for out in stage.outs: o = os.path.relpath(out.path, self.project.root_dir) self.g.add_node(o) self.g.add_edge(name, o) From cc108e15502a0ed21c19e424a0b0b361c485e1ab Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 7 Jan 2018 17:13:40 +0200 Subject: [PATCH 6/8] show: fix workflow Signed-off-by: Ruslan Kuprieiev --- dvc/command/show_workflow.py | 75 +++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/dvc/command/show_workflow.py b/dvc/command/show_workflow.py index 875f1a674..fc92695c4 100644 --- a/dvc/command/show_workflow.py +++ b/dvc/command/show_workflow.py @@ -1,16 +1,87 @@ +import re +from collections import defaultdict + +from dvc.logger import Logger from dvc.command.common.base import CmdBase -from dvc.git_wrapper import GitWrapper +from dvc.graph.workflow import Workflow +from dvc.graph.commit import Commit +from dvc.executor import Executor, ExecutorError class CmdShowWorkflow(CmdBase): + COMMIT_LEN = 7 + LOG_SEPARATOR = '|' + LOG_FORMAT = ['%h', '%p', '%an', '%ai', '%s'] + + def get_all_commits(self, target): + # git log --all --abbrev=7 --pretty=format:"%h|%p|%an|%ai|%s" + try: + merges_map = self.get_merges_map() + + format_str = self.LOG_SEPARATOR.join(self.LOG_FORMAT) + git_cmd = ['git', 'log', '--all', '--abbrev={}'.format(self.COMMIT_LEN), + '--pretty=format:{}'.format(format_str)] + lines = Executor.exec_cmd_only_success(git_cmd).split('\n') + + branches_multimap = self.branches_multimap() + + wf = Workflow(target, merges_map, branches_multimap) + for line in lines: + items = line.split(self.LOG_SEPARATOR, len(self.LOG_FORMAT)) + assert len(items) == 5, 'Git wrapper: git log format has {} items, 5 expected'.format(len(items)) + hash, parent_hash, name, date, comment = items + + commit = Commit(hash, parent_hash, name, date, comment, + *self.was_target_changed(hash, target), + branch_tips=branches_multimap.get(hash)) + wf.add_commit(commit) + + return wf + except ExecutorError: + raise + + def was_target_changed(self, hash, target): + git_cmd = ['git', 'show', '--pretty=', '--name-only', hash] + changed_files = set(Executor.exec_cmd_only_success(git_cmd).split('\n')) + + if target not in changed_files: + return False, None + + metric = utils.parse_target_metric_file(target) + if metric is None: + return False, None + + return True, metric + + def get_merges_map(self): + # git log --merges --all --abbrev=7 + # {'a4b56f1': back_to_600_est} + git_cmd = ['git', 'log', '--all', '--merges', + '--abbrev={}'.format(self.COMMIT_LEN)] + # lines = map(str.strip, Executor.exec_cmd_only_success(git_cmd).split('\n')) + # lines.map + return {} + + def branches_multimap(self): + git_cmd = ['git', 'show-ref', '--abbrev={}'.format(self.COMMIT_LEN)] + lines = Executor.exec_cmd_only_success(git_cmd).split('\n') + items_full = map(unicode.split, lines) + items = map(lambda it: (it[0], re.sub(r'^refs/heads/', '', it[1])), items_full) + + result = defaultdict(list) + for (hash, branch) in items: + result[hash].append(branch) + return result + def run(self): target = self.args.target if not target: target = self.project.config._config['Global'].get('Target', '') self.project.logger.debug(u'Set show workflow target as {}'.format(target)) - wf = GitWrapper.get_all_commits(target, self.settings) + wf = self.get_all_commits(target) wf.build_graph(self.args.dvc_commits, self.args.all_commits, self.args.max_commits) return 0 + From fcb19ba41b6cff22ef71f114938f482bb4bb3cec Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 7 Jan 2018 17:13:54 +0200 Subject: [PATCH 7/8] dvc: remove GitWrapper Signed-off-by: Ruslan Kuprieiev --- dvc/git_wrapper.py | 448 --------------------------------------------- 1 file changed, 448 deletions(-) delete mode 100644 dvc/git_wrapper.py diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py deleted file mode 100644 index 97b531acd..000000000 --- a/dvc/git_wrapper.py +++ /dev/null @@ -1,448 +0,0 @@ -import os -import re -from collections import defaultdict - -from dvc import utils -from dvc.exceptions import DvcException -from dvc.logger import Logger -from dvc.config import Config -from dvc.executor import Executor, ExecutorError -from dvc.system import System -from dvc.graph.workflow import Workflow -from dvc.graph.commit import Commit - - -class GitWrapperError(DvcException): - def __init__(self, msg): - DvcException.__init__(self, msg) - - -class GitWrapperI(object): - COMMIT_LEN = 7 - - def __init__(self, git_dir=None, commit=None): - self._git_dir = git_dir - self._commit = commit - - @property - def git_dir(self): - return self._git_dir - - @property - def lock_file(self): - return os.path.join(self.git_dir_abs, Config.CONFIG_DIR, '.' + Config.CONFIG + '.lock') - - @property - def git_dir_abs(self): - return System.realpath(self.git_dir) - - @property - def curr_dir_abs(self): - return os.path.abspath(os.curdir) - - @property - def curr_commit(self): - return self._commit - - def is_ready_to_go(self): - return True - - @staticmethod - def git_file_statuses(): - Logger.debug('[dvc-git] Getting file statuses. Command: git status --porcelain') - code, out, err = Executor.exec_cmd(['git', 'status', '--porcelain']) - if code != 0: - raise ExecutorError('[dvc-git] File status command error - {}'.format(err)) - Logger.debug('[dvc-git] Getting file statuses. Success.') - - return GitWrapper.parse_porcelain_files(out) - - @staticmethod - def git_config_get(name): - code, out, err = Executor.exec_cmd(['git', 'config', '--get', name]) - Logger.debug('[dvc-git] "git config --get {}": code({}), out({}), err({})'.format(name, - code, - out, - err)) - if code != 0: - return None - return out - - @staticmethod - def git_path_to_system_path(path): - if os.name == 'nt': - return path.replace('/', '\\') - return path - - @staticmethod - def parse_porcelain_files(out): - result = [] - if len(out) > 0: - lines = out.split('\n') - for line in lines: - status = line[:2] - file = GitWrapperI.git_path_to_system_path(line[3:]) - result.append((status, file)) - return result - - def abs_paths_to_dvc(self, files): - result = [] - for file in files: - result.append(os.path.relpath(os.path.abspath(file), self.git_dir_abs)) - - return result - - def commit_all_changes_and_log_status(self, message): - pass - - @staticmethod - def _file_contains(fname, s): - if not os.path.isfile(fname): - return False - - with open(fname, 'r') as fd: - if s in fd.read(): - return True - - return False - - @staticmethod - def modify_gitignore(flist): - for fname in flist: - # We're modifying .gitignore in the same dir as the file itself, so - # that if dir consists only of data files it will be still tracked by - # git and we will not have any problems with --porcelain output for - # specific changed files. - gitignore = os.path.join(os.path.dirname(fname), '.gitignore') - entry = os.path.basename(fname) - - if GitWrapperI._file_contains(gitignore, entry + '\n'): - return - - with open(gitignore, 'a') as fd: - fd.write('\n' + entry) - - -class GitWrapper(GitWrapperI): - FLOATS_FROM_STRING = re.compile(r'[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?') - - def __init__(self): - super(GitWrapper, self).__init__() - - def is_inside_work_tree(self): - code, out, err = Executor.exec_cmd(['git', 'rev-parse', '--is-inside-work-tree']) - if code != 0: - Logger.error('[dvc-git] Not inside git tree'); - return code == 0 - - def is_ready_to_go(self): - if not self.is_inside_work_tree(): - return False - - statuses = self.git_file_statuses() - if len(statuses) > 0: - Logger.error('[dvc-git] Commit all changed files before running ' - 'reproducible command. Changed files:') - for status, file in statuses: - Logger.error("{} {}".format(status, file)) - return False - - return self.check_config() - - def check_config(self): - # Sanity check to make sure we will be able to commit - name = self.git_config_get('user.name') - if name == None: - Logger.error('[dvc-git] Please setup user.name in git config') - return False - - email = self.git_config_get('user.email') - if email == None: - Logger.error('[dvc-git] Please setup user.email in git config') - return False - - return True - - @property - def curr_dir_dvc(self): - return os.path.relpath(os.path.abspath(os.curdir), self.git_dir_abs) - - @property - def git_dir(self): - if self._git_dir: - return self._git_dir - - try: - Logger.debug('[dvc-git] Getting git directory. Command: git rev-parse --show-toplevel') - code, out, err = Executor.exec_cmd(['git', 'rev-parse', '--show-toplevel']) - - if code != 0: - raise ExecutorError('[dvc-git] Git directory command error - {}'.format(err)) - Logger.debug('[dvc-git] Getting git directory. Success.') - - self._git_dir = out - return self._git_dir - except ExecutorError: - raise - except Exception as e: - raise ExecutorError('Unable to run git command: {}'.format(e)) - pass - - def curr_commit(self, branches=False): - Logger.debug('[dvc-git] Getting current git commit. Command: git rev-parse --short HEAD') - - if branches: - cmd = 'git rev-parse --abbrev-ref HEAD' - else: - cmd = 'git rev-parse HEAD' - - code, out, err = Executor.exec_cmd(cmd.split()) - if code != 0: - raise ExecutorError('[dvc-git] Commit command error - {}'.format(err)) - Logger.debug('[dvc-git] Getting current git commit. Success.') - return out - - @property - def curr_branch_or_commit(self): - Logger.debug('[dvc-git] Getting current git branch or commit.') - - branch = self.curr_commit(branches=True) - if branch == '' or branch == 'HEAD': - result = self.curr_commit(branches=False) - else: - result = branch - - Logger.debug('[dvc-git] Getting current git branch or commit. Result {}. Success.'.format(result)) - return result - - @staticmethod - def commit_all_changes(message): - Logger.debug('[dvc-git] Commit all changes. Commands: {} && {} && {}'.format( - 'git add --all', 'git status --porcelain', 'git commit -m')) - - Executor.exec_cmd_only_success(['git', 'add', '--all']) - out_status = Executor.exec_cmd_only_success(['git', 'status', '--porcelain']) - changes = GitWrapper.parse_porcelain_files(out_status) - if len(changes) == 0: - return [] - - Executor.exec_cmd_only_success(['git', 'commit', '-m', '\'{}\''.format(message)]) - Logger.debug('[dvc-git] Commit all changes. Success.') - return changes - - def commit_all_changes_and_log_status(self, message): - statuses = self.commit_all_changes(message) - Logger.debug('[dvc-git] A new commit {} was made in the current branch. ' - 'Added files:'.format(self.curr_commit())) - for status, file in statuses: - Logger.debug('[dvc-git]\t{} {}'.format(status, file)) - pass - - @staticmethod - def abs_paths_to_relative(files): - cur_dir = System.realpath(os.curdir) - - result = [] - for file in files: - result.append(os.path.relpath(System.realpath(file), cur_dir)) - - return result - - def dvc_paths_to_abs(self, files): - results = [] - - for file in files: - results.append(os.path.join(self.git_dir_abs, file)) - - return results - - def were_files_changed(self, code_dependencies, path_factory, changed_files): - code_files, code_dirs = self.separate_dependency_files_and_dirs(code_dependencies) - code_files_set = set([path_factory.path(x).dvc for x in code_files]) - for changed_file in changed_files: - if changed_file in code_files_set: - return True - - for dir in code_dirs: - if changed_file.startswith(dir): - return True - - return False - - @staticmethod - def get_changed_files(target_commit): - Logger.debug('[dvc-git] Identify changes. Command: ' - 'git diff --name-only HEAD {}'.format(target_commit)) - - changed_files_str = Executor.exec_cmd_only_success(['git', - 'diff', - '--name-only', - 'HEAD', - target_commit]) - changed_files = changed_files_str.strip('"').split('\n') - - Logger.debug('[dvc-git] Identify changes. Success. Changed files: {}'.format( - ', '.join(changed_files))) - return changed_files - - @staticmethod - def get_target_commit(fname): - Logger.debug('[dvc-git] Get target commit. Command: ' - 'git log -1 --pretty=format:"%h" {}'.format(fname)) - try: - commit = Executor.exec_cmd_only_success(['git', - 'log', - '-1', - '--pretty=format:"%h"', - fname]) - return commit.strip('"') - except ExecutorError: - return None - - @staticmethod - def get_last_merge_changed_files(): - cmd = 'git diff --name-only @{1}' - Logger.debug('[dvc-git] Get last merge changed files. Command: {}'.format(cmd)) - try: - fstr = Executor.exec_cmd_only_success(cmd.split()) - return fstr.split('\n') - except ExecutorError: - return None - - @staticmethod - def checkout_file_before_last_merge(fname): - cmd = 'git checkout @{1} -- ' + fname - Logger.debug('[dvc-git] Checkout file {} before last merge. Command: {}'.format(fname, cmd)) - Executor.exec_cmd_only_success(cmd.split()) - - @staticmethod - def get_file_content_before_last_merge(fname): - return GitWrapper.get_file_content(fname, '@{1}') - - @staticmethod - def checkout(commit_or_branch, create_new=False): - prefix = '-b ' if create_new else '' - Logger.debug('[dvc-git] Checkout {}{}'.format(prefix, commit_or_branch)) - cmd = 'git checkout {}{}'.format(prefix, commit_or_branch) - return Executor.exec_cmd(cmd.split()) - - @staticmethod - def checkout_previous(): - Logger.debug('[dvc-git] Checkout previous') - cmd = 'git checkout -' - return Executor.exec_cmd(cmd.split()) - - def separate_dependency_files_and_dirs(self, code_dependencies): - code_files = [] - code_dirs = [] - - code_dependencies_abs = self.dvc_paths_to_abs(code_dependencies) - for code in code_dependencies_abs: - if os.path.isdir(code): - code_dirs.append(code) - else: - code_files.append(code) - - return code_files, code_dirs - - LOG_SEPARATOR = '|' - LOG_FORMAT = ['%h', '%p', '%an', '%ai', '%s'] - - def get_all_commits(self, target, settings): - # git log --all --abbrev=7 --pretty=format:"%h|%p|%an|%ai|%s" - try: - merges_map = GitWrapper.get_merges_map() - - format_str = GitWrapper.LOG_SEPARATOR.join(GitWrapper.LOG_FORMAT) - git_cmd = ['git', 'log', '--all', '--abbrev={}'.format(GitWrapper.COMMIT_LEN), - '--pretty=format:{}'.format(format_str)] - lines = Executor.exec_cmd_only_success(git_cmd).split('\n') - - branches_multimap = GitWrapper.branches_multimap() - - wf = Workflow(target, merges_map, branches_multimap) - for line in lines: - items = line.split(GitWrapper.LOG_SEPARATOR, len(GitWrapper.LOG_FORMAT)) - assert len(items) == 5, 'Git wrapper: git log format has {} items, 5 expected'.format(len(items)) - hash, parent_hash, name, date, comment = items - - commit = Commit(hash, parent_hash, name, date, comment, - *self.was_target_changed(hash, target, settings), - branch_tips=branches_multimap.get(hash)) - wf.add_commit(commit) - - return wf - except ExecutorError: - raise - - def was_target_changed(self, hash, target, settings): - git_cmd = ['git', 'show', '--pretty=', '--name-only', hash] - changed_files = set(Executor.exec_cmd_only_success(git_cmd).split('\n')) - - if target not in changed_files: - return False, None - - metric = utils.parse_target_metric_file(target) - if metric is None: - return False, None - - return True, metric - - @staticmethod - def get_merges_map(): - # git log --merges --all --abbrev=7 - # {'a4b56f1': back_to_600_est} - git_cmd = ['git', 'log', '--all', '--merges', - '--abbrev={}'.format(GitWrapper.COMMIT_LEN)] - # lines = map(str.strip, Executor.exec_cmd_only_success(git_cmd).split('\n')) - # lines.map - return {} - - @staticmethod - def branches_multimap(): - git_cmd = ['git', 'show-ref', '--abbrev={}'.format(GitWrapper.COMMIT_LEN)] - lines = Executor.exec_cmd_only_success(git_cmd).split('\n') - items_full = map(unicode.split, lines) - items = map(lambda it: (it[0], re.sub(r'^refs/heads/', '', it[1])), items_full) - - result = defaultdict(list) - for (hash, branch) in items: - result[hash].append(branch) - return result - - @staticmethod - def git_prev_commit(): - cmd = 'git check-ref-format --branch @{-1}' - code, out, err = Executor.exec_cmd(cmd.split()) - if code != 0: - return '' - return out - - @staticmethod - def branches(regexp): - cmd = 'git branch --format "%(refname)"' - lines = Executor.exec_cmd_only_success(cmd.split()).split('\n') - lines = map(lambda l: l.strip('"'), lines) - - if not regexp or '/' not in regexp: - # remove remotes like 'refs/heads' - lines = map(lambda s: s.split('/')[-1], lines) - - if not regexp: - return lines - - reg = re.compile(regexp) - return list(filter(reg.search, lines)) - - @staticmethod - def get_file_content(fname, branch): - cmd = 'git show {}:{}'.format(branch, fname) - return Executor.exec_cmd_only_success(cmd.split()) - - @staticmethod - def all_untracked_files(): - cmd = 'git ls-files --others --exclude-standard' - res = Executor.exec_cmd_only_success(cmd.split()) - if not res.strip(): - return [] - return res.split('\n') From 733fe7aac475f36460ff54a1ef846cffc980f9a4 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 7 Jan 2018 17:16:04 +0200 Subject: [PATCH 8/8] test: remove test-merge.py Signed-off-by: Ruslan Kuprieiev --- functest/run.sh | 3 +-- functest/test-merge.sh | 57 ------------------------------------------ 2 files changed, 1 insertion(+), 59 deletions(-) delete mode 100755 functest/test-merge.sh diff --git a/functest/run.sh b/functest/run.sh index ea81b6038..1a3fd5e10 100755 --- a/functest/run.sh +++ b/functest/run.sh @@ -8,7 +8,6 @@ TESTS+=" test-init.sh" TESTS+=" test-add-remove.sh" TESTS+=" test-sync.sh" TESTS+=" test-repro-code.sh" -#TESTS+=" test-merge.sh" TESTS+=" test-checkout.sh" TESTS+=" test-gc.sh" TESTS+=" test-lock.sh" @@ -19,4 +18,4 @@ for t in $TESTS; do ./$t done -#rm -rf $TEST_REPO +rm -rf $TEST_REPO diff --git a/functest/test-merge.sh b/functest/test-merge.sh deleted file mode 100755 index 21b01d991..000000000 --- a/functest/test-merge.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -set -e - -source common.sh - -dvc_create_repo - -# Adding changes to the first branch -git checkout -b foo -cp code/code.sh code/code1.sh -chmod +x code/code1.sh -cp code/code1.sh code/code2.sh -cp code/code1.sh code/code3.sh -cp code/code1.sh code/code4.sh -git add code -git commit -s -m"add code" - -cp $DATA_CACHE/foo data/data -dvc add data/data - -ORIG_DATA=$(dvc_md5 data/data) - -dvc run -f copy_data_data1.dvc -d code/code1.sh -d data/data -o data/data1 ./code/code1.sh data/data data/data1 -dvc run -f copy_data1_data2.dvc -d code/code2.sh -d data/data1 -o data/data2 ./code/code2.sh data/data1 data/data2 -dvc run -f copy_data2_data3.dvc -d code/code3.sh -d data/data2 -o data/data3 ./code/code3.sh data/data2 data/data3 -dvc run -f copy_data3_data4.dvc -d code/code4.sh -d data/data3 -o data/data4 ./code/code4.sh data/data3 data/data4 - -dvc config Global.Target data/data4 -git commit -am 'Set Target' - -# Create new branch to work on small data set -git checkout -b bar -dvc remove -c -l data/data -cp $DATA_CACHE/bar data/data -dvc add data/data -dvc repro - -echo -e "\n\n\n\n" >> code/code1.sh -git commit -am"change code1.sh" -dvc repro - -# Merge bar into foo -git checkout foo -git merge bar -# Notice we're running 'dvc merge' on foo branch. This -# way dvc knows to favor data files from current branch. -dvc merge - -# Verify that data and respective cache state file are restored properly -if [ "$(dvc_md5 data/data)" != "$ORIG_DATA" ]; then - dvc_fail -fi - -grep $(echo "$ORIG_DATA" | cut -d " " -f1) data/data.dvc || dvc_fail - -dvc_pass