diff --git a/dvc/cli.py b/dvc/cli.py index a82d3a854..3a0379f2a 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -29,6 +29,7 @@ import dvc.command.root as root import dvc.command.lock as lock import dvc.command.pipeline as pipeline import dvc.command.daemon as daemon +import dvc.command.commit as commit from dvc.exceptions import DvcParserError from dvc import VERSION @@ -56,6 +57,7 @@ COMMANDS = [ lock, pipeline, daemon, + commit, ] diff --git a/dvc/command/commit.py b/dvc/command/commit.py new file mode 100644 index 000000000..f45763e3b --- /dev/null +++ b/dvc/command/commit.py @@ -0,0 +1,63 @@ +from __future__ import unicode_literals + +import dvc.logger as logger +from dvc.exceptions import DvcException +from dvc.command.base import CmdBase + + +class CmdCommit(CmdBase): + def run(self): + if not self.args.targets: + self.args.targets = [None] + + for target in self.args.targets: + try: + self.repo.commit( + target, + with_deps=self.args.with_deps, + recursive=self.args.recursive, + force=self.args.force, + ) + except DvcException: + logger.error( + "failed to commit{}".format( + (" " + target) if target else "" + ) + ) + return 1 + return 0 + + +def add_parser(subparsers, parent_parser): + COMMIT_HELP = "Record changes to the repository." + commit_parser = subparsers.add_parser( + "commit", + parents=[parent_parser], + description=COMMIT_HELP, + help=COMMIT_HELP, + ) + commit_parser.add_argument( + "-f", + "--force", + action="store_true", + default=False, + help="Commit even if checksums for dependencies/outputs changed.", + ) + commit_parser.add_argument( + "-d", + "--with-deps", + action="store_true", + default=False, + help="Commit all dependencies of the specified target.", + ) + commit_parser.add_argument( + "-R", + "--recursive", + action="store_true", + default=False, + help="Commit cache for subdirectories of the specified directory.", + ) + commit_parser.add_argument( + "targets", nargs="*", default=None, help="DVC files." + ) + commit_parser.set_defaults(func=CmdCommit) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 7b7a74291..889abc768 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -28,6 +28,7 @@ class Repo(object): from dvc.repo.pull import pull from dvc.repo.status import status from dvc.repo.gc import gc + from dvc.repo.commit import commit def __init__(self, root_dir=None): from dvc.config import Config diff --git a/dvc/repo/commit.py b/dvc/repo/commit.py new file mode 100644 index 000000000..ae4b9007c --- /dev/null +++ b/dvc/repo/commit.py @@ -0,0 +1,11 @@ +def commit(self, target, with_deps=False, recursive=False, force=False): + if target and not recursive: + stages = self.collect(target, with_deps=with_deps) + else: + stages = self.active_stages(target) + + with self.state: + for stage in stages: + stage.check_can_commit(force=force) + stage.commit() + stage.dump() diff --git a/dvc/stage.py b/dvc/stage.py index 187201f45..b3c51d3d2 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -67,6 +67,10 @@ class StageBadCwdError(DvcException): super(StageBadCwdError, self).__init__(msg.format(cwd)) +class StageCommitError(DvcException): + pass + + class MissingDep(DvcException): def __init__(self, deps): assert len(deps) > 0 @@ -528,6 +532,35 @@ class Stage(object): self.md5 = self._compute_md5() + @staticmethod + def _changed_entries(entries): + ret = [] + for entry in entries: + if entry.checksum and entry.changed_checksum(): + ret.append(entry.rel_path) + return ret + + def check_can_commit(self, force): + changed_deps = self._changed_entries(self.deps) + changed_outs = self._changed_entries(self.outs) + + if changed_deps or changed_outs or self.changed_md5(): + msg = ( + "dependencies {}".format(changed_deps) if changed_deps else "" + ) + msg += " and " if (changed_deps and changed_outs) else "" + msg += "outputs {}".format(changed_outs) if changed_outs else "" + msg += "md5" if not (changed_deps or changed_outs) else "" + msg += " of '{}' changed. Are you sure you commit it?".format( + self.relpath + ) + if not force and not prompt.confirm(msg): + raise StageCommitError( + "unable to commit changed '{}'. Use `-f|--force` to " + "force.`".format(self.relpath) + ) + self.save() + def commit(self): for out in self.outs: out.commit() diff --git a/tests/test_add.py b/tests/test_add.py index bceb84aa6..a835fce03 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -266,9 +266,14 @@ class TestShouldUpdateStateEntryForDirectoryAfterAdd(TestDvc): self.assertEqual(file_md5_counter.mock.call_count, 3) -class TestAddNoCommit(TestDvc): +class TestAddCommit(TestDvc): def test(self): ret = main(["add", self.FOO, "--no-commit"]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.FOO)) self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 0) + + ret = main(["commit", self.FOO + ".dvc"]) + self.assertEqual(ret, 0) + self.assertTrue(os.path.isfile(self.FOO)) + self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 1) diff --git a/tests/test_commit.py b/tests/test_commit.py new file mode 100644 index 000000000..68581225a --- /dev/null +++ b/tests/test_commit.py @@ -0,0 +1,91 @@ +import yaml +import shutil + +from tests.basic_env import TestDvc +from dvc.stage import StageCommitError + + +class TestCommitRecursive(TestDvc): + def test(self): + stages = self.dvc.add(self.DATA_DIR, recursive=True, no_commit=True) + self.assertEqual(len(stages), 2) + + self.assertNotEqual(self.dvc.status(), {}) + + self.dvc.commit(self.DATA_DIR, recursive=True) + + self.assertEqual(self.dvc.status(), {}) + + +class TestCommitForce(TestDvc): + def test(self): + stages = self.dvc.add(self.FOO, no_commit=True) + self.assertEqual(len(stages), 1) + stage = stages[0] + + with self.dvc.state: + self.assertTrue(stage.outs[0].changed_cache()) + + with open(self.FOO, "a") as fobj: + fobj.write(self.FOO_CONTENTS) + + with self.dvc.state: + self.assertTrue(stage.outs[0].changed_cache()) + + with self.assertRaises(StageCommitError): + self.dvc.commit(stage.path) + + with self.dvc.state: + self.assertTrue(stage.outs[0].changed_cache()) + + self.dvc.commit(stage.path, force=True) + + self.assertEqual(self.dvc.status(stage.path), {}) + + +class TestCommitWithDeps(TestDvc): + def test(self): + stages = self.dvc.add(self.FOO, no_commit=True) + self.assertEqual(len(stages), 1) + foo_stage = stages[0] + self.assertTrue(foo_stage is not None) + self.assertEqual(len(foo_stage.outs), 1) + + fname = "file" + stage = self.dvc.run( + cmd="python {} {} {}".format(self.CODE, self.FOO, fname), + outs=[fname], + deps=[self.FOO, self.CODE], + no_commit=True, + ) + self.assertTrue(stage is not None) + self.assertEqual(len(stage.outs), 1) + + with self.dvc.state: + self.assertTrue(foo_stage.outs[0].changed_cache()) + self.assertTrue(stage.outs[0].changed_cache()) + + self.dvc.commit(stage.path, with_deps=True) + with self.dvc.state: + self.assertFalse(foo_stage.outs[0].changed_cache()) + self.assertFalse(stage.outs[0].changed_cache()) + + +class TestCommitChangedMd5(TestDvc): + def test(self): + stages = self.dvc.add(self.FOO, no_commit=True) + self.assertEqual(len(stages), 1) + stage = stages[0] + + st = None + with open(stage.path, "r") as fobj: + st = yaml.safe_load(fobj) + + st["md5"] = "1111111111" + with open(stage.path, "w") as fobj: + yaml.safe_dump(st, fobj, default_flow_style=False) + + with self.assertRaises(StageCommitError): + self.dvc.commit(stage.path) + + self.dvc.commit(stage.path, force=True) diff --git a/tests/test_run.py b/tests/test_run.py index ed48b9db1..98eaa1f75 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -630,12 +630,17 @@ class TestRunDeterministicChangedCmd(TestRunDeterministicBase): self._run() -class TestRunNoCommit(TestDvc): +class TestRunCommit(TestDvc): def test(self): fname = "test" ret = main( - ["run", "-o", "foo", "--no-commit", "echo", "test", ">", fname] + ["run", "-o", self.FOO, "--no-commit", "echo", "test", ">", fname] ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(fname)) self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 0) + + ret = main(["commit", self.FOO + ".dvc"]) + self.assertEqual(ret, 0) + self.assertTrue(os.path.isfile(fname)) + self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 1) diff --git a/tests/unit/command/commit.py b/tests/unit/command/commit.py new file mode 100644 index 000000000..e69de29bb