dvc: introduce dvc commit

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/cli.py

@@ -29,6 +29,7 @@ import dvc.command.root as root
 import dvc.command.lock as lock
 import dvc.command.pipeline as pipeline
 import dvc.command.daemon as daemon
+import dvc.command.commit as commit
 from dvc.exceptions import DvcParserError
 from dvc import VERSION
 
@@ -56,6 +57,7 @@ COMMANDS = [
     lock,
     pipeline,
     daemon,
+    commit,
 ]
 
 

dvc/command/commit.py

@@ -0,0 +1,63 @@
+from __future__ import unicode_literals
+
+import dvc.logger as logger
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+
+
+class CmdCommit(CmdBase):
+    def run(self):
+        if not self.args.targets:
+            self.args.targets = [None]
+
+        for target in self.args.targets:
+            try:
+                self.repo.commit(
+                    target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                    force=self.args.force,
+                )
+            except DvcException:
+                logger.error(
+                    "failed to commit{}".format(
+                        (" " + target) if target else ""
+                    )
+                )
+                return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    COMMIT_HELP = "Record changes to the repository."
+    commit_parser = subparsers.add_parser(
+        "commit",
+        parents=[parent_parser],
+        description=COMMIT_HELP,
+        help=COMMIT_HELP,
+    )
+    commit_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Commit even if checksums for dependencies/outputs changed.",
+    )
+    commit_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Commit all dependencies of the specified target.",
+    )
+    commit_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Commit cache for subdirectories of the specified directory.",
+    )
+    commit_parser.add_argument(
+        "targets", nargs="*", default=None, help="DVC files."
+    )
+    commit_parser.set_defaults(func=CmdCommit)

dvc/repo/__init__.py

@@ -28,6 +28,7 @@ class Repo(object):
     from dvc.repo.pull import pull
     from dvc.repo.status import status
     from dvc.repo.gc import gc
+    from dvc.repo.commit import commit
 
     def __init__(self, root_dir=None):
         from dvc.config import Config

dvc/repo/commit.py

@@ -0,0 +1,11 @@
+def commit(self, target, with_deps=False, recursive=False, force=False):
+    if target and not recursive:
+        stages = self.collect(target, with_deps=with_deps)
+    else:
+        stages = self.active_stages(target)
+
+    with self.state:
+        for stage in stages:
+            stage.check_can_commit(force=force)
+            stage.commit()
+            stage.dump()

dvc/stage.py

@@ -67,6 +67,10 @@ class StageBadCwdError(DvcException):
         super(StageBadCwdError, self).__init__(msg.format(cwd))
 
 
+class StageCommitError(DvcException):
+    pass
+
+
 class MissingDep(DvcException):
     def __init__(self, deps):
         assert len(deps) > 0
@@ -528,6 +532,35 @@ class Stage(object):
 
         self.md5 = self._compute_md5()
 
+    @staticmethod
+    def _changed_entries(entries):
+        ret = []
+        for entry in entries:
+            if entry.checksum and entry.changed_checksum():
+                ret.append(entry.rel_path)
+        return ret
+
+    def check_can_commit(self, force):
+        changed_deps = self._changed_entries(self.deps)
+        changed_outs = self._changed_entries(self.outs)
+
+        if changed_deps or changed_outs or self.changed_md5():
+            msg = (
+                "dependencies {}".format(changed_deps) if changed_deps else ""
+            )
+            msg += " and " if (changed_deps and changed_outs) else ""
+            msg += "outputs {}".format(changed_outs) if changed_outs else ""
+            msg += "md5" if not (changed_deps or changed_outs) else ""
+            msg += " of '{}' changed. Are you sure you commit it?".format(
+                self.relpath
+            )
+            if not force and not prompt.confirm(msg):
+                raise StageCommitError(
+                    "unable to commit changed '{}'. Use `-f|--force` to "
+                    "force.`".format(self.relpath)
+                )
+            self.save()
+
     def commit(self):
         for out in self.outs:
             out.commit()

tests/test_add.py

@@ -266,9 +266,14 @@ class TestShouldUpdateStateEntryForDirectoryAfterAdd(TestDvc):
             self.assertEqual(file_md5_counter.mock.call_count, 3)
 
 
-class TestAddNoCommit(TestDvc):
+class TestAddCommit(TestDvc):
     def test(self):
         ret = main(["add", self.FOO, "--no-commit"])
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(self.FOO))
         self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 0)
+
+        ret = main(["commit", self.FOO + ".dvc"])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.FOO))
+        self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 1)

tests/test_commit.py

@@ -0,0 +1,91 @@
+import yaml
+import shutil
+
+from tests.basic_env import TestDvc
+from dvc.stage import StageCommitError
+
+
+class TestCommitRecursive(TestDvc):
+    def test(self):
+        stages = self.dvc.add(self.DATA_DIR, recursive=True, no_commit=True)
+        self.assertEqual(len(stages), 2)
+
+        self.assertNotEqual(self.dvc.status(), {})
+
+        self.dvc.commit(self.DATA_DIR, recursive=True)
+
+        self.assertEqual(self.dvc.status(), {})
+
+
+class TestCommitForce(TestDvc):
+    def test(self):
+        stages = self.dvc.add(self.FOO, no_commit=True)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+
+        with self.dvc.state:
+            self.assertTrue(stage.outs[0].changed_cache())
+
+        with open(self.FOO, "a") as fobj:
+            fobj.write(self.FOO_CONTENTS)
+
+        with self.dvc.state:
+            self.assertTrue(stage.outs[0].changed_cache())
+
+        with self.assertRaises(StageCommitError):
+            self.dvc.commit(stage.path)
+
+        with self.dvc.state:
+            self.assertTrue(stage.outs[0].changed_cache())
+
+        self.dvc.commit(stage.path, force=True)
+
+        self.assertEqual(self.dvc.status(stage.path), {})
+
+
+class TestCommitWithDeps(TestDvc):
+    def test(self):
+        stages = self.dvc.add(self.FOO, no_commit=True)
+        self.assertEqual(len(stages), 1)
+        foo_stage = stages[0]
+        self.assertTrue(foo_stage is not None)
+        self.assertEqual(len(foo_stage.outs), 1)
+
+        fname = "file"
+        stage = self.dvc.run(
+            cmd="python {} {} {}".format(self.CODE, self.FOO, fname),
+            outs=[fname],
+            deps=[self.FOO, self.CODE],
+            no_commit=True,
+        )
+        self.assertTrue(stage is not None)
+        self.assertEqual(len(stage.outs), 1)
+
+        with self.dvc.state:
+            self.assertTrue(foo_stage.outs[0].changed_cache())
+            self.assertTrue(stage.outs[0].changed_cache())
+
+        self.dvc.commit(stage.path, with_deps=True)
+        with self.dvc.state:
+            self.assertFalse(foo_stage.outs[0].changed_cache())
+            self.assertFalse(stage.outs[0].changed_cache())
+
+
+class TestCommitChangedMd5(TestDvc):
+    def test(self):
+        stages = self.dvc.add(self.FOO, no_commit=True)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+
+        st = None
+        with open(stage.path, "r") as fobj:
+            st = yaml.safe_load(fobj)
+
+        st["md5"] = "1111111111"
+        with open(stage.path, "w") as fobj:
+            yaml.safe_dump(st, fobj, default_flow_style=False)
+
+        with self.assertRaises(StageCommitError):
+            self.dvc.commit(stage.path)
+
+        self.dvc.commit(stage.path, force=True)

tests/test_run.py

@@ -630,12 +630,17 @@ class TestRunDeterministicChangedCmd(TestRunDeterministicBase):
             self._run()
 
 
-class TestRunNoCommit(TestDvc):
+class TestRunCommit(TestDvc):
     def test(self):
         fname = "test"
         ret = main(
-            ["run", "-o", "foo", "--no-commit", "echo", "test", ">", fname]
+            ["run", "-o", self.FOO, "--no-commit", "echo", "test", ">", fname]
         )
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(fname))
         self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 0)
+
+        ret = main(["commit", self.FOO + ".dvc"])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(fname))
+        self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 1)