Merge pull request #361 from dataversioncontrol/after_design_change_fixes

After design change fixes

dvc/cli.py

@@ -118,7 +118,6 @@ def parse_args(argv=None):
                         help='Lock data item - disable reproduction.')
     run_parser.add_argument('-f',
                         '--file',
-                        default=Stage.STAGE_FILE,
                         help='Specify name of the state file')
     run_parser.add_argument('-c',
                         '--cwd',

dvc/command/checkout.py

@@ -1,7 +1,35 @@
+import os
+
 from dvc.command.common.base import CmdBase
+from dvc.logger import Logger
+from dvc.system import System
 
 
 class CmdCheckout(CmdBase):
     def run(self):
+        self.remove_untracked_hardlinks()
         self.project.checkout()
         return 0
+
+    def remove_untracked_hardlinks(self):
+        untracked_cache_files = self.untracked_hardlinks_files()
+
+        for file in untracked_cache_files:
+            Logger.info(u'Remove \'{}\''.format(file))
+            os.remove(file)
+
+            dir = os.path.dirname(file)
+            if not os.listdir(dir):
+                Logger.info(u'Remove empty directory \'{}\''.format(dir))
+                os.removedirs(dir)
+        pass
+
+    def untracked_hardlinks_files(self):
+        untracked_files = set(self.project.scm.untracked_files())
+        cached_files = []
+        for cache_file in self.project.cache.all():
+            hardlinks = list(filter(lambda f: System.samefile(cache_file, f), untracked_files))
+            cached_files.extend(hardlinks)
+            untracked_files = untracked_files - set(hardlinks)
+
+        return cached_files

dvc/command/common/base.py

@@ -1,5 +1,6 @@
 from dvc.project import Project
 
+
 class CmdBase(object):
     def __init__(self, args):
         self.project = Project('.')

dvc/command/init.py

@@ -1,5 +1,6 @@
 from dvc.project import Project
 
+
 class CmdInit(object):
     def __init__(self, args):
         self.args = args

dvc/command/run.py

@@ -1,14 +1,41 @@
+import os
+
 from dvc.command.common.base import CmdBase
+from dvc.logger import Logger
+from dvc.stage import Stage
 
 
 class CmdRun(CmdBase):
     def run(self):
+        fname = self.stage_file_name(self.args.file, self.args.outs, self.args.outs_no_cache)
+
         self.project.run(cmd=' '.join(self.args.command),
                          outs=self.args.outs,
                          outs_no_cache=self.args.outs_no_cache,
                          deps=self.args.deps,
                          deps_no_cache=self.args.deps_no_cache,
                          locked=self.args.lock,
-                         fname=self.args.file,
+                         fname=fname,
                          cwd=self.args.cwd)
         return 0
+
+    @staticmethod
+    def stage_file_name(args_file, args_outs, args_outs_no_cache):
+        if args_file:
+            return args_file
+
+        if args_outs:
+            result = CmdRun._stage_file_basename(args_outs[0])
+        elif args_outs_no_cache:
+            result = CmdRun._stage_file_basename(args_outs_no_cache[0])
+        else:
+            result = Stage.STAGE_FILE
+
+        Logger.info(u'Using \'{}\' as a stage file'.format(result))
+        return result
+
+    @staticmethod
+    def _stage_file_basename(fname):
+        result = os.path.basename(fname)
+        result += Stage.STAGE_FILE_SUFFIX
+        return result

dvc/project.py

@@ -86,6 +86,9 @@ class Project(object):
         scm.ignore_list([cache.cache_dir,
                          state.state_file,
                          lock.lock_file])
+
+        ignore_file = os.path.join(dvc_dir, scm.ignore_file())
+        scm.add([config.config_file, ignore_file])
         scm.commit('DVC init')
 
         return Project(root_dir)

dvc/scm.py

@@ -17,15 +17,15 @@ class Base(object):
     def ignore(self, path):
         pass
 
+    def ignore_file(self):
+        pass
+
     def ignore_list(self, p_list):
         return [self.ignore(path) for path in p_list]
 
-    def add(self, path):
+    def add(self, paths):
         pass
 
-    def add_list(self, p_list):
-        return [self.add(path) for path in p_list]
-
     def commit(self, msg):
         pass
 
@@ -38,6 +38,9 @@ class Base(object):
     def brancher(self, branch, new_branch):
         return BranchChanger(self, branch, new_branch)
 
+    def untracked_files(self):
+        pass
+
 
 class Git(Base):
     GITIGNORE = '.gitignore'
@@ -59,13 +62,23 @@ class Git(Base):
         if not gitignore.startswith(self.root_dir):
             raise FileNotInRepoError()
 
-        if os.path.exists(gitignore) and entry in open(gitignore, 'r').read():
-            return
+        ignore_list = []
+        if os.path.exists(gitignore):
+            ignore_list = open(gitignore, 'r').readlines()
+            if entry in ignore_list:
+                return
 
-        open(gitignore, 'a').write('\n' + entry)
+        content = entry
+        if ignore_list:
+            content = '\n' + content
 
-    def add(self, path):
-        self.repo.index.add(path)
+        open(gitignore, 'a').write(content)
+
+    def ignore_file(self):
+        return self.GITIGNORE
+
+    def add(self, paths):
+        self.repo.index.add(paths)
 
     def commit(self, msg):
         self.repo.index.commit(msg)
@@ -79,6 +92,9 @@ class Git(Base):
     def branch(self, branch):
         self.repo.git.branch(branch)
 
+    def untracked_files(self):
+        return self.repo.untracked_files
+
 
 def SCM(root_dir):
     if Git.is_repo(root_dir):

dvc/stage.py

@@ -31,7 +31,9 @@ class Output(object):
         if not self.use_cache:
             changed = self._changed_md5()
         else:
-            changed = not os.path.samefile(self.path, self.cache)
+            changed = os.path.exists(self.path) \
+                      and os.path.exists(self.cache) \
+                      and not os.path.samefile(self.path, self.cache)
 
         if changed:
             self.project.logger.debug('{} changed'.format(self.path))
@@ -173,8 +175,9 @@ class Stage(object):
 
     def remove_outs(self):
         for out in self.outs:
-            self.project.logger.debug("Removing '{}'".format(out.path))
-            os.unlink(out.path)
+            if os.path.exists(out.path):
+                self.project.logger.debug("Removing '{}'".format(out.path))
+                os.unlink(out.path)
 
     def remove(self):
         self.remove_outs()

functest/test-lock.sh

@@ -6,7 +6,7 @@ source common.sh
 
 dvc_create_repo
 
-dvc run -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
+dvc run -f Dvcfile -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
 cat Dvcfile | grep 'locked' | grep 'false' || dvc_fatal "stage locked after 'dvc run'"
 dvc lock
 cat Dvcfile | grep 'locked' | grep 'true' || dvc_fatal "stage not locked after 'dvc lock'"

functest/test-repro-code.sh

@@ -17,7 +17,7 @@ function test_generic() {
 	git commit -am 'Change code'
 	    
 	dvc_info 'Reproduce foo1'
-	dvc repro
+	dvc repro foo1.dvc
 	dvc_check_files data/foo1 data/foo1.dvc
 	if [ "$(cat data/foo1)" != "foo" ]; then
 		dvc_fail
@@ -28,7 +28,7 @@ function test_generic() {
 	cp $DATA_CACHE/bar data/foo
 
 	dvc_info 'Reproduce foo1 as default target'
-	dvc repro
+	dvc repro foo1.dvc
 	dvc_check_files data/foo1 data/foo1.dvc
 	if [ "$(cat data/foo1)" != "bar" ]; then
 		dvc_fail

functest/tutorial_branches.sh

@@ -28,40 +28,48 @@ dvc init
 
 git checkout -b input_100K
 mkdir data
-dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz data/
-dvc run tar zxf data/Posts.xml.tgz -C data/
-dvc run python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
-
+wget https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz -O data/Posts.xml.tgz
+#cp ../stackoverflow_small_xml/100K/Posts.xml.tgz data/
+dvc add data/Posts.xml.tgz
+dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/
+dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
+git add .
+git commit -m 'Extract TSV for 100K dataset'
 
 git checkout master
 dvc checkout
 git checkout -b input_25K
 dvc checkout
 mkdir data
-dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz data/
-dvc run tar zxf data/Posts.xml.tgz -C data/
+wget https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz -O data/Posts.xml.tgz
+#cp ../stackoverflow_small_xml/25K/Posts.xml.tgz data/
+dvc add data/Posts.xml.tgz
+dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/
 
-dvc run python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
-dvc run python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv
-dvc run python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p
+dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
+dvc run -d code/split_train_test.py -d data/Posts.tsv -o data/Posts-train.tsv -o data/Posts-test.tsv python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv
+dvc run -d code/featurization.py -d data/Posts-train.tsv -d data/Posts-test.tsv -o data/matrix-train.p -o data/matrix-test.p python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p
 
-dvc run python code/train_model.py data/matrix-train.p 20170426 data/model.p
+dvc run -d code/train_model.py -d data/matrix-train.p -o data/model.p python code/train_model.py data/matrix-train.p 20170426 data/model.p
 
-dvc run python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt
+dvc run -d code/evaluate.py -d data/model.p -d data/matrix-test.p -O data/eval_auc.txt python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt
 
-dvc config Global.Target data/eval_auc.txt
-git commit -am 'Set Target'
+#dvc config Global.Target data/eval_auc.txt
+#git commit -am 'Set Target'
 
 cat data/eval_auc.txt
 # AUC: 0.596182
 
+git add .
+git commit -m 'Full pipeline'
+
 
 git checkout input_100K
 dvc checkout
 git merge -X theirs input_25K # git merge -X theirs input_25K
-# Resolve conflicts by hends!!!
-git add .
-git commit -m 'Merge conflicts'
+## Resolve conflicts by hands!!!
+#git add .
+#git commit -m 'Merge conflicts'
 dvc merge
 dvc repro       # <-- Error. Reproducng Posts.xml and Posts.tsv