Merge pull request #896 from efiop/master

dvc: support `dvc add` for external files

dvc/command/run.py

@@ -8,18 +8,13 @@ from dvc.exceptions import DvcException
 
 class CmdRun(CmdBase):
     def run(self):
-        fname = self.stage_file_name(self.args.file,
-                                     self.args.outs,
-                                     self.args.outs_no_cache,
-                                     self.args.metrics_no_cache)
-
         try:
             self.project.run(cmd=' '.join(self.args.command),
                              outs=self.args.outs,
                              outs_no_cache=self.args.outs_no_cache,
                              metrics_no_cache=self.args.metrics_no_cache,
                              deps=self.args.deps,
-                             fname=fname,
+                             fname=self.args.file,
                              cwd=self.args.cwd,
                              no_exec=self.args.no_exec)
         except DvcException as ex:
@@ -27,28 +22,3 @@ class CmdRun(CmdBase):
             return 1
 
         return 0
-
-    @staticmethod
-    def stage_file_name(args_file, args_outs, args_outs_no_cache, args_metrics_no_cache):
-        if args_file:
-            return args_file
-
-        if args_outs:
-            result = CmdRun._stage_file_basename(args_outs[0])
-        elif args_outs_no_cache:
-            result = CmdRun._stage_file_basename(args_outs_no_cache[0])
-        elif args_metrics_no_cache:
-            result = CmdRun._stage_file_basename(args_metrics_no_cache[0])
-        else:
-            result = Stage.STAGE_FILE
-
-        Logger.info(u'Using \'{}\' as a stage file'.format(result))
-        return result
-
-    @staticmethod
-    def _stage_file_basename(fname):
-        result = os.path.basename(fname)
-        if len(result) == 0:
-            result = os.path.basename(os.path.dirname(fname))
-        result += Stage.STAGE_FILE_SUFFIX
-        return result

dvc/project.py

@@ -130,15 +130,8 @@ class Project(object):
         return os.path.relpath(path, self.root_dir)
 
     def add(self, fname):
-        out = os.path.basename(os.path.normpath(fname))
-        stage_fname = out + Stage.STAGE_FILE_SUFFIX
-        cwd = os.path.dirname(os.path.abspath(fname))
         stage = Stage.loads(project=self,
-                            cmd=None,
-                            deps=[],
-                            outs=[out],
-                            fname=stage_fname,
-                            cwd=cwd)
+                            outs=[fname])
 
         stage.save()
         stage.dump()
@@ -527,8 +520,11 @@ class Project(object):
     def pipelines(self):
         G, G_active = self.graph()
 
+        if len(G.nodes()) == 0:
+            return []
+
         # find pipeline ends aka "output stages"
-        ends = [node for node, in_degree in G.in_degree if in_degree == 0]
+        ends = [node for node, in_degree in G.in_degree() if in_degree == 0]
 
         # filter out subgraphs that didn't exist in original G
         pipelines = []

dvc/stage.py

@@ -3,6 +3,7 @@ import yaml
 import itertools
 import subprocess
 import schema
+import posixpath
 
 import dvc.dependency as dependency
 import dvc.output as output
@@ -52,7 +53,7 @@ class Stage(object):
         schema.Optional(PARAM_LOCKED): bool,
     }
 
-    def __init__(self, project, path=None, cmd=None, cwd=None, deps=[], outs=[], md5=None, locked=False):
+    def __init__(self, project, path=None, cmd=None, cwd=os.curdir, deps=[], outs=[], md5=None, locked=False):
         self.project = project
         self.path = path
         self.cmd = cmd
@@ -184,6 +185,25 @@ class Stage(object):
 
         return stage
 
+    @classmethod
+    def _stage_fname_cwd(cls, fname, cwd, outs):
+        if fname and cwd:
+            return (fname, cwd)
+
+        if not outs:
+            return (cls.STAGE_FILE, cwd if cwd else os.getcwd())
+
+        out = outs[0]
+        if out.path_info['scheme'] == 'local':
+            path = os.path
+        else:
+            path = posixpath
+
+        fname = fname if fname else path.basename(out.path) + cls.STAGE_FILE_SUFFIX
+        cwd = cwd if cwd else path.dirname(out.path)
+
+        return (fname, cwd)
+
     @staticmethod
     def loads(project=None,
               cmd=None,
@@ -194,13 +214,9 @@ class Stage(object):
               fname=None,
               cwd=os.curdir,
               locked=False):
-        cwd = os.path.abspath(cwd)
-        path = os.path.join(cwd, fname)
-
         stage = Stage(project=project,
-                      path=path,
-                      cmd=cmd,
                       cwd=cwd,
+                      cmd=cmd,
                       locked=locked)
 
         stage.outs = output.loads_from(stage, outs, use_cache=True)
@@ -208,6 +224,14 @@ class Stage(object):
         stage.outs += output.loads_from(stage, metrics_no_cache, use_cache=False, metric=True)
         stage.deps = dependency.loads_from(stage, deps)
 
+        fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs)
+
+        cwd = os.path.abspath(cwd)
+        path = os.path.join(cwd, fname)
+
+        stage.cwd = cwd
+        stage.path = path
+
         return stage
 
     @staticmethod

requirements.txt

@@ -8,7 +8,7 @@ google-cloud==0.32.0
 PyInstaller==3.3.1
 colorama>=0.3.9
 configobj>=5.0.6
-networkx>=1.11
+networkx==2.1
 pyyaml>=3.12
 gitpython>=2.1.8
 ntfsutils>=0.1.4

setup.py

@@ -11,7 +11,7 @@ install_requires = [
     "future>=0.16.0",
     "colorama>=0.3.9",
     "configobj>=5.0.6",
-    "networkx>=1.11",
+    "networkx==2.1",
     "pyyaml>=3.12",
     "gitpython>=2.1.8",
     "ntfsutils>=0.1.4",

tests/test_run.py

@@ -70,23 +70,3 @@ class TestCmdRun(TestDvc):
         ret = main(['run',
                     'non-existing-command'])
         self.assertNotEqual(ret, 0)
-
-    def test_stage_file_name(self):
-        fname = 'path/to/file'
-        outs = [fname, 'dummy']
-        dvcfile = os.path.basename(fname) + Stage.STAGE_FILE_SUFFIX
-
-        ret = CmdRun.stage_file_name(None, [], [], [])
-        self.assertEqual(ret, Stage.STAGE_FILE)
-
-        ret = CmdRun.stage_file_name(None, [], outs, [])
-        self.assertEqual(ret, dvcfile)
-
-        ret = CmdRun.stage_file_name(None, outs, [], [])
-        self.assertEqual(ret, dvcfile)
-
-        ret = CmdRun.stage_file_name(None, [], [], outs)
-        self.assertEqual(ret, dvcfile)
-
-        ret = CmdRun.stage_file_name(fname, [], [], [])
-        self.assertEqual(ret, fname)