Merge pull request #48 from dataversioncontrol/implicit_code_dependency

Include implicit code dependencies from argv. Close #46

dvc/command/repro.py

@@ -194,9 +194,10 @@ class ReproChange(object):
             Logger.info('Reproducing run command: {}. Args: {}'.format(
                 self._data_item.data.relative, ' '.join(self.state.argv)))
 
-            data_items_from_args = self.cmd_obj.data_items_from_args(self.state.argv)
+            data_items_from_args, not_data_items_from_args = self.cmd_obj.argv_files_by_type(self.state.argv)
             if cmd.run_and_commit_if_needed(self.state.argv,
                                             data_items_from_args,
+                                            not_data_items_from_args,
                                             self.state.stdout,
                                             self.state.stderr,
                                             self.state.shell,

dvc/command/run.py

@@ -5,7 +5,7 @@ from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 from dvc.git_wrapper import GitWrapper
 from dvc.logger import Logger
-from dvc.path.data_item import NotInDataDirError
+from dvc.path.data_item import NotInDataDirError, NotInGitDirError
 from dvc.repository_change import RepositoryChange
 from dvc.runtime import Runtime
 from dvc.state_file import StateFile
@@ -62,8 +62,10 @@ class CmdRun(CmdBase):
                 return 1
 
         try:
+            data_items_from_args, not_data_items_from_args = self.argv_files_by_type(self.command_args)
             return self.run_and_commit_if_needed(self.command_args,
-                                                 self.data_items_from_args(self.command_args),
+                                                 data_items_from_args,
+                                                 not_data_items_from_args,
                                                  self.parsed_args.stdout,
                                                  self.parsed_args.stderr,
                                                  self.parsed_args.shell)
@@ -71,20 +73,22 @@ class CmdRun(CmdBase):
             if self.is_locker:
                 lock.release()
 
-    def run_and_commit_if_needed(self, command_args, command_args_data_items,
+    def run_and_commit_if_needed(self, command_args, data_items_from_args, not_data_items_from_args,
                                  stdout, stderr, shell, check_if_ready=True):
         if check_if_ready and not self.skip_git_actions and not self.git.is_ready_to_go():
             return 1
 
         self.run_command(command_args,
-                         command_args_data_items,
+                         data_items_from_args,
+                         not_data_items_from_args,
                          stdout,
                          stderr,
                          shell)
 
         return self.commit_if_needed('DVC run: {}'.format(' '.join(self.args)))
 
-    def run_command(self, cmd_args, data_items_from_args, stdout=None, stderr=None, shell=False):
+    def run_command(self, cmd_args, data_items_from_args, not_data_items_from_args,
+                    stdout=None, stderr=None, shell=False):
         Logger.debug('Run command with args: {}. Data items from args: {}. stdout={}, stderr={}, shell={}'.format(
                      ' '.join(cmd_args),
                      ', '.join([x.data.dvc for x in data_items_from_args]),
@@ -104,7 +108,7 @@ class CmdRun(CmdBase):
         input_set = set(data_items_from_args + self.declaration_input_data_items) - output_set
         input_files_dvc = [x.data.dvc for x in input_set]
 
-        code_dependencies_dvc = self.git.abs_paths_to_dvc(self.code_dependencies)
+        code_dependencies_dvc = self.git.abs_paths_to_dvc(self.code_dependencies + not_data_items_from_args)
 
         result = []
         for data_item in repo_change.changed_data_items:
@@ -155,18 +159,23 @@ class CmdRun(CmdBase):
 
         return not error
 
-    def data_items_from_args(self, argv):
-        result = []
+    def argv_files_by_type(self, argv):
+        data_items = []
+        not_data_items = []
 
         for arg in argv:
             try:
                 if os.path.isfile(arg):
                     data_item = self.settings.path_factory.data_item(arg)
-                    result.append(data_item)
+                    data_items.append(data_item)
+            except NotInGitDirError as ex:
+                msg = 'File {} from argv is outside of git directory and cannot be traced: {}'
+                Logger.warn(msg.format(arg, ex))
             except NotInDataDirError:
+                not_data_items.append(arg)
                 pass
 
-        return result
+        return data_items, not_data_items
 
     def _data_items_from_params(self, files, param_text):
         if not files:

dvc/path/data_item.py

@@ -33,6 +33,16 @@ class DataItemInStatusDirError(NotInDataDirError):
         return DataItemInStatusDirError(msg)
 
 
+class NotInGitDirError(NotInDataDirError):
+    def __init__(self, msg):
+        super(NotInGitDirError, self).__init__(msg)
+
+    @staticmethod
+    def build(file, git_dir):
+        msg = 'the file "{}" is not in git directory "{}"'.format(file, git_dir)
+        return NotInGitDirError(msg)
+
+
 class DataItem(object):
     STATE_FILE_SUFFIX = '.state'
     CACHE_FILE_SEP = '_'
@@ -44,6 +54,9 @@ class DataItem(object):
 
         self._data = Path(data_file, git)
 
+        if not self._data.abs.startswith(self._git.git_dir_abs):
+            raise NotInGitDirError.build(data_file, self._git.git_dir_abs)
+
         if self._data.abs.startswith(self.state_dir_abs):
             raise DataItemInStatusDirError.build(data_file, self._config.data_dir)
 

tests/test_cmd_run.py

@@ -56,7 +56,7 @@ class TestRunOutsideData(RunBasicTest):
         self.settings._args = []
         cmd_run = CmdRun(self.settings)
         with self.assertRaises(RunError):
-            cmd_run.run_command(['echo', 'test'], [], shell=True,
+            cmd_run.run_command(['echo', 'test'], [], [], shell=True,
                                 stdout='file1', stderr='file2')
         pass
 
@@ -75,7 +75,7 @@ class RunTwoFilesBase(RunBasicTest):
 
         self.file_name1 = os.path.join('data', 'file1')
         self.file_name2 = os.path.join('data', 'file2')
-        self.state_objs = cmd_run.run_command(['echo', 'test'], [], shell=True,
+        self.state_objs = cmd_run.run_command(['echo', 'test'], [], [], shell=True,
                                               stdout=self.file_name1,
                                               stderr=self.file_name2)