RFC: move cmdline parsing to cli.py and settings RFC

This will make the structure of dvc more linear
making it easier to read. Plus, these changes
are needed in preparation for libdvc, as they
move all cmdline parsing to one common place.

This patch also refactors our settings system
also make it more linear and remove abstraction
levels that are not needed and only cause confusion.
There is a lot more to be done in preparation for
dvc lib and RFC overall, but this a first step
in that long journey.

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/__init__.py

@@ -0,0 +1 @@
+VERSION = '0.8.7'

dvc/cli.py

@@ -0,0 +1,271 @@
+from __future__ import print_function
+
+import os
+import sys
+import argparse
+import configparser
+from multiprocessing import cpu_count
+
+from dvc.command.init import CmdInit
+from dvc.command.remove import CmdRemove
+from dvc.command.run import CmdRun
+from dvc.command.repro import CmdRepro
+from dvc.command.data_sync import CmdDataSync
+from dvc.command.lock import CmdLock
+from dvc.command.gc import CmdGC
+from dvc.command.import_file import CmdImportFile
+from dvc.command.target import CmdTarget
+from dvc.command.test import CmdTest
+from dvc.config import Config
+from dvc import VERSION
+
+def parse_args(argv=None):
+    # Common args
+    parent_parser = argparse.ArgumentParser(add_help=False)
+    parent_parser.add_argument('-q',
+                        '--quiet',
+                        action='store_true',
+                        default=False,
+                        help='Be quiet.')
+    parent_parser.add_argument('-v',
+                        '--verbose',
+                        action='store_true',
+                        default=False,
+                        help='Be verbose.')
+    parent_parser.add_argument('-G',
+                        '--no-git-actions',
+                        action='store_true',
+                        default=False,
+                        help='Skip all git actions including reproducibility check and commits.')
+
+    desc = 'Data Version Control'
+    parser = argparse.ArgumentParser(description=desc, parents=[parent_parser],
+             formatter_class=argparse.RawTextHelpFormatter)
+
+    parser.add_argument('-V',
+                        '--version',
+                        action='version',
+                        version='%(prog)s ' + VERSION)
+
+    # Sub commands
+    subparsers = parser.add_subparsers(
+                        dest='cmd',
+                        help='Use dvc CMD --help for command-specific help')
+
+    # Init
+    init_parser = subparsers.add_parser(
+                        'init',
+                        parents=[parent_parser],
+                        help='Initialize dvc over a directory (should already be a git dir).')
+    init_parser.add_argument(
+                        '--data-dir',
+                        default='data',
+                        help='Data directory.')
+    init_parser.add_argument(
+                        '--cache-dir',
+                        default='.cache',
+                        help='Cache directory.')
+    init_parser.add_argument(
+                        '--state-dir',
+                        default='.state',
+                        help='State directory.')
+    init_parser.add_argument(
+                        '--target-file',
+                        default=Config.TARGET_FILE_DEFAULT,
+                        help='Target file.')
+    init_parser.set_defaults(func=CmdInit)
+
+    # Run
+    run_parser = subparsers.add_parser(
+                        'run',
+                        parents=[parent_parser],
+                        help='Run command')
+    run_parser.add_argument(
+                        '--stdout',
+                        help='Output std output to a file.')
+    run_parser.add_argument(
+                        '--stderr',
+                        help='Output std error to a file.')
+    run_parser.add_argument('-i',
+                        '--input',
+                        action='append',
+                        help='Declare input data items for reproducible cmd.')
+    run_parser.add_argument('-o',
+                        '--output',
+                        action='append',
+                        help='Declare output data items for reproducible cmd.')
+    run_parser.add_argument('-c',
+                        '--code',
+                        action='append',
+                        help='Code dependencies which produce the output.')
+    run_parser.add_argument(
+                        '--shell',
+                        action='store_true',
+                        default=False,
+                        help='Shell command')
+    run_parser.add_argument('-l',
+                        '--lock',
+                        action='store_true',
+                        default=False,
+                        help='Lock data item - disable reproduction.')
+    run_parser.add_argument(
+                        'command',
+                        nargs=argparse.REMAINDER,
+                        help='Command to execute')
+    run_parser.set_defaults(func=CmdRun)
+
+    # Sync
+    sync_parser = subparsers.add_parser(
+                        'sync',
+                        parents=[parent_parser],
+                        help='Synchronize data file with cloud (cloud settings already setup.')
+    sync_parser.add_argument(
+                        'targets',
+                        metavar='',
+                        nargs='*',
+                        help='File or directory to sync.')
+    sync_parser.add_argument('-j',
+                        '--jobs',
+                        type=int,
+                        default=cpu_count(),
+                        help='Number of jobs to run simultaneously.')
+    sync_parser.set_defaults(func=CmdDataSync)
+
+    # Repro
+    repro_parser = subparsers.add_parser(
+                        'repro',
+                        parents=[parent_parser],
+                        help='Reproduce data')
+    repro_parser.add_argument(
+                        'target',
+                        metavar='',
+                        nargs='*',
+                        help='Data items to reproduce.')
+    repro_parser.add_argument('-f',
+                        '--force',
+                        action='store_true',
+                        default=False,
+                        help='Reproduce even if dependencies were not changed.')
+    repro_parser.add_argument('-s',
+                        '--single-item',
+                        action='store_true',
+                        default=False,
+                        help='Reproduce only single data item without recursive dependencies check.')
+    repro_parser.set_defaults(func=CmdRun)
+
+    # Remove
+    remove_parser = subparsers.add_parser(
+                        'remove',
+                        parents=[parent_parser],
+                        help='Remove data item from data directory.')
+    remove_parser.add_argument('target',
+                        metavar='',
+                        nargs='*',
+                        help='Target to remove - file or directory.')
+    remove_parser.add_argument('-l',
+                        '--keep-in-cloud',
+                        action='store_true',
+                        default=False,
+                        help='Do not remove data from cloud.')
+    remove_parser.add_argument('-r',
+                        '--recursive',
+                        action='store_true',
+                        help='Remove directory recursively.')
+    remove_parser.add_argument('-c',
+                        '--keep-in-cache',
+                        action='store_true',
+                        default=False,
+                        help='Do not remove data from cache.')
+    remove_parser.set_defaults(func=CmdRemove)
+
+    # Import
+    import_parser = subparsers.add_parser(
+                        'import',
+                        parents=[parent_parser],
+                        help='Import file to data directory.')
+    import_parser.add_argument(
+                        'input',
+                        nargs='+',
+                        help='Input file/files.')
+    import_parser.add_argument(
+                        'output',
+                        help='Output file/directory.')
+    import_parser.add_argument('-l',
+                        '--lock',
+                        action='store_true',
+                        default=False,
+                        help='Lock data item - disable reproduction.')
+    import_parser.add_argument('-j',
+                        '--jobs',
+                        type=int,
+                        default=cpu_count(),
+                        help='Number of jobs to run simultaneously.')
+    import_parser.set_defaults(func=CmdImportFile)
+
+    # Lock
+    lock_parser = subparsers.add_parser(
+                        'lock',
+                        parents=[parent_parser],
+                        help='Lock')
+    lock_parser.add_argument('-l',
+                        '--lock',
+                        action='store_true',
+                        default=False,
+                        help='Lock data item - disable reproduction.')
+    lock_parser.add_argument('-u',
+                        '--unlock',
+                        action='store_true',
+                        default=False,
+                        help='Unlock data item - enable reproduction.')
+    lock_parser.add_argument(
+                        'files',
+                        metavar='',
+                        nargs='*',
+                        help='Data items to lock or unlock.')
+    lock_parser.set_defaults(func=CmdLock)
+
+    # Garbage collector
+    gc_parser = subparsers.add_parser(
+                        'gc',
+                        parents=[parent_parser],
+                        help='Collect garbage')
+    gc_parser.add_argument('target',
+                        metavar='',
+                        nargs='*',
+                        help='Target to remove - file or directory.')
+    gc_parser.add_argument('-l',
+                        '--keep-in-cloud',
+                        action='store_true',
+                        default=False,
+                        help='Do not remove data from cloud.')
+    gc_parser.add_argument('-r',
+                        '--recursive',
+                        action='store_true',
+                        help='Remove directory recursively.')
+    gc_parser.add_argument('-c',
+                        '--keep-in-cache',
+                        action='store_false',
+                        default=False,
+                        help='Do not remove data from cache.')
+    gc_parser.set_defaults(func=CmdGC)
+
+    # Target
+    target_parser = subparsers.add_parser(
+                        'target',
+                        parents=[parent_parser],
+                        help='Set default target')
+    target_parser.add_argument('target_file',
+                        metavar='',
+                        nargs='?',
+                        help='Target data item.')
+    target_parser.add_argument('-u',
+                        '--unset',
+                        action='store_true',
+                        default=False,
+                        help='Reset target.')
+    target_parser.set_defaults(func=CmdTarget)
+
+    if isinstance(argv, str):
+        argv = argv.split()
+
+    return parser.parse_args(argv)

dvc/command/base.py

@@ -40,29 +40,24 @@ class CmdBase(object):
     def __init__(self, settings):
         self._settings = settings
 
-        parser = argparse.ArgumentParser()
-        self.define_common_args(parser)
-        self.define_args(parser)
-
-        self._parsed_args, self._command_args = parser.parse_known_args(args=self.args)
-
-        self.process_common_args()
+        if settings._parsed_args.quiet and not settings._parsed_args.verbose:
+            Logger.be_quiet()
+        elif not settings._parsed_args.quiet and settings._parsed_args.verbose:
+            Logger.be_verbose()
 
     @property
     def settings(self):
         return self._settings
 
+    #NOTE: this name is really confusing. It should really be called "command" or smth,
+    # because it is only used for "command" argument from CmdRun.
     @property
     def args(self):
         return self._settings.args
 
     @property
     def parsed_args(self):
-        return self._parsed_args
-
-    @property
-    def command_args(self):
-        return self._command_args
+        return self._settings._parsed_args
 
     @property
     def config(self):
@@ -76,34 +71,6 @@ class CmdBase(object):
     def git(self):
         return self._settings.git
 
-    def define_args(self, parser):
-        pass
-
-    def set_no_git_actions(self, parser):
-        parser.add_argument('--no-git-actions', '-G', action='store_true', default=False,
-                            help='Skip all git actions including reproducibility check and commits.')
-
-    def set_lock_action(self, parser):
-        parser.add_argument('--lock', '-l', action='store_true', default=False,
-                            help='Lock data item - disable reproduction. ' +
-                                 'It can be enabled by `dvc lock` command or by forcing reproduction.')
-
-    def define_common_args(self, parser):
-        parser.add_argument('--quiet', '-q', action='store_true', default=False, help='Be quiet.')
-        parser.add_argument('--verbose', '-v', action='store_true', default=False, help='Be verbose.')
-        parser.add_argument('--jobs', '-j', type=int, default=cpu_count(),
-                            help='Number of jobs to run simultaneously.')
-
-    @staticmethod
-    def set_reset_flag(parser, short_name, long_name, message):
-        parser.add_argument(short_name, long_name, action='store_true', default=False, help=message)
-
-    def process_common_args(self):
-        if self._parsed_args.quiet and not self._parsed_args.verbose:
-            Logger.be_quiet()
-        elif not self._parsed_args.quiet and self._parsed_args.verbose:
-            Logger.be_verbose()
-
     @property
     def no_git_actions(self):
         return self.parsed_args.no_git_actions
@@ -132,24 +99,5 @@ class CmdBase(object):
     def not_committed_changes_warning():
         Logger.warn('changes were not committed to git')
 
-    def add_string_arg(self, parser, name, message, default = None,
-                       conf_section=None, conf_name=None):
-        if conf_section and conf_name:
-            section = self.config[conf_section]
-            if not section:
-                raise ConfigError("")
-            default_value = section.get(conf_section, default)
-        else:
-            default_value = default
-
-        parser.add_argument(name,
-                            metavar='',
-                            default=default_value,
-                            help=message)
-
     def run(self):
         pass
-
-    @staticmethod
-    def warning_dvc_is_busy():
-        Logger.warn('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.')

dvc/command/data_sync.py

@@ -3,7 +3,6 @@ from multiprocessing.pool import ThreadPool
 
 from dvc.command.base import CmdBase, DvcLock
 from dvc.exceptions import DvcException
-from dvc.runtime import Runtime
 from dvc.system import System
 from dvc.data_cloud import DataCloud
 from dvc.utils import map_progress
@@ -18,12 +17,6 @@ class CmdDataSync(CmdBase):
     def __init__(self, settings):
         super(CmdDataSync, self).__init__(settings)
 
-    def define_args(self, parser):
-        parser.add_argument('targets',
-                            metavar='',
-                            help='File or directory to sync.',
-                            nargs='*')
-
     def run(self):
         with DvcLock(self.is_locker, self.git):
             cloud = DataCloud(self.settings)
@@ -44,6 +37,3 @@ class CmdDataSync(CmdBase):
 
             map_progress(cloud.sync, targets, self.parsed_args.jobs)
         pass
-
-if __name__ == '__main__':
-    Runtime.run(CmdDataSync)

dvc/command/gc.py

@@ -3,20 +3,12 @@ import os
 from dvc.command.traverse import Traverse
 from dvc.logger import Logger
 from dvc.path.data_item import DataItemError
-from dvc.runtime import Runtime
 
 
 class CmdGC(Traverse):
     def __init__(self, settings):
         super(CmdGC, self).__init__(settings, "garbage collect", do_not_start_from_root=False)
 
-    def define_args(self, parser):
-        super(CmdGC, self).define_args(parser)
-        parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively.')
-        parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False,
-                            help='Do not remove data from cache.')
-        pass
-
     def process_file(self, target):
         Logger.debug(u'[Cmd-GC] GC file {}.'.format(target))
 
@@ -60,7 +52,3 @@ class CmdGC(Traverse):
     @staticmethod
     def not_committed_changes_warning():
         pass
-
-
-if __name__ == '__main__':
-    Runtime.run(CmdGC)

dvc/command/import_file.py

@@ -6,7 +6,6 @@ from dvc.command.base import CmdBase, DvcLock
 from dvc.data_cloud import sizeof_fmt
 from dvc.logger import Logger
 from dvc.exceptions import DvcException
-from dvc.runtime import Runtime
 from dvc.state_file import StateFile
 from dvc.system import System
 from dvc.progress import progress
@@ -21,16 +20,6 @@ class CmdImportFile(CmdBase):
     def __init__(self, settings):
         super(CmdImportFile, self).__init__(settings)
 
-    def define_args(self, parser):
-        self.set_no_git_actions(parser)
-        self.set_lock_action(parser)
-
-        parser.add_argument('input',
-                            help='Input file/files.',
-                            nargs='+')
-        parser.add_argument('output',
-                            help='Output file/directory.')
-
     def run(self):
         targets = []
         with DvcLock(self.is_locker, self.git):
@@ -221,7 +210,3 @@ class CmdImportFile(CmdBase):
     @staticmethod
     def is_url(url):
         return CmdImportFile.URL_REGEX.match(url) is not None
-
-
-if __name__ == '__main__':
-    Runtime.run(CmdImportFile)

dvc/command/init.py

@@ -5,7 +5,6 @@ from dvc.command.base import CmdBase
 from dvc.logger import Logger
 from dvc.config import Config
 from dvc.exceptions import DvcException
-from dvc.runtime import Runtime
 from dvc.state_file import StateFile
 from dvc.system import System
 
@@ -62,15 +61,6 @@ ProjectName =
     def __init__(self, settings):
         super(CmdInit, self).__init__(settings)
 
-    def define_args(self, parser):
-        self.set_no_git_actions(parser)
-
-        self.add_string_arg(parser, '--data-dir',  'Data directory.', 'data')
-        self.add_string_arg(parser, '--cache-dir', 'Cache directory.', '.cache')
-        self.add_string_arg(parser, '--state-dir', 'State directory.', '.state')
-        self.add_string_arg(parser, '--target-file', 'Target file.', Config.TARGET_FILE_DEFAULT)
-        pass
-
     def get_not_existing_path(self, dir):
         path = Path(os.path.join(self.git.git_dir, dir))
         if path.exists():
@@ -162,6 +152,3 @@ ProjectName =
             fd.write('\n{}'.format(os.path.basename(self.git.lock_file)))
 
         Logger.info('Directory {} was added to .gitignore file'.format(cache_dir_name))
-
-if __name__ == '__main__':
-    Runtime.run(CmdInit, False)

dvc/command/lock.py

@@ -1,6 +1,5 @@
 from dvc.command.base import CmdBase, DvcLock
 from dvc.logger import Logger
-from dvc.runtime import Runtime
 from dvc.state_file import StateFile
 
 
@@ -8,12 +7,6 @@ class CmdLock(CmdBase):
     def __init__(self, settings):
         super(CmdLock, self).__init__(settings)
 
-    def define_args(self, parser):
-        self.set_no_git_actions(parser)
-        CmdLock.set_reset_flag(parser, '-u', '--unlock', 'Unlock data item - enable reproduction.')
-        parser.add_argument('files', metavar='', help='Data items to lock or unlock.', nargs='*')
-        pass
-
     def run(self):
         with DvcLock(self.is_locker, self.git):
             return self.lock_files(self.parsed_args.files, not self.parsed_args.unlock)
@@ -47,7 +40,3 @@ class CmdLock(CmdBase):
             self.commit_if_needed('DVC lock: {}'.format(' '.join(self.args)))
 
         return 0
-
-
-if __name__ == '__main__':
-    Runtime.run(CmdLock, False)

dvc/command/remove.py

@@ -3,20 +3,12 @@ import os
 from dvc.command.traverse import Traverse
 from dvc.logger import Logger
 from dvc.path.data_item import DataItemError
-from dvc.runtime import Runtime
 
 
 class CmdRemove(Traverse):
     def __init__(self, settings):
         super(CmdRemove, self).__init__(settings, "remove")
 
-    def define_args(self, parser):
-        super(CmdRemove, self).define_args(parser)
-        parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively.')
-        parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False,
-                            help='Do not remove data from cache.')
-        pass
-
     def process_file(self, target):
         Logger.debug(u'[Cmd-Remove] Remove file {}.'.format(target))
 
@@ -83,7 +75,3 @@ class CmdRemove(Traverse):
 
     def remove_all(self):
         return self._traverse_all()
-
-
-if __name__ == '__main__':
-    Runtime.run(CmdRemove)

dvc/command/repro.py

@@ -7,7 +7,6 @@ from dvc.command.run import CmdRun
 from dvc.logger import Logger
 from dvc.exceptions import DvcException
 from dvc.path.data_item import NotInDataDirError
-from dvc.runtime import Runtime
 from dvc.state_file import StateFile
 from dvc.system import System
 
@@ -28,16 +27,6 @@ class CmdRepro(CmdRun):
     def code_dependencies(self):
         return self._code
 
-    def define_args(self, parser):
-        self.set_no_git_actions(parser)
-
-        parser.add_argument('target', metavar='', help='Data items to reproduce.', nargs='*')
-        parser.add_argument('-f', '--force', action='store_true', default=False,
-                            help='Reproduce even if dependencies were not changed.')
-        parser.add_argument('-s', '--single-item', action='store_true', default=False,
-                            help='Reproduce only single data item without recursive dependencies check.')
-        pass
-
     @property
     def lock(self):
         return True
@@ -209,6 +198,10 @@ class ReproChange(object):
                                             self.state.stdout,
                                             self.state.stderr,
                                             self.state.shell,
+                                            [],
+                                            [],
+                                            [],
+                                            True,
                                             check_if_ready=False) != 0:
                 raise ReproError('Run command reproduction failed')
             return True
@@ -317,7 +310,3 @@ class ReproChange(object):
             dependency_data_items.append(data_item)
 
         return dependency_data_items
-
-
-if __name__ == '__main__':
-    Runtime.run(CmdRepro)

dvc/command/run.py

@@ -1,4 +1,5 @@
 import os
+import sys
 
 from dvc.command.base import CmdBase, DvcLock
 from dvc.exceptions import DvcException
@@ -6,7 +7,6 @@ from dvc.git_wrapper import GitWrapper
 from dvc.logger import Logger
 from dvc.path.data_item import NotInDataDirError, NotInGitDirError
 from dvc.repository_change import RepositoryChange
-from dvc.runtime import Runtime
 from dvc.state_file import StateFile
 from dvc.utils import cached_property
 
@@ -20,21 +20,6 @@ class CmdRun(CmdBase):
     def __init__(self, settings):
         super(CmdRun, self).__init__(settings)
 
-    def define_args(self, parser):
-        self.set_no_git_actions(parser)
-        self.set_lock_action(parser)
-
-        parser.add_argument('--stdout', help='Output std output to a file.')
-        parser.add_argument('--stderr', help='Output std error to a file.')
-        parser.add_argument('--input', '-i', action='append',
-                            help='Declare input data items for reproducible cmd.')
-        parser.add_argument('--output', '-o', action='append',
-                            help='Declare output data items for reproducible cmd.')
-        parser.add_argument('--code', '-c', action='append',
-                            help='Code dependencies which produce the output.')
-        parser.add_argument('--shell', help='Shell command', action='store_true', default=False)
-        pass
-
     @property
     def lock(self):
         return self.parsed_args.lock
@@ -53,8 +38,8 @@ class CmdRun(CmdBase):
 
     def run(self):
         with DvcLock(self.is_locker, self.git):
-            data_items_from_args, not_data_items_from_args = self.argv_files_by_type(self.command_args)
-            return self.run_and_commit_if_needed(self.command_args,
+            data_items_from_args, not_data_items_from_args = self.argv_files_by_type(self.parsed_args.command)
+            return self.run_and_commit_if_needed(self.parsed_args.command,
                                                  data_items_from_args,
                                                  not_data_items_from_args,
                                                  self.parsed_args.stdout,
@@ -63,7 +48,12 @@ class CmdRun(CmdBase):
         pass
 
     def run_and_commit_if_needed(self, command_args, data_items_from_args, not_data_items_from_args,
-                                 stdout, stderr, shell, check_if_ready=True):
+                                 stdout, stderr, shell, 
+                                 output_data_items=None,
+                                 input_data_items=None,
+                                 code_dependencies=None,
+                                 lock=None,
+                                 check_if_ready=True):
         if check_if_ready and not self.no_git_actions and not self.git.is_ready_to_go():
             return 1
 
@@ -72,12 +62,34 @@ class CmdRun(CmdBase):
                          not_data_items_from_args,
                          stdout,
                          stderr,
-                         shell)
+                         shell,
+                         output_data_items,
+                         input_data_items,
+                         code_dependencies,
+                         lock)
 
-        return self.commit_if_needed('DVC run: {}'.format(' '.join(self.args)))
+        return self.commit_if_needed('DVC run: {}'.format(' '.join(command_args)))
 
     def run_command(self, cmd_args, data_items_from_args, not_data_items_from_args,
-                    stdout=None, stderr=None, shell=False):
+                    stdout=None, stderr=None, shell=False,
+                    output_data_items=None,
+                    input_data_items=None,
+                    code_dependencies=None,
+                    lock=None):
+
+        # Repro sets these from state file
+        if output_data_items == None:
+            output_data_items = self.declaration_output_data_items
+
+        if input_data_items == None:
+            input_data_items = self.declaration_input_data_items
+
+        if code_dependencies == None:
+            code_dependencies = self.code_dependencies
+
+        if lock == None:
+            lock = self.lock
+
         Logger.debug(u'Run command with args: {}. Data items from args: {}. stdout={}, stderr={}, shell={}'.format(
                      ' '.join(cmd_args),
                      ', '.join([x.data.dvc for x in data_items_from_args]),
@@ -91,13 +103,13 @@ class CmdRun(CmdBase):
             self.remove_new_files(repo_change)
             raise RunError('Errors occurred.')
 
-        output_set = set(self.declaration_output_data_items + repo_change.changed_data_items)
+        output_set = set(output_data_items + repo_change.changed_data_items)
         output_files_dvc = [x.data.dvc for x in output_set]
 
-        input_set = set(data_items_from_args + self.declaration_input_data_items) - output_set
+        input_set = set(data_items_from_args + input_data_items) - output_set
         input_files_dvc = [x.data.dvc for x in input_set]
 
-        code_dependencies_dvc = self.git.abs_paths_to_dvc(self.code_dependencies + not_data_items_from_args)
+        code_dependencies_dvc = self.git.abs_paths_to_dvc(code_dependencies + not_data_items_from_args)
 
         result = []
         for data_item in repo_change.changed_data_items:
@@ -114,7 +126,7 @@ class CmdRun(CmdBase):
                                    output_files_dvc,
                                    code_dependencies_dvc,
                                    argv=cmd_args,
-                                   lock=self.lock,
+                                   lock=lock,
                                    stdout=self._stdout_to_dvc(stdout),
                                    stderr=self._stdout_to_dvc(stderr),
                                    shell=shell)
@@ -176,7 +188,3 @@ class CmdRun(CmdBase):
                 param_text, ', '.join(external))
             )
         return data_items
-
-
-if __name__ == '__main__':
-    Runtime.run(CmdRun)

dvc/command/target.py

@@ -2,22 +2,15 @@ import os
 
 from dvc.command.base import CmdBase, DvcLock
 from dvc.logger import Logger
-from dvc.runtime import Runtime
 
 
 class CmdTarget(CmdBase):
     def __init__(self, settings):
         super(CmdTarget, self).__init__(settings)
 
-    def define_args(self, parser):
-        self.set_no_git_actions(parser)
-        CmdTarget.set_reset_flag(parser, '-u', '--unset', 'Reset target.')
-        parser.add_argument('target', metavar='', nargs='?', help='Target data item.')
-        pass
-
     def run(self):
         with DvcLock(self.is_locker, self.git):
-            target = self.parsed_args.target
+            target = self.parsed_args.target_file
             unset = self.parsed_args.unset
 
             if target and unset:
@@ -61,7 +54,3 @@ class CmdTarget(CmdBase):
         open(target_conf_file_path, 'a').close()
 
         return self.commit_if_needed('DVC target unset')
-
-
-if __name__ == '__main__':
-    Runtime.run(CmdTarget, False)

dvc/command/test.py

@@ -5,14 +5,10 @@ from pathlib import Path
 
 from google.cloud import storage
 
+import dvc
 from dvc.command.base import CmdBase
 from dvc.logger import Logger
 from dvc.config import Config
-from dvc.runtime import Runtime
-
-from dvc.settings import SettingsError
-
-
 
 class CmdTest(CmdBase):
     def __init__(self, settings):
@@ -33,7 +29,7 @@ class CmdTest(CmdBase):
 
             if self.settings._config.gc_project_name == '':
                 Logger.error('Please specify the google cloud project name in dvc.conf')
-                raise SettingsError('must specify GC ProjectName')
+                raise dvc.settings.SettingsError('must specify GC ProjectName')
 
             try:
                 client = storage.Client(project=self.settings._config.gc_project_name)
@@ -52,7 +48,3 @@ class CmdTest(CmdBase):
                 sys.exit(-1)
             Logger.info('bucket %s visible: google cloud seems to be configured correctly' % sb)
             sys.exit(0)
-
-
-if __name__ == '__main__':
-    Runtime.run(CmdTest, False)

dvc/command/traverse.py

@@ -18,15 +18,6 @@ class Traverse(CmdBase):
         self.cloud = DataCloud(self.settings)
         self._do_not_start_from_root = do_not_start_from_root
 
-    def define_args(self, parser):
-        self.set_no_git_actions(parser)
-
-        parser.add_argument('target', metavar='', help='Target to remove - file or directory.', nargs='*')
-        # parser.add_argument('-r', '--recursive', action='store_true', help='CmdGarbage collect directory recursively.')
-        parser.add_argument('-l', '--keep-in-cloud', action='store_true', default=False,
-                            help='Do not remove data from cloud.')
-        pass
-
     def run(self):
         with DvcLock(self.is_locker, self.git):
             if not self._traverse_all():

dvc/config.py

@@ -46,7 +46,7 @@ class ConfigI(object):
 class Config(ConfigI):
     CONFIG = 'dvc.conf'
 
-    def __init__(self, conf_file, conf_pseudo_file=None):
+    def __init__(self, conf_file=CONFIG, conf_pseudo_file=None):
         """
         Params:
             conf_file (String): configuration file
@@ -59,7 +59,7 @@ class Config(ConfigI):
             self._config.readfp(conf_pseudo_file)
         else:
             if not os.path.isfile(conf_file):
-                raise ConfigError('Config file "{}" does not exist'.format(conf_file))
+                raise ConfigError('Config file "{}" does not exist {}'.format(conf_file, os.getcwd()))
             self._config.read(conf_file)
 
         level = self._config['Global']['LogLevel']

dvc/data_cloud.py

@@ -8,10 +8,10 @@ import tempfile
 from boto.s3.connection import S3Connection
 from google.cloud import storage as gc
 
+import dvc
 from dvc.logger import Logger
 from dvc.exceptions import DvcException
 from dvc.config import ConfigError
-from dvc.settings import Settings
 from dvc.progress import progress
 from dvc.utils import copyfile
 
@@ -382,7 +382,7 @@ class DataCloud(object):
     }
 
     def __init__(self, settings):
-        assert isinstance(settings, Settings) 
+        assert isinstance(settings, dvc.settings.Settings) 
 
         #To handle ConfigI case
         if not hasattr(settings.config, '_config'):

dvc/main.py

@@ -1,92 +1,11 @@
-from __future__ import print_function
-
-from dvc.command.import_file import CmdImportFile
-
 """
 main entry point / argument parsing for dvc
 """
 
 import sys
-
-from dvc.runtime import Runtime
-from dvc.command.init import CmdInit
-from dvc.command.remove import CmdRemove
-from dvc.command.run import CmdRun
-from dvc.command.repro import CmdRepro
-from dvc.command.data_sync import CmdDataSync
-from dvc.command.lock import CmdLock
-from dvc.command.gc import CmdGC
-from dvc.command.target import CmdTarget
-from dvc.command.test import CmdTest
-
-VERSION = '0.8.7'
-
-def print_usage():
-    usage = ('',
-            'usage: dvc [--version] [--help] command [<args>]',
-            '',
-            'These are common DVC commands:',
-            '',
-            'start a working area',
-            '    init           Initialize dvc over a directory (should already be a git dir).',
-            '    run            Run command.',
-            '    import         Import file to data directory.',
-            '    remove         Remove data item from data directory.',
-            '',
-            'synchronize data between remote and local',
-            '    data sync      Synchronize data file with cloud (cloud settings already setup).',
-)
-    print('\n'.join(usage))
+from dvc.settings import Settings
 
 def main():
-    cmds = ['--help', '--version', 'init', 'run', 'sync', 'repro', 'data', 'remove', 'import', 'lock', 'cloud', \
-            'cloud', 'cloud-run', 'cloud-instance-create', 'cloud-instance-remove', 'cloud-instance-describe', \
-            'test', 'test-aws', 'test-gcloud', 'test-cloud', 'gc', 'target']
-
-    if len(sys.argv) < 2 or sys.argv[1] not in cmds:
-        if len(sys.argv) >= 2:
-            print('Unimplemented or unrecognized command: ' + ' '.join(sys.argv[1:]))
-        print_usage()
-        sys.exit(-1)
-
-    cmd = sys.argv[1]
-
-    if cmd == '--help':
-        print_usage()
-    elif cmd == '--version':
-        print('dvc version {}'.format(VERSION))
-    elif cmd == 'init':
-        Runtime.run(CmdInit, parse_config=False)
-    elif cmd == 'run':
-        Runtime.run(CmdRun)
-    elif cmd == 'repro':
-        Runtime.run(CmdRepro)
-    elif cmd == 'sync':
-        Runtime.run(CmdDataSync)
-    elif cmd == 'import':
-        Runtime.run(CmdImportFile)
-    elif cmd == 'remove':
-        Runtime.run(CmdRemove)
-    elif cmd == 'lock':
-        Runtime.run(CmdLock)
-    elif cmd == 'gc':
-        Runtime.run(CmdGC)
-    elif cmd == 'target':
-        Runtime.run(CmdTarget)
-    elif cmd == 'cloud-run':
-        print('cloud-run unimplemented')
-    elif cmd == 'cloud-instance-create':
-        print('cloud-instance-create unimplemented')
-    elif cmd == 'clould-instance-remove':
-        print('cloud-instance-remove unimplemented')
-    elif cmd == 'cloud-instance-describe':
-        print('cloud-instance-describe unimplemented')
-
-    elif cmd == 'test-aws':
-        print('TODO: test aws credentials')
-    elif cmd == 'test-gcloud':
-        Runtime.run(CmdTest)
-    else:
-        print('Unimplemented or unrecognized command. ' + ' '.join(sys.argv[1]))
-        print_usage()
-        sys.exit(-1)
+    settings = Settings(sys.argv[1:])
+    instance = settings._parsed_args.func(settings)
+    sys.exit(instance.run())

dvc/progress.py.back

@@ -0,0 +1,90 @@
+from __future__ import print_function
+import sys
+import threading 
+from Queue import Queue
+
+# (2/100): very_long_nam... [#########     ] 60%
+
+class ProgressInfo(object):
+    name = ""
+    current = 0
+    total = None
+    finished = False
+
+class Progress(object):
+    _n_finished = 0
+    _n_total = 0
+    _lock = None
+    _queue = None
+
+    def __init__(self, total):
+        self._n_total = total
+        self._lock = threading.Lock()
+        self._queue = Queue(maxsize=0)
+
+    def clearln(self):
+        if sys.stdout.isatty():
+            print('\r\x1b[K', end='')
+
+    def writeln(self, line):
+        if sys.stdout.isatty():
+            self.clearln()
+            print(line, end='')
+            sys.stdout.flush()
+
+    def update_target(self, name, current, total):
+        info = ProgressInfo()
+        info.name = name
+        info.current = current
+        info.total = total
+
+        self._queue.put(info)
+
+    def finish_target(self, name):
+        info = ProgressInfo()
+        info.name = name
+        info.current = 100
+        info.total = 100
+        info.finished = True
+
+        self._queue.put(info)
+
+    def finish(self):
+        if sys.stdout.isatty():
+            print()
+
+    def _bar(self, info):
+        total_len = 80
+        bar_len = 40
+
+        if info.total == None:
+            progress = 0
+            percent = "?% "
+        else:
+            progress = (100 * info.current)/info.total
+            percent = str(progress) + "% "
+
+        num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
+
+        n_sh = (progress * bar_len)/100
+        n_sp = bar_len - n_sh
+        bar = "[" + '#'*n_sh + ' '*n_sp + "] "
+
+        name_len = len(num + bar + percent) + 1
+        name = info.name[:name_len] if len(info.name) > name_len else info.name
+
+        return num + bar + percent + name
+
+    def wait(self):
+        while self._n_finished != self._n_total:
+            while not self._queue.empty():
+                info = self._queue.get()
+                bar = self._bar(info)
+                with self._lock:
+                    if info.finished:
+                        self.clearln()
+                        print(bar)
+                        self._n_finished += 1
+                    else:
+                        self.writeln(bar)
+        self.finish()

dvc/runtime.py

@@ -1,44 +0,0 @@
-import os
-import sys
-
-from dvc.config import Config, ConfigI
-from dvc.exceptions import DvcException
-from dvc.git_wrapper import GitWrapper
-from dvc.logger import Logger
-from dvc.settings import Settings
-from dvc.system import System
-
-
-class Runtime(object):
-    CONFIG = 'dvc.conf'
-
-    @staticmethod
-    def conf_file_path(git_dir):
-        return System.realpath(os.path.join(git_dir, Runtime.CONFIG))
-
-    @staticmethod
-    def run(cmd_class, parse_config=True, args_start_loc=2):
-        """
-
-        Arguments:
-            args_start_loc (int): where the arguments this command should use start
-        """
-
-        try:
-            runtime_git = GitWrapper()
-
-            if parse_config:
-                runtime_config = Config(Runtime.conf_file_path(runtime_git.git_dir))
-            else:
-                runtime_config = ConfigI()
-
-            args = sys.argv[args_start_loc:]
-
-            # To make argparse print "usage: dvc cmd" instead of "usage: dvc"
-            sys.argv[0] = sys.argv[0] + " " + sys.argv[1]
-
-            instance = cmd_class(Settings(args, runtime_git, runtime_config))
-            sys.exit(instance.run())
-        except DvcException as e:
-            Logger.error(e)
-            sys.exit(1)

dvc/settings.py

@@ -1,6 +1,10 @@
+import os
+
+from dvc.git_wrapper import GitWrapperI, GitWrapper
+from dvc.config import ConfigI, Config
 from dvc.exceptions import DvcException
 from dvc.path.factory import PathFactory
-
+from dvc.cli import parse_args
 
 class SettingsError(DvcException):
     def __init__(self, msg):
@@ -8,18 +12,27 @@ class SettingsError(DvcException):
 
 
 class Settings(object):
-    def __init__(self, args, git, config):
-        self._args = args
+    def __init__(self, argv=None, git=None, config=None):
+        self._args = None
+        args = None
+
+        if argv != None and len(argv) != 0:
+            args = parse_args(argv)
+            self._args = argv[2:]
+
+        if git == None:
+            git = GitWrapper()
+
+        if config == None:
+            if args != None and args.cmd != 'init':
+                config = Config()
+            else:
+                config = ConfigI()
+
         self._git = git
         self._config = config
         self._path_factory = PathFactory(git, config)
-
-        # self._dvc_home = os.environ.get('DVC_HOME')
-        # if not self._dvc_home:
-        #     raise SettingsError('DVC_HOME environment variable is not defined')
-        # if not os.path.exists(self._dvc_home):
-        #     raise SettingsError("DVC_HOME directory doesn't exists")
-        pass
+        self._parsed_args = args
 
     @property
     def args(self):
@@ -28,6 +41,13 @@ class Settings(object):
     def set_args(self, args):
         self._args = args
 
+    @property
+    def parsed_args(self):
+        return self._parsed_args
+
+    def parse_args(self, args):
+        self._parsed_args = parse_args(args)
+
     @property
     def git(self):
         return self._git
@@ -42,4 +62,4 @@ class Settings(object):
 
     @property
     def path_factory(self):
-        return self._path_factory
+        return self._path_factory

setup.py

@@ -1,7 +1,7 @@
 import platform
 from setuptools import setup, find_packages
 from distutils.errors import DistutilsPlatformError
-from dvc.main import VERSION
+from dvc import VERSION
 
 install_requires = [
     'altgraph==0.13',

tests/basic_env.py

@@ -10,7 +10,6 @@ from dvc.settings import Settings
 from dvc.system import System
 from dvc.utils import rmtree
 
-
 class BasicEnvironment(TestCase):
     def init_environment(self,
                          test_dir=System.get_long_path(tempfile.mkdtemp()),

tests/test_cmd_gc.py

@@ -10,7 +10,6 @@ from dvc.settings import Settings
 from dvc.system import System
 from tests.basic_env import BasicEnvironment
 
-
 class TestCmdDataRemove(BasicEnvironment):
     def setUp(self):
         self._test_dir = os.path.join(os.path.sep, 'tmp', 'dvc_unit_test')
@@ -24,7 +23,7 @@ class TestCmdDataRemove(BasicEnvironment):
         self._git = GitWrapperI(git_dir=self._test_git_dir, commit=self._commit)
         self._config = ConfigI('data', 'cache', 'state')
         self.path_factory = PathFactory(self._git, self._config)
-        self.settings = Settings([], self._git, self._config)
+        self.settings = Settings('gc', self._git, self._config)
 
         self.dir1 = 'dir1'
 
@@ -53,11 +52,10 @@ class TestCmdDataRemove(BasicEnvironment):
         self.assertEqual(5, self.cache_file_nums('file2*'))
         self.assertEqual(5, self.cache_file_nums(os.path.join(self.dir1, 'file3*')))
 
+        self.settings.parse_args('gc --no-git-actions data')
         cmd = CmdGC(self.settings)
         os.chdir(self._test_git_dir)
 
-        cmd.parsed_args.no_git_actions = True
-        cmd.parsed_args.target = ['data']
         cmd.gc_all()
 
         self.assertEqual(1, self.cache_file_nums('file1*'))
@@ -72,10 +70,10 @@ class TestCmdDataRemove(BasicEnvironment):
         self.assertEqual(5, self.cache_file_nums('file1*'))
         self.assertEqual(5, self.cache_file_nums('file2*'))
 
+        self.settings.parse_args('gc --no-git-actions')
         cmd = CmdGC(self.settings)
         os.chdir(self._test_git_dir)
 
-        cmd.parsed_args.no_git_actions = True
         cmd.gc_all()
 
         self.assertEqual(5, self.cache_file_nums('file1*'))

tests/test_cmd_remove.py

@@ -3,14 +3,13 @@ import os
 from dvc.command.remove import CmdRemove
 from tests.basic_env import DirHierarchyEnvironment
 
-
 class TestCmdRemove(DirHierarchyEnvironment):
     def setUp(self):
         DirHierarchyEnvironment.init_environment(self)
 
     def test_file_by_file_removal(self):
+        self.settings.parse_args('remove --keep-in-cloud')
         cmd = CmdRemove(self.settings)
-        cmd.parsed_args.keep_in_cloud = True
 
         dir1_dvc_name = os.path.join('data', self.dir1)
         self.assertTrue(os.path.exists(dir1_dvc_name))
@@ -19,9 +18,8 @@ class TestCmdRemove(DirHierarchyEnvironment):
         self.assertFalse(os.path.exists(dir1_dvc_name))
 
     def test_remove_deep_dir(self):
+        self.settings.parse_args('remove --keep-in-cloud --recursive')
         cmd = CmdRemove(self.settings)
-        cmd.parsed_args.keep_in_cloud = True
-        cmd.parsed_args.recursive = True
 
         dir1_dvc_name = os.path.join('data', self.dir1)
         self.assertTrue(os.path.exists(dir1_dvc_name))
@@ -30,9 +28,8 @@ class TestCmdRemove(DirHierarchyEnvironment):
         self.assertFalse(os.path.exists(dir1_dvc_name))
 
     def test_not_recursive_removal(self):
+        self.settings.parse_args('remove --keep-in-cloud')
         cmd = CmdRemove(self.settings)
-        cmd.parsed_args.keep_in_cloud = True
-        cmd.parsed_args.recursive = False
 
         dir1_dvc_name = os.path.join('data', self.dir1)
         self.assertTrue(os.path.exists(dir1_dvc_name))
@@ -41,9 +38,8 @@ class TestCmdRemove(DirHierarchyEnvironment):
         pass
 
     def test_data_dir_removal(self):
+        self.settings.parse_args('remove --keep-in-cloud --recursive')
         cmd = CmdRemove(self.settings)
-        cmd.parsed_args.keep_in_cloud = True
-        cmd.parsed_args.recursive = True
 
         data_dir = 'data'
         self.assertTrue(os.path.exists(data_dir))
@@ -57,8 +53,8 @@ class TestRemoveDataItem(DirHierarchyEnvironment):
         DirHierarchyEnvironment.init_environment(self)
 
     def test_remove_data_instance(self):
+        self.settings.parse_args('remove --keep-in-cloud')
         cmd = CmdRemove(self.settings)
-        cmd.parsed_args.keep_in_cloud = True
 
         self.assertTrue(os.path.isfile(self.file1))
         self.assertTrue(os.path.isfile(self.cache1))
@@ -70,9 +66,8 @@ class TestRemoveDataItem(DirHierarchyEnvironment):
         self.assertFalse(os.path.exists(self.state1))
 
     def test_keep_cache(self):
+        self.settings.parse_args('remove --keep-in-cloud --keep-in-cache')
         cmd = CmdRemove(self.settings)
-        cmd.parsed_args.keep_in_cloud = True
-        cmd.parsed_args.keep_in_cache = True
 
         self.assertTrue(os.path.isfile(self.file1))
         self.assertTrue(os.path.isfile(self.cache1))
@@ -84,8 +79,8 @@ class TestRemoveDataItem(DirHierarchyEnvironment):
         self.assertFalse(os.path.exists(self.state1))
 
     def test_remove_data_instance_without_cache(self):
+        self.settings.parse_args('remove --keep-in-cloud')
         cmd = CmdRemove(self.settings)
-        cmd.parsed_args.keep_in_cloud = True
 
         self.assertTrue(os.path.isfile(self.file6))
         self.assertIsNone(self.cache6)
@@ -101,16 +96,11 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment):
         DirHierarchyEnvironment.init_environment(self)
 
     def test_end_to_end_with_no_error(self):
-        cmd = CmdRemove(self.settings)
-        cmd.parsed_args.keep_in_cloud = True
-
         dir11_full = os.path.join('data', self.dir11)
         dir2_full = os.path.join('data', self.dir2)
 
-        cmd.parsed_args.target = [dir11_full, self.file5]
-        cmd.parsed_args.recursive = True
-
-        cmd.parsed_args.no_git_actions = True
+        self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5))
+        cmd = CmdRemove(self.settings)
 
         self.assertTrue(os.path.exists(dir11_full))
         self.assertTrue(os.path.exists(self.file5))
@@ -125,16 +115,11 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment):
         self.assertTrue(os.path.exists(self.file1))
 
     def test_run(self):
-        cmd = CmdRemove(self.settings)
-        cmd.parsed_args.keep_in_cloud = True
-
         dir11_full = os.path.join('data', self.dir11)
         dir2_full = os.path.join('data', self.dir2)
 
-        cmd.parsed_args.target = [dir11_full, self.file5]
-        cmd.parsed_args.recursive = True
-
-        cmd.parsed_args.no_git_actions = True
+        self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5))
+        cmd = CmdRemove(self.settings)
 
         self.assertTrue(os.path.exists(dir11_full))
         self.assertTrue(os.path.exists(self.file5))
@@ -154,16 +139,11 @@ class TestRemoveEndToEndWithAnError(DirHierarchyEnvironment):
         DirHierarchyEnvironment.init_environment(self)
 
     def test(self):
-        cmd = CmdRemove(self.settings)
-        cmd.parsed_args.keep_in_cloud = True
-
         dir11_full = os.path.join(self._config.data_dir, self.dir11)
         dir2_full = os.path.join(self._config.data_dir, self.dir2)
 
-        cmd.parsed_args.target = [dir11_full, self.file5]
-        cmd.parsed_args.recursive = True
-
-        cmd.parsed_args.no_git_actions = True
+        self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5))
+        cmd = CmdRemove(self.settings)
 
         self.assertTrue(os.path.exists(dir11_full))
         self.assertTrue(os.path.exists(self.file5))

tests/test_cmd_run.py

@@ -31,7 +31,7 @@ class RunBasicTest(TestCase):
         self.config = ConfigI('data', 'cache', 'state')
         self.path_factory = PathFactory(self.git, self.config)
 
-        self.settings = Settings([], self.git, self.config)
+        self.settings = Settings(['run'], self.git, self.config)
         pass
 
     def init_git_repo(self):
@@ -65,13 +65,12 @@ class RunTwoFilesBase(RunBasicTest):
     def setUp(self):
         super(RunTwoFilesBase, self).setUp()
 
+        self.input_param_file = os.path.join('data', 'extra_input')
+        self.extra_output_file = os.path.join('data', 'extra_output')
+
+        self.settings.parse_args('run --input {} --output {}'.format(self.input_param_file, self.extra_output_file))
         self.settings._args = []
         cmd_run = CmdRun(self.settings)
-        self.input_param_file = os.path.join('data', 'extra_input')
-        cmd_run.parsed_args.input = [self.input_param_file]
-
-        self.extra_output_file = os.path.join('data', 'extra_output')
-        cmd_run.parsed_args.output = [self.extra_output_file]
 
         self.file_name1 = os.path.join('data', 'file1')
         self.file_name2 = os.path.join('data', 'file2')

tests/test_cmd_target.py

@@ -22,7 +22,7 @@ class TestCmdTarget(BasicEnvironment):
         self._git = GitWrapperI(git_dir=self._test_git_dir, commit=self._commit)
         self._config = ConfigI('data', 'cache', 'state', '.target')
         self.path_factory = PathFactory(self._git, self._config)
-        self.settings = Settings([], self._git, self._config)
+        self.settings = Settings(['target'], self._git, self._config)
 
         os.chdir(self._test_git_dir)
 
@@ -45,53 +45,49 @@ class TestCmdTarget(BasicEnvironment):
         self.assertFalse(os.path.exists('.target'))
 
     def test_single_file(self):
+        self.settings.parse_args('target {}'.format(self.file1_data))
         cmd = CmdTarget(self.settings)
-        cmd.parsed_args.target = self.file1_data
-        cmd.parsed_args.unset = False
 
         self.assertEqual(cmd.run(), 0)
         self.assertEqual(open('.target').read(), self.file1_data)
 
     def test_multiple_files(self):
+        self.settings.parse_args('target {}'.format(self.file1_data))
         cmd = CmdTarget(self.settings)
-        cmd.parsed_args.target = self.file1_data
-        cmd.parsed_args.unset = False
         cmd.run()
 
         # Another target
-        cmd.parsed_args.target = self.file2_data
+        self.settings.parse_args('target {}'.format(self.file2_data))
+        cmd = CmdTarget(self.settings)
         self.assertEqual(cmd.run(), 0)
         self.assertEqual(open('.target').read(), self.file2_data)
 
         # Unset target
-        cmd.parsed_args.target = ''
-        cmd.parsed_args.unset = True
+        self.settings.parse_args('target --unset')
+        cmd = CmdTarget(self.settings)
         self.assertEqual(cmd.run(), 0)
         self.assertEqual(open('.target').read(), '')
 
     def test_initial_unset(self):
+        self.settings.parse_args('target --unset')
         cmd = CmdTarget(self.settings)
-        cmd.parsed_args.target = ''
-        cmd.parsed_args.unset = True
         self.assertEqual(cmd.run(), 1)
         self.assertFalse(os.path.exists('.target'))
 
     def test_unset_existing_target(self):
+        self.settings.parse_args('target {}'.format(self.file1_data))
         cmd = CmdTarget(self.settings)
-        cmd.parsed_args.target = self.file1_data
-        cmd.parsed_args.unset = False
         self.assertEqual(cmd.run(), 0)
         self.assertEqual(open('.target').read(), self.file1_data)
 
-        cmd.parsed_args.target = ''
-        cmd.parsed_args.unset = True
+        self.settings.parse_args('target --unset')
+        cmd = CmdTarget(self.settings)
         self.assertEqual(cmd.run(), 0)
         self.assertEqual(open('.target').read(), '')
 
     def test_the_same(self):
+        self.settings.parse_args('target {}'.format(self.file1_data))
         cmd = CmdTarget(self.settings)
-        cmd.parsed_args.target = self.file1_data
-        cmd.parsed_args.unset = False
         self.assertEqual(cmd.run(), 0)
         self.assertEqual(open('.target').read(), self.file1_data)
 
@@ -99,13 +95,11 @@ class TestCmdTarget(BasicEnvironment):
         self.assertEqual(open('.target').read(), self.file1_data)
 
     def test_args_conflict(self):
+        self.settings.parse_args('target {} --unset'.format(self.file1_data))
         cmd = CmdTarget(self.settings)
-        cmd.parsed_args.target = self.file1_data
-        cmd.parsed_args.unset = True
         self.assertEqual(cmd.run(), 1)
 
     def test_no_args(self):
+        self.settings.parse_args('target --unset')
         cmd = CmdTarget(self.settings)
-        cmd.parsed_args.target = ''
-        cmd.parsed_args.unset = False
-        self.assertEqual(cmd.run(), 1)
+        self.assertEqual(cmd.run(), 1)

tests/test_import.py

@@ -18,7 +18,7 @@ class TestCmdRemove(DirHierarchyEnvironment):
 
             dir = os.path.join('data', self.dir1)
             settings = copy(self.settings)
-            settings.set_args([temp.name, dir])
+            settings.parse_args('import {} {}'.format(temp.name, dir))
             cmd = CmdImportFile(settings)
             cmd.run()
 
@@ -44,7 +44,7 @@ class TestCmdRemove(DirHierarchyEnvironment):
             target_file_name = 'targ.txt'
             target_file = os.path.join('data', self.dir1, target_file_name)
             settings = copy(self.settings)
-            settings.set_args([temp.name, target_file])
+            settings.parse_args('import {} {}'.format(temp.name, target_file))
             cmd = CmdImportFile(settings)
             cmd.run()
 
@@ -60,7 +60,7 @@ class TestCmdRemove(DirHierarchyEnvironment):
 
             dir = os.path.join('data', self.dir11)
             settings = copy(self.settings)
-            settings.set_args([temp.name, dir])
+            settings.parse_args('import {} {}'.format(temp.name, dir))
             cmd = CmdImportFile(settings)
             cmd.run()
 
@@ -78,7 +78,7 @@ class TestCmdRemove(DirHierarchyEnvironment):
 
             dir = os.path.join('data', self.dir1, 'not_existing_dir/')
             settings = copy(self.settings)
-            settings.set_args([temp.name, dir])
+            settings.parse_args('import {} {}'.format(temp.name, dir))
             cmd = CmdImportFile(settings)
 
             with self.assertRaises(ImportFileError):
@@ -103,7 +103,7 @@ class TestCmdRemove(DirHierarchyEnvironment):
 
             dir = os.path.join('data', self.dir11)
             settings = copy(self.settings)
-            settings.set_args([temp1.name, temp2.name, dir])
+            settings.parse_args('import {} {} {}'.format(temp1.name, temp2.name, dir))
             cmd = CmdImportFile(settings)
             cmd.run()
 
@@ -115,4 +115,4 @@ class TestCmdRemove(DirHierarchyEnvironment):
 
             self.assertTrue(os.path.exists(dvc_name2))
             self.assertTrue(open(dvc_name2).read(), content)
-        pass
+        pass

tests/test_repro.py

@@ -14,8 +14,10 @@ class ReproBasicEnv(RunBasicTest):
         self.file_name1 = os.path.join('data', 'file1')
         self.file1_code_file = 'file1.py'
         self.create_file_and_commit(self.file1_code_file, 'print("Hello")' + os.linesep + 'print("Mary")')
-        self.settings._args = ['python', self.file1_code_file, '--not-repro',
-                                '--stdout', self.file_name1, '--code', self.file1_code_file]
+        self.settings.parse_args(['run',
+                                  '--stdout', self.file_name1,
+                                  '--code', self.file1_code_file,
+                                  'python', self.file1_code_file, '--not-repro'])
         cmd_file1 = CmdRun(self.settings)
         self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file])
         cmd_file1.run()
@@ -25,16 +27,19 @@ class ReproBasicEnv(RunBasicTest):
         self.create_file_and_commit(self.file11_code_file,
                                     'import sys' + os.linesep + 'print(open(sys.argv[1]).readline().strip())')
 
-        self.settings._args = ['python', self.file11_code_file, self.file_name1,
-                               '--stdout', self.file_name11, '--code', self.file11_code_file]
+        self.settings.parse_args(['run',
+                                  '--stdout', self.file_name11,
+                                  '--code', self.file11_code_file,
+                                  'python', self.file11_code_file, self.file_name1])
         CmdRun(self.settings).run()
 
         self.file_name2 = os.path.join('data', 'file2')
         self.file2_code_file = 'file2.py'
         self.create_file_and_commit(self.file2_code_file,
                                     'print("Bobby")')
-        self.settings._args = ['python', self.file2_code_file,
-                               '--not-repro', '--stdout', self.file_name2]
+        self.settings.parse_args(['run',
+                                  '--stdout', self.file_name2,
+                                  'python', self.file2_code_file, '--not-repro'])
         CmdRun(self.settings).run()
 
         self.file_res_code_file = 'code_res.py'
@@ -44,11 +49,13 @@ class ReproBasicEnv(RunBasicTest):
                                     'text2 = open(sys.argv[2]).read()' + os.linesep +
                                     'print(text1 + text2)')
         self.file_name_res = os.path.join('data', 'file_res')
-        self.settings._args = ['python', self.file_res_code_file,
-                               self.file_name11,
-                               self.file_name2,
-                               '--stdout', self.file_name_res,
-                               '--code', self.file_res_code_file]
+        self.settings.parse_args(['run',
+                                  '--stdout', self.file_name_res,
+                                  '--code', self.file_res_code_file,
+                                  'python', self.file_res_code_file,
+                                  self.file_name11,
+                                  self.file_name2])
+
         cmd_res = CmdRun(self.settings)
         self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file])
         cmd_res.run()
@@ -76,7 +83,7 @@ class ReproCodeDependencyTest(ReproBasicEnv):
     def test(self):
         self.modify_file_and_commit(self.file_res_code_file)
 
-        self.settings._args = [self.file_name_res]
+        self.settings.parse_args('repro {}'.format(self.file_name_res))
         CmdRepro(self.settings).run()
 
         self.assertEqual(open(self.file_name_res).read().strip(), 'Hello\nBobby')
@@ -86,19 +93,21 @@ class ReproChangedDependency(ReproBasicEnv):
     def test(self):
         self.recreate_file1()
 
-        self.settings._args = [self.file_name11]
+        self.settings.parse_args('repro {}'.format(self.file_name11))
         CmdRepro(self.settings).run()
 
         self.assertEqual(open(self.file_name11).read(), 'Goodbye\n')
 
     def recreate_file1(self):
-        self.settings._args = [self.file_name1, '--keep-in-cloud']
+        self.settings.parse_args('remove {} --keep-in-cloud'.format(self.file_name1))
         CmdRemove(self.settings).run()
 
         file1_code_file = 'file1_2.py'
         self.create_file_and_commit(file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")')
-        self.settings._args = ['python', file1_code_file, '--not-repro',
-                               '--stdout', self.file_name1, '--code', file1_code_file]
+        self.settings.parse_args(['run',
+                                 '--stdout', self.file_name1,
+                                 '--code', file1_code_file,
+                                 'python', file1_code_file, '--not-repro'])
 
         CmdRun(self.settings).run()
 
@@ -107,7 +116,7 @@ class ReproChangedDeepDependency(ReproChangedDependency):
     def test(self):
         self.recreate_file1()
 
-        self.settings._args = [self.file_name_res]
+        self.settings.parse_args('repro {}'.format(self.file_name_res))
         CmdRepro(self.settings).run()
 
         self.assertEqual(open(self.file_name_res).read().strip(), 'Goodbye\nBobby')