dvc: add pull/push commands

We already have 'sync' command, but users report it to be confusing
when both local and remote files exist, so lets introduce two direct
commands to implicitly push and pull from cloud.

Fixes #157.

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/cli.py

@@ -10,7 +10,7 @@ from dvc.command.init import CmdInit
 from dvc.command.remove import CmdRemove
 from dvc.command.run import CmdRun
 from dvc.command.repro import CmdRepro
-from dvc.command.data_sync import CmdDataSync
+from dvc.command.data_sync import CmdDataSync, CmdDataPush, CmdDataPull
 from dvc.command.lock import CmdLock
 from dvc.command.gc import CmdGC
 from dvc.command.import_file import CmdImportFile
@@ -121,22 +121,41 @@ def parse_args(argv=None):
                         help='Arguments of a command')
     run_parser.set_defaults(func=CmdRun)
 
-    # Sync
-    sync_parser = subparsers.add_parser(
-                        'sync',
-                        parents=[parent_parser],
-                        help='Synchronize data file with cloud (cloud settings already setup.')
-    sync_parser.add_argument(
+    # Parent parser used in sync/pull/push
+    parent_sync_parser = argparse.ArgumentParser(
+                        add_help=False,
+                        parents=[parent_parser])
+    parent_sync_parser.add_argument(
                         'targets',
                         nargs='+',
                         help='File or directory to sync.')
-    sync_parser.add_argument('-j',
+    parent_sync_parser.add_argument('-j',
                         '--jobs',
                         type=int,
                         default=cpu_count(),
                         help='Number of jobs to run simultaneously.')
+
+    # Sync
+    sync_parser = subparsers.add_parser(
+                        'sync',
+                        parents=[parent_sync_parser],
+                        help='Synchronize data file with cloud (cloud settings already setup.')
     sync_parser.set_defaults(func=CmdDataSync)
 
+    # Pull
+    pull_parser = subparsers.add_parser(
+                        'pull',
+                        parents=[parent_sync_parser],
+                        help='Pull data files from the cloud')
+    pull_parser.set_defaults(func=CmdDataPull)
+
+    # Push
+    push_parser = subparsers.add_parser(
+                        'push',
+                        parents=[parent_sync_parser],
+                        help='Push data files to the cloud')
+    push_parser.set_defaults(func=CmdDataPush)
+
     # Repro
     repro_parser = subparsers.add_parser(
                         'repro',

dvc/command/base.py

@@ -55,6 +55,10 @@ class CmdBase(object):
     def args(self):
         return self._settings.args
 
+    @property
+    def cloud(self):
+        return self._settings.cloud
+
     @property
     def parsed_args(self):
         return self._settings._parsed_args

dvc/command/data_sync.py

@@ -6,4 +6,22 @@ class CmdDataSync(CmdBase):
 
     def run(self):
         with DvcLock(self.is_locker, self.git):
-            self.settings.cloud.sync(self.parsed_args.targets, self.parsed_args.jobs)
+            self.cloud.sync(self.parsed_args.targets, self.parsed_args.jobs)
+
+
+class CmdDataPull(CmdBase):
+    def __init__(self, settings):
+        super(CmdDataPull, self).__init__(settings)
+
+    def run(self):
+        with DvcLock(self.is_locker, self.git):
+            self.cloud.pull(self.parsed_args.targets, self.parsed_args.jobs)
+
+
+class CmdDataPush(CmdBase):
+    def __init__(self, settings):
+        super(CmdDataPush, self).__init__(settings)
+
+    def run(self):
+        with DvcLock(self.is_locker, self.git):
+            self.cloud.push(self.parsed_args.targets, self.parsed_args.jobs)

dvc/data_cloud.py

@@ -105,15 +105,14 @@ class DataCloudBase(object):
     def sync_from_cloud(self, item):
         pass
 
-    def sync(self, fname):
-        item = self._settings.path_factory.data_item(fname)
+    def sync(self, item):
         if os.path.isfile(item.resolved_cache.dvc):
             self.sync_to_cloud(item)
         else:
-            self.create_directory(fname, item)
+            self.create_directory(item)
             self.sync_from_cloud(item)
 
-    def create_directory(self, fname, item):
+    def create_directory(self, item):
         self._lock.acquire()
         try:
             dir = os.path.dirname(item.cache.relative)
@@ -125,7 +124,7 @@ class DataCloudBase(object):
                     raise DataCloudError(u'Cannot create directory {}: {}'.format(dir, ex))
             elif not os.path.isdir(dir):
                 msg = u'File {} cannot be synced because {} is not a directory'
-                raise DataCloudError(msg.format(fname, dir))
+                raise DataCloudError(msg.format(item.cache.relative, dir))
         finally:
             self._lock.release()
 
@@ -441,13 +440,16 @@ class DataCloud(object):
 
         for root, dirs, files in os.walk(d):
             for f in files:
-                targets.append(os.path.join(root, f))
+                path = os.path.join(root, f)
+                item = self._settings.path_factory.data_item(path)
+                targets.append(item)
 
         return targets
 
     def _collect_target(self, target):
         if System.islink(target):
-            return [target]
+            item = self._settings.path_factory.data_item(target)
+            return [item]
         elif os.path.isdir(target):
             return self._collect_dir(target)
 
@@ -463,10 +465,19 @@ class DataCloud(object):
 
         return collected
 
-    def sync(self, targets, jobs=1):
+    def _map_targets(self, f, targets, jobs):
         collected = self._collect_targets(targets)
 
-        map_progress(self._cloud.sync, collected, jobs)
+        map_progress(f, collected, jobs)
+
+    def sync(self, targets, jobs=1):
+        self._map_targets(self._cloud.sync, targets, jobs)
+
+    def push(self, targets, jobs=1):
+        self._map_targets(self._cloud.sync_to_cloud, targets, jobs)
+
+    def pull(self, targets, jobs=1):
+        self._map_targets(self._cloud.sync_from_cloud, targets, jobs)
 
     def remove_from_cloud(self, item):
         return self._cloud.remove_from_cloud(item)