From db43e6bc91767f0db04d0d8efab714ea2d3a2a30 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 12 Aug 2017 20:16:33 +0300 Subject: [PATCH] dvc: add pull/push commands We already have 'sync' command, but users report it to be confusing when both local and remote files exist, so lets introduce two direct commands to implicitly push and pull from cloud. Fixes #157. Signed-off-by: Ruslan Kuprieiev --- dvc/cli.py | 35 +++++++++++++++++++++++++++-------- dvc/command/base.py | 4 ++++ dvc/command/data_sync.py | 20 +++++++++++++++++++- dvc/data_cloud.py | 29 ++++++++++++++++++++--------- 4 files changed, 70 insertions(+), 18 deletions(-) diff --git a/dvc/cli.py b/dvc/cli.py index 51bc22372..742b11aa9 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -10,7 +10,7 @@ from dvc.command.init import CmdInit from dvc.command.remove import CmdRemove from dvc.command.run import CmdRun from dvc.command.repro import CmdRepro -from dvc.command.data_sync import CmdDataSync +from dvc.command.data_sync import CmdDataSync, CmdDataPush, CmdDataPull from dvc.command.lock import CmdLock from dvc.command.gc import CmdGC from dvc.command.import_file import CmdImportFile @@ -121,22 +121,41 @@ def parse_args(argv=None): help='Arguments of a command') run_parser.set_defaults(func=CmdRun) - # Sync - sync_parser = subparsers.add_parser( - 'sync', - parents=[parent_parser], - help='Synchronize data file with cloud (cloud settings already setup.') - sync_parser.add_argument( + # Parent parser used in sync/pull/push + parent_sync_parser = argparse.ArgumentParser( + add_help=False, + parents=[parent_parser]) + parent_sync_parser.add_argument( 'targets', nargs='+', help='File or directory to sync.') - sync_parser.add_argument('-j', + parent_sync_parser.add_argument('-j', '--jobs', type=int, default=cpu_count(), help='Number of jobs to run simultaneously.') + + # Sync + sync_parser = subparsers.add_parser( + 'sync', + parents=[parent_sync_parser], + help='Synchronize data file with cloud (cloud settings already setup.') sync_parser.set_defaults(func=CmdDataSync) + # Pull + pull_parser = subparsers.add_parser( + 'pull', + parents=[parent_sync_parser], + help='Pull data files from the cloud') + pull_parser.set_defaults(func=CmdDataPull) + + # Push + push_parser = subparsers.add_parser( + 'push', + parents=[parent_sync_parser], + help='Push data files to the cloud') + push_parser.set_defaults(func=CmdDataPush) + # Repro repro_parser = subparsers.add_parser( 'repro', diff --git a/dvc/command/base.py b/dvc/command/base.py index 208939b6f..440a302e9 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -55,6 +55,10 @@ class CmdBase(object): def args(self): return self._settings.args + @property + def cloud(self): + return self._settings.cloud + @property def parsed_args(self): return self._settings._parsed_args diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 6ee12e64b..ce6c0b317 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -6,4 +6,22 @@ class CmdDataSync(CmdBase): def run(self): with DvcLock(self.is_locker, self.git): - self.settings.cloud.sync(self.parsed_args.targets, self.parsed_args.jobs) + self.cloud.sync(self.parsed_args.targets, self.parsed_args.jobs) + + +class CmdDataPull(CmdBase): + def __init__(self, settings): + super(CmdDataPull, self).__init__(settings) + + def run(self): + with DvcLock(self.is_locker, self.git): + self.cloud.pull(self.parsed_args.targets, self.parsed_args.jobs) + + +class CmdDataPush(CmdBase): + def __init__(self, settings): + super(CmdDataPush, self).__init__(settings) + + def run(self): + with DvcLock(self.is_locker, self.git): + self.cloud.push(self.parsed_args.targets, self.parsed_args.jobs) diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 118c78cde..76f8297ae 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -105,15 +105,14 @@ class DataCloudBase(object): def sync_from_cloud(self, item): pass - def sync(self, fname): - item = self._settings.path_factory.data_item(fname) + def sync(self, item): if os.path.isfile(item.resolved_cache.dvc): self.sync_to_cloud(item) else: - self.create_directory(fname, item) + self.create_directory(item) self.sync_from_cloud(item) - def create_directory(self, fname, item): + def create_directory(self, item): self._lock.acquire() try: dir = os.path.dirname(item.cache.relative) @@ -125,7 +124,7 @@ class DataCloudBase(object): raise DataCloudError(u'Cannot create directory {}: {}'.format(dir, ex)) elif not os.path.isdir(dir): msg = u'File {} cannot be synced because {} is not a directory' - raise DataCloudError(msg.format(fname, dir)) + raise DataCloudError(msg.format(item.cache.relative, dir)) finally: self._lock.release() @@ -441,13 +440,16 @@ class DataCloud(object): for root, dirs, files in os.walk(d): for f in files: - targets.append(os.path.join(root, f)) + path = os.path.join(root, f) + item = self._settings.path_factory.data_item(path) + targets.append(item) return targets def _collect_target(self, target): if System.islink(target): - return [target] + item = self._settings.path_factory.data_item(target) + return [item] elif os.path.isdir(target): return self._collect_dir(target) @@ -463,10 +465,19 @@ class DataCloud(object): return collected - def sync(self, targets, jobs=1): + def _map_targets(self, f, targets, jobs): collected = self._collect_targets(targets) - map_progress(self._cloud.sync, collected, jobs) + map_progress(f, collected, jobs) + + def sync(self, targets, jobs=1): + self._map_targets(self._cloud.sync, targets, jobs) + + def push(self, targets, jobs=1): + self._map_targets(self._cloud.sync_to_cloud, targets, jobs) + + def pull(self, targets, jobs=1): + self._map_targets(self._cloud.sync_from_cloud, targets, jobs) def remove_from_cloud(self, item): return self._cloud.remove_from_cloud(item)