diff --git a/dvc/cli.py b/dvc/cli.py index 51bc22372..742b11aa9 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -10,7 +10,7 @@ from dvc.command.init import CmdInit from dvc.command.remove import CmdRemove from dvc.command.run import CmdRun from dvc.command.repro import CmdRepro -from dvc.command.data_sync import CmdDataSync +from dvc.command.data_sync import CmdDataSync, CmdDataPush, CmdDataPull from dvc.command.lock import CmdLock from dvc.command.gc import CmdGC from dvc.command.import_file import CmdImportFile @@ -121,22 +121,41 @@ def parse_args(argv=None): help='Arguments of a command') run_parser.set_defaults(func=CmdRun) - # Sync - sync_parser = subparsers.add_parser( - 'sync', - parents=[parent_parser], - help='Synchronize data file with cloud (cloud settings already setup.') - sync_parser.add_argument( + # Parent parser used in sync/pull/push + parent_sync_parser = argparse.ArgumentParser( + add_help=False, + parents=[parent_parser]) + parent_sync_parser.add_argument( 'targets', nargs='+', help='File or directory to sync.') - sync_parser.add_argument('-j', + parent_sync_parser.add_argument('-j', '--jobs', type=int, default=cpu_count(), help='Number of jobs to run simultaneously.') + + # Sync + sync_parser = subparsers.add_parser( + 'sync', + parents=[parent_sync_parser], + help='Synchronize data file with cloud (cloud settings already setup.') sync_parser.set_defaults(func=CmdDataSync) + # Pull + pull_parser = subparsers.add_parser( + 'pull', + parents=[parent_sync_parser], + help='Pull data files from the cloud') + pull_parser.set_defaults(func=CmdDataPull) + + # Push + push_parser = subparsers.add_parser( + 'push', + parents=[parent_sync_parser], + help='Push data files to the cloud') + push_parser.set_defaults(func=CmdDataPush) + # Repro repro_parser = subparsers.add_parser( 'repro', diff --git a/dvc/command/base.py b/dvc/command/base.py index 208939b6f..440a302e9 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -55,6 +55,10 @@ class CmdBase(object): def args(self): return self._settings.args + @property + def cloud(self): + return self._settings.cloud + @property def parsed_args(self): return self._settings._parsed_args diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 6ee12e64b..ce6c0b317 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -6,4 +6,22 @@ class CmdDataSync(CmdBase): def run(self): with DvcLock(self.is_locker, self.git): - self.settings.cloud.sync(self.parsed_args.targets, self.parsed_args.jobs) + self.cloud.sync(self.parsed_args.targets, self.parsed_args.jobs) + + +class CmdDataPull(CmdBase): + def __init__(self, settings): + super(CmdDataPull, self).__init__(settings) + + def run(self): + with DvcLock(self.is_locker, self.git): + self.cloud.pull(self.parsed_args.targets, self.parsed_args.jobs) + + +class CmdDataPush(CmdBase): + def __init__(self, settings): + super(CmdDataPush, self).__init__(settings) + + def run(self): + with DvcLock(self.is_locker, self.git): + self.cloud.push(self.parsed_args.targets, self.parsed_args.jobs) diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 118c78cde..76f8297ae 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -105,15 +105,14 @@ class DataCloudBase(object): def sync_from_cloud(self, item): pass - def sync(self, fname): - item = self._settings.path_factory.data_item(fname) + def sync(self, item): if os.path.isfile(item.resolved_cache.dvc): self.sync_to_cloud(item) else: - self.create_directory(fname, item) + self.create_directory(item) self.sync_from_cloud(item) - def create_directory(self, fname, item): + def create_directory(self, item): self._lock.acquire() try: dir = os.path.dirname(item.cache.relative) @@ -125,7 +124,7 @@ class DataCloudBase(object): raise DataCloudError(u'Cannot create directory {}: {}'.format(dir, ex)) elif not os.path.isdir(dir): msg = u'File {} cannot be synced because {} is not a directory' - raise DataCloudError(msg.format(fname, dir)) + raise DataCloudError(msg.format(item.cache.relative, dir)) finally: self._lock.release() @@ -441,13 +440,16 @@ class DataCloud(object): for root, dirs, files in os.walk(d): for f in files: - targets.append(os.path.join(root, f)) + path = os.path.join(root, f) + item = self._settings.path_factory.data_item(path) + targets.append(item) return targets def _collect_target(self, target): if System.islink(target): - return [target] + item = self._settings.path_factory.data_item(target) + return [item] elif os.path.isdir(target): return self._collect_dir(target) @@ -463,10 +465,19 @@ class DataCloud(object): return collected - def sync(self, targets, jobs=1): + def _map_targets(self, f, targets, jobs): collected = self._collect_targets(targets) - map_progress(self._cloud.sync, collected, jobs) + map_progress(f, collected, jobs) + + def sync(self, targets, jobs=1): + self._map_targets(self._cloud.sync, targets, jobs) + + def push(self, targets, jobs=1): + self._map_targets(self._cloud.sync_to_cloud, targets, jobs) + + def pull(self, targets, jobs=1): + self._map_targets(self._cloud.sync_from_cloud, targets, jobs) def remove_from_cloud(self, item): return self._cloud.remove_from_cloud(item)