From dbc6e1ac88e457a8011e6eb6cd5029b24c437348 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 4 Mar 2018 15:10:33 +0200 Subject: [PATCH] dvc: add push/pull/status per dvc-file Fixes #453 Signed-off-by: Ruslan Kuprieiev --- dvc/cli.py | 6 ++++++ dvc/command/data_sync.py | 30 ++++++++++++++++++++++-------- dvc/project.py | 33 +++++++++++++++++++-------------- 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/dvc/cli.py b/dvc/cli.py index a1be92d47..63eab0292 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -132,6 +132,12 @@ def parse_args(argv=None): type=int, default=cpu_count(), help='Number of jobs to run simultaneously.') + parent_cache_parser.add_argument( + 'targets', + nargs='*', + default=None, + help='DVC files.') + # Cache cache_parser = subparsers.add_parser( 'cache', diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index caa33c4ef..7c2559976 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -6,27 +6,41 @@ from dvc.command.common.base import CmdBase from dvc.exceptions import DvcException -class CmdDataPull(CmdBase): +class CmdDataBase(CmdBase): + def do_run(self, target): + pass + def run(self): + if not self.args.targets: + return self.do_run() + + ret = 0 + for target in self.args.targets: + if self.do_run(target): + ret = 1 + return ret + +class CmdDataPull(CmdDataBase): + def do_run(self, target=None): try: - self.project.pull(self.args.jobs) + self.project.pull(target=target, jobs=self.args.jobs) except DvcException as exc: self.project.logger.error('Failed to pull data from the cloud', exc) return 1 return 0 -class CmdDataPush(CmdBase): - def run(self): +class CmdDataPush(CmdDataBase): + def do_run(self, target=None): try: - self.project.push(self.args.jobs) + self.project.push(target=target, jobs=self.args.jobs) except DvcException as exc: self.project.logger.error('Failed to push data to the cloud', exc) return 1 return 0 -class CmdDataStatus(CmdBase): +class CmdDataStatus(CmdDataBase): def _show(self, status): for s in status: target, ret = s @@ -44,9 +58,9 @@ class CmdDataStatus(CmdBase): self.project.logger.info('\t{}\t{}'.format(prefix_map[ret], path)) - def run(self): + def do_run(self, target=None): try: - status = self.project.status(self.args.jobs) + status = self.project.status(target=target, jobs=self.args.jobs) self._show(status) except DvcException as exc: self.project.logger.error('Failed to obtain data status', exc) diff --git a/dvc/project.py b/dvc/project.py index f7522ecc9..2df906b8c 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -159,19 +159,24 @@ class Project(object): for stage in self.stages(): stage.checkout() - def _used_cache(self): - clist = [] - for stage in self.stages(): + def _used_cache(self, target=None): + cache_set = set() + + if target: + stages = [Stage.load(self, target)] + else: + stages = self.stages() + + for stage in stages: for out in stage.outs: if not out.use_cache: continue - if out.cache not in clist: - clist.append(out.cache) + cache_set |= set([out.cache]) if out.is_dir_cache(out.cache) and os.path.isfile(out.cache): dir_cache = out.dir_cache() - clist.extend(dir_cache.values()) - clist.append(out.cache) - return clist + cache_set |= set(dir_cache.values()) + + return list(cache_set) def _remove_cache_file(self, cache): os.chmod(cache, stat.S_IWRITE) @@ -199,16 +204,16 @@ class Project(object): self._remove_cache(cache) self.logger.info(u'\'{}\' was removed'.format(self.to_dvc_path(cache))) - def push(self, jobs=1): - return self.cloud.push(self._used_cache(), jobs) + def push(self, target=None, jobs=1): + return self.cloud.push(self._used_cache(target), jobs) - def pull(self, jobs=1): - ret = self.cloud.pull(self._used_cache(), jobs) + def pull(self, target=None, jobs=1): + ret = self.cloud.pull(self._used_cache(target), jobs) self.checkout() return ret - def status(self, jobs=1): - return self.cloud.status(self._used_cache(), jobs) + def status(self, target=None, jobs=1): + return self.cloud.status(self._used_cache(target), jobs) def graph(self): G = nx.DiGraph()