dvc: add push/pull/status per dvc-file

Fixes #453

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-03-04 15:10:33 +02:00
parent 0e25984ccc
commit dbc6e1ac88
3 changed files with 47 additions and 22 deletions
+6
View File
@@ -132,6 +132,12 @@ def parse_args(argv=None):
type=int,
default=cpu_count(),
help='Number of jobs to run simultaneously.')
parent_cache_parser.add_argument(
'targets',
nargs='*',
default=None,
help='DVC files.')
# Cache
cache_parser = subparsers.add_parser(
'cache',
+22 -8
View File
@@ -6,27 +6,41 @@ from dvc.command.common.base import CmdBase
from dvc.exceptions import DvcException
class CmdDataPull(CmdBase):
class CmdDataBase(CmdBase):
def do_run(self, target):
pass
def run(self):
if not self.args.targets:
return self.do_run()
ret = 0
for target in self.args.targets:
if self.do_run(target):
ret = 1
return ret
class CmdDataPull(CmdDataBase):
def do_run(self, target=None):
try:
self.project.pull(self.args.jobs)
self.project.pull(target=target, jobs=self.args.jobs)
except DvcException as exc:
self.project.logger.error('Failed to pull data from the cloud', exc)
return 1
return 0
class CmdDataPush(CmdBase):
def run(self):
class CmdDataPush(CmdDataBase):
def do_run(self, target=None):
try:
self.project.push(self.args.jobs)
self.project.push(target=target, jobs=self.args.jobs)
except DvcException as exc:
self.project.logger.error('Failed to push data to the cloud', exc)
return 1
return 0
class CmdDataStatus(CmdBase):
class CmdDataStatus(CmdDataBase):
def _show(self, status):
for s in status:
target, ret = s
@@ -44,9 +58,9 @@ class CmdDataStatus(CmdBase):
self.project.logger.info('\t{}\t{}'.format(prefix_map[ret], path))
def run(self):
def do_run(self, target=None):
try:
status = self.project.status(self.args.jobs)
status = self.project.status(target=target, jobs=self.args.jobs)
self._show(status)
except DvcException as exc:
self.project.logger.error('Failed to obtain data status', exc)
+19 -14
View File
@@ -159,19 +159,24 @@ class Project(object):
for stage in self.stages():
stage.checkout()
def _used_cache(self):
clist = []
for stage in self.stages():
def _used_cache(self, target=None):
cache_set = set()
if target:
stages = [Stage.load(self, target)]
else:
stages = self.stages()
for stage in stages:
for out in stage.outs:
if not out.use_cache:
continue
if out.cache not in clist:
clist.append(out.cache)
cache_set |= set([out.cache])
if out.is_dir_cache(out.cache) and os.path.isfile(out.cache):
dir_cache = out.dir_cache()
clist.extend(dir_cache.values())
clist.append(out.cache)
return clist
cache_set |= set(dir_cache.values())
return list(cache_set)
def _remove_cache_file(self, cache):
os.chmod(cache, stat.S_IWRITE)
@@ -199,16 +204,16 @@ class Project(object):
self._remove_cache(cache)
self.logger.info(u'\'{}\' was removed'.format(self.to_dvc_path(cache)))
def push(self, jobs=1):
return self.cloud.push(self._used_cache(), jobs)
def push(self, target=None, jobs=1):
return self.cloud.push(self._used_cache(target), jobs)
def pull(self, jobs=1):
ret = self.cloud.pull(self._used_cache(), jobs)
def pull(self, target=None, jobs=1):
ret = self.cloud.pull(self._used_cache(target), jobs)
self.checkout()
return ret
def status(self, jobs=1):
return self.cloud.status(self._used_cache(), jobs)
def status(self, target=None, jobs=1):
return self.cloud.status(self._used_cache(target), jobs)
def graph(self):
G = nx.DiGraph()