Merge pull request #892 from efiop/master

Misc enhancements
This commit is contained in:
Ruslan Kuprieiev
2018-07-16 13:53:04 +03:00
committed by GitHub
5 changed files with 96 additions and 19 deletions
+5
View File
@@ -86,6 +86,11 @@ def parse_args(argv=None):
action='store_true',
default=False,
help="Initiate dvc in directory that is not tracked by any scm tool(e.g. git)")
init_parser.add_argument('-f',
'--force',
action='store_true',
default=False,
help='Force initialization even if \'.dvc\' exists')
init_parser.set_defaults(func=CmdInit)
+1 -1
View File
@@ -8,7 +8,7 @@ class CmdInit(object):
def run_cmd(self):
try:
Project.init('.', no_scm=self.args.no_scm)
Project.init('.', no_scm=self.args.no_scm, force=self.args.force)
except InitError as e:
Logger.error('Failed to initiate dvc', e)
return 1
+1 -1
View File
@@ -8,7 +8,7 @@ from dvc.project import NotDvcFileError
class CmdPipelineShow(CmdBase):
def _show(self, target, commands):
G = self.project.graph()
G = self.project.graph()[0]
stages = networkx.get_node_attributes(G, 'stage')
node = os.path.relpath(os.path.abspath(target), self.project.root_dir)
if node not in stages:
+74 -17
View File
@@ -65,7 +65,7 @@ class Project(object):
self.updater.check()
@staticmethod
def init(root_dir=os.curdir, no_scm=False):
def init(root_dir=os.curdir, no_scm=False, force=False):
"""
Initiate dvc project in directory.
@@ -86,6 +86,12 @@ class Project(object):
msg = "{} is not tracked by any supported scm tool(e.g. git).".format(root_dir)
raise InitError(msg)
if os.path.isdir(dvc_dir):
if not force:
msg = "'{}' exists. Use '-f' to force."
raise InitError(msg.format(os.path.relpath(dvc_dir)))
shutil.rmtree(dvc_dir)
os.mkdir(dvc_dir)
config = Config.init(dvc_dir)
@@ -227,14 +233,21 @@ class Project(object):
def _reproduce_stage(self, stages, node, force):
stage = stages[node].reproduce(force=force)
stage = stages[node]
if stage.locked:
msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
'going to be reproduced.'
self.logger.warn(msg.format(stage.relpath))
stage = stage.reproduce(force=force)
if not stage:
return []
stage.dump()
return [stage]
def reproduce(self, target, recursive=True, force=False):
G = self.graph()
G = self.graph()[1]
stages = nx.get_node_attributes(G, 'stage')
node = os.path.relpath(os.path.abspath(target), self.root_dir)
if node not in stages:
@@ -262,7 +275,7 @@ class Project(object):
self.link_state.remove_unused(used)
def checkout(self, target=None):
all_stages = self.stages()
all_stages = self.active_stages()
if target:
if not Stage.is_stage_file(target):
@@ -274,9 +287,14 @@ class Project(object):
self._cleanup_unused_links(all_stages)
for stage in stages:
if stage.locked:
msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
'going to be checked out.'
self.logger.warn(msg.format(stage.relpath))
stage.checkout()
def _used_cache(self, target=None, all_branches=False):
def _used_cache(self, target=None, all_branches=False, active=True):
cache = {}
cache['local'] = []
cache['s3'] = []
@@ -287,10 +305,17 @@ class Project(object):
for branch in self.scm.brancher(all_branches=all_branches):
if target:
stages = [Stage.load(self, target)]
elif active:
stages = self.active_stages()
else:
stages = self.stages()
for stage in stages:
if active and not target and stage.locked:
msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
'going to be pushed/pulled/fetched.'
self.logger.warn(msg.format(stage.relpath))
for out in stage.outs:
if not out.use_cache or not out.info:
continue
@@ -300,7 +325,9 @@ class Project(object):
return cache
def gc(self, all_branches=False, cloud=False, remote=None):
clist = self._used_cache(target=None, all_branches=all_branches)
clist = self._used_cache(target=None,
all_branches=all_branches,
active=False)
self.cache.local.gc(clist)
if self.cache.s3:
@@ -337,7 +364,7 @@ class Project(object):
if target:
stages = [Stage.load(self, target)]
else:
stages = self.stages()
stages = self.active_stages()
for stage in stages:
status.update(stage.status())
@@ -416,7 +443,8 @@ class Project(object):
def metrics_show(self, path=None, json_path=None, tsv_path=None, htsv_path=None, all_branches=False):
res = {}
for branch in self.scm.brancher(all_branches=all_branches):
metrics = filter(lambda o: o.metric, self.outs())
outs = [out for stage in self.active_stages() for out in stage.outs]
metrics = filter(lambda o: o.metric, outs)
fnames = [path] if path else map(lambda o: o.path, metrics)
for fname in fnames:
rel = os.path.relpath(fname)
@@ -467,14 +495,20 @@ class Project(object):
def graph(self):
G = nx.DiGraph()
G_active = nx.DiGraph()
stages = self.stages()
outs = self.outs()
outs = []
for stage in stages:
outs += stage.outs
# collect the whole DAG
for stage in stages:
node = os.path.relpath(stage.path, self.root_dir)
G.add_node(node, stage=stage)
if stage.locked:
continue
G_active.add_node(node, stage=stage)
for dep in stage.deps:
for out in outs:
if out.path != dep.path and not dep.path.startswith(out.path + out.sep):
@@ -484,8 +518,31 @@ class Project(object):
dep_node = os.path.relpath(dep_stage.path, self.root_dir)
G.add_node(dep_node, stage=dep_stage)
G.add_edge(node, dep_node)
if not stage.locked:
G_active.add_node(dep_node, stage=dep_stage)
G_active.add_edge(node, dep_node)
return G
return G, G_active
def pipelines(self):
G, G_active = self.graph()
# find pipeline ends aka "output stages"
ends = [node for node, in_degree in G.in_degree if in_degree == 0]
# filter out subgraphs that didn't exist in original G
pipelines = []
for c in nx.weakly_connected_components(G_active):
H = G_active.subgraph(c)
found = False
for node in ends:
if node in H:
found = True
break
if found:
pipelines.append(H)
return pipelines
def stages(self):
stages = []
@@ -497,8 +554,8 @@ class Project(object):
stages.append(Stage.load(self, path))
return stages
def outs(self):
outs = []
for stage in self.stages():
outs += stage.outs
return outs
def active_stages(self):
stages = []
for G in self.pipelines():
stages.extend(list(nx.get_node_attributes(G, 'stage').values()))
return stages
+15
View File
@@ -24,6 +24,21 @@ class TestInit(TestGit):
self._test_init()
class TestDoubleInit(TestInit):
def test(self):
ret = main(['init'])
self.assertEqual(ret, 0)
self._test_init()
ret = main(['init'])
self.assertNotEqual(ret, 0)
self._test_init()
ret = main(['init', '--force'])
self.assertEqual(ret, 0)
self._test_init()
class TestInitNoSCMFail(TestDir):
def test_api(self):
with self.assertRaises(InitError):