Merge pull request #2230 from shcheklein/optimize-2203

graph outputs overlapping check optimization
This commit is contained in:
Ruslan Kuprieiev
2019-07-05 15:35:49 +03:00
committed by GitHub
3 changed files with 108 additions and 37 deletions
+49 -9
View File
@@ -3,6 +3,7 @@ from __future__ import unicode_literals, absolute_import
import sys
import os
import posixpath
from funcy import cached_property
from dvc.utils.compat import str, builtin_str, basestring, is_py2
@@ -100,6 +101,27 @@ class PosixPathInfo(PathInfo, pathlib.PurePosixPath):
pass
class _URLPathParents(object):
def __init__(self, pathcls, scheme, netloc, path):
self._scheme = scheme
self._netloc = netloc
self._parents = path.parents
self._pathcls = pathcls
def __len__(self):
return len(self._parents)
def __getitem__(self, idx):
return self._pathcls.from_parts(
scheme=self._scheme,
netloc=self._netloc,
path=self._parents[idx].fspath,
)
def __repr__(self):
return "<{}.parents>".format(self._pathcls.__name__)
class URLInfo(object):
DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22}
@@ -124,12 +146,7 @@ class URLInfo(object):
@cached_property
def url(self):
p = self.parsed
netloc = p.hostname
if p.username:
netloc = p.username + "@" + netloc
if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme):
netloc += ":" + str(p.port)
return "{}://{}{}".format(p.scheme, netloc, p.path)
return "{}://{}{}".format(p.scheme, self.netloc, p.path)
def __str__(self):
return self.url
@@ -140,7 +157,12 @@ class URLInfo(object):
def __eq__(self, other):
if isinstance(other, basestring):
other = self.__class__(other)
return str(self) == str(other)
return (
self.__class__ == other.__class__
and self.scheme == other.scheme
and self.netloc == other.netloc
and self._path == other._path
)
def __hash__(self):
return hash(self.url)
@@ -158,6 +180,16 @@ class URLInfo(object):
def __getattr__(self, name):
return getattr(self.parsed, name)
@cached_property
def netloc(self):
p = self.parsed
netloc = p.hostname
if p.username:
netloc = p.username + "@" + netloc
if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme):
netloc += ":" + str(p.port)
return netloc
@property
def port(self):
return self.parsed.port or self.DEFAULT_PORTS.get(self.parsed.scheme)
@@ -184,12 +216,20 @@ class URLInfo(object):
@property
def bucket(self):
return self.netloc
return self.parsed.netloc
@property
def parent(self):
return self.from_parts(
scheme=self.scheme, netloc=self.netloc, path=self._path.parent
scheme=self.scheme,
netloc=self.parsed.netloc,
path=self._path.parent.fspath,
)
@property
def parents(self):
return _URLPathParents(
type(self), self.scheme, self.parsed.netloc, self._path
)
def relative_to(self, other):
+23 -28
View File
@@ -3,6 +3,8 @@ from __future__ import unicode_literals
import os
import logging
from itertools import chain
from dvc.config import Config
from dvc.exceptions import (
NotDvcRepoError,
@@ -300,30 +302,25 @@ class Repo(object):
G_active = nx.DiGraph()
stages = stages or self.stages(from_directory, check_dag=False)
stages = [stage for stage in stages if stage]
outs = []
outs = {}
for stage in stages:
for out in stage.outs:
existing = []
for o in outs:
if o.path_info == out.path_info:
existing.append(o.stage)
in_o_dir = out.path_info.isin(o.path_info)
in_out_dir = o.path_info.isin(out.path_info)
if in_o_dir or in_out_dir:
raise OverlappingOutputPathsError(o, out)
if existing:
stages = [stage.relpath, existing[0].relpath]
if out.path_info in outs:
stages = [stage.relpath, outs[out.path_info].stage.relpath]
raise OutputDuplicationError(str(out), stages)
outs[out.path_info] = out
outs.append(out)
for stage in stages:
for out in stage.outs:
for p in out.path_info.parents:
if p in outs:
raise OverlappingOutputPathsError(outs[p], out)
for stage in stages:
stage_path_info = PathInfo(stage.path)
for out in outs:
if stage_path_info.isin(out.path_info):
for p in chain([stage_path_info], stage_path_info.parents):
if p in outs:
raise StagePathAsOutputError(stage.wdir, stage.relpath)
for stage in stages:
@@ -335,19 +332,17 @@ class Repo(object):
for dep in stage.deps:
for out in outs:
if (
out.path_info != dep.path_info
and not dep.path_info.isin(out.path_info)
and not out.path_info.isin(dep.path_info)
out == dep.path_info
or dep.path_info.isin(out)
or out.isin(dep.path_info)
):
continue
dep_stage = out.stage
dep_node = relpath(dep_stage.path, self.root_dir)
G.add_node(dep_node, stage=dep_stage)
G.add_edge(node, dep_node)
if not stage.locked:
G_active.add_node(dep_node, stage=dep_stage)
G_active.add_edge(node, dep_node)
dep_stage = outs[out].stage
dep_node = relpath(dep_stage.path, self.root_dir)
G.add_node(dep_node, stage=dep_stage)
G.add_edge(node, dep_node)
if not stage.locked:
G_active.add_node(dep_node, stage=dep_stage)
G_active.add_edge(node, dep_node)
self._check_cyclic_graph(G)
+36
View File
@@ -0,0 +1,36 @@
import pytest
from dvc.path_info import URLInfo, CloudURLInfo
@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo])
def test_url_info_str(cls):
u = cls("ssh://[email protected]:/test1/")
assert u.url == "ssh://[email protected]/test1/"
assert str(u) == u.url
@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo])
def test_url_info_eq(cls):
u1 = cls("ssh://[email protected]:/test1/")
u2 = cls("ssh://[email protected]/test1")
assert u1 == u2
@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo])
def test_url_info_parent(cls):
u1 = cls("ssh://[email protected]:/test1/test2")
p = u1.parent
u3 = cls("ssh://[email protected]/test1")
assert u3 == p
assert str(p) == "ssh://[email protected]/test1"
@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo])
def test_url_info_parents(cls):
u1 = cls("ssh://[email protected]:/test1/test2/test3")
assert list(u1.parents) == [
cls("ssh://[email protected]/test1/test2"),
cls("ssh://[email protected]/test1"),
cls("ssh://[email protected]/"),
]