mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-15 23:09:47 +01:00
Merge pull request #2230 from shcheklein/optimize-2203
graph outputs overlapping check optimization
This commit is contained in:
+49
-9
@@ -3,6 +3,7 @@ from __future__ import unicode_literals, absolute_import
|
||||
import sys
|
||||
import os
|
||||
import posixpath
|
||||
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.utils.compat import str, builtin_str, basestring, is_py2
|
||||
@@ -100,6 +101,27 @@ class PosixPathInfo(PathInfo, pathlib.PurePosixPath):
|
||||
pass
|
||||
|
||||
|
||||
class _URLPathParents(object):
|
||||
def __init__(self, pathcls, scheme, netloc, path):
|
||||
self._scheme = scheme
|
||||
self._netloc = netloc
|
||||
self._parents = path.parents
|
||||
self._pathcls = pathcls
|
||||
|
||||
def __len__(self):
|
||||
return len(self._parents)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return self._pathcls.from_parts(
|
||||
scheme=self._scheme,
|
||||
netloc=self._netloc,
|
||||
path=self._parents[idx].fspath,
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return "<{}.parents>".format(self._pathcls.__name__)
|
||||
|
||||
|
||||
class URLInfo(object):
|
||||
DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22}
|
||||
|
||||
@@ -124,12 +146,7 @@ class URLInfo(object):
|
||||
@cached_property
|
||||
def url(self):
|
||||
p = self.parsed
|
||||
netloc = p.hostname
|
||||
if p.username:
|
||||
netloc = p.username + "@" + netloc
|
||||
if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme):
|
||||
netloc += ":" + str(p.port)
|
||||
return "{}://{}{}".format(p.scheme, netloc, p.path)
|
||||
return "{}://{}{}".format(p.scheme, self.netloc, p.path)
|
||||
|
||||
def __str__(self):
|
||||
return self.url
|
||||
@@ -140,7 +157,12 @@ class URLInfo(object):
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, basestring):
|
||||
other = self.__class__(other)
|
||||
return str(self) == str(other)
|
||||
return (
|
||||
self.__class__ == other.__class__
|
||||
and self.scheme == other.scheme
|
||||
and self.netloc == other.netloc
|
||||
and self._path == other._path
|
||||
)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.url)
|
||||
@@ -158,6 +180,16 @@ class URLInfo(object):
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.parsed, name)
|
||||
|
||||
@cached_property
|
||||
def netloc(self):
|
||||
p = self.parsed
|
||||
netloc = p.hostname
|
||||
if p.username:
|
||||
netloc = p.username + "@" + netloc
|
||||
if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme):
|
||||
netloc += ":" + str(p.port)
|
||||
return netloc
|
||||
|
||||
@property
|
||||
def port(self):
|
||||
return self.parsed.port or self.DEFAULT_PORTS.get(self.parsed.scheme)
|
||||
@@ -184,12 +216,20 @@ class URLInfo(object):
|
||||
|
||||
@property
|
||||
def bucket(self):
|
||||
return self.netloc
|
||||
return self.parsed.netloc
|
||||
|
||||
@property
|
||||
def parent(self):
|
||||
return self.from_parts(
|
||||
scheme=self.scheme, netloc=self.netloc, path=self._path.parent
|
||||
scheme=self.scheme,
|
||||
netloc=self.parsed.netloc,
|
||||
path=self._path.parent.fspath,
|
||||
)
|
||||
|
||||
@property
|
||||
def parents(self):
|
||||
return _URLPathParents(
|
||||
type(self), self.scheme, self.parsed.netloc, self._path
|
||||
)
|
||||
|
||||
def relative_to(self, other):
|
||||
|
||||
+23
-28
@@ -3,6 +3,8 @@ from __future__ import unicode_literals
|
||||
import os
|
||||
import logging
|
||||
|
||||
from itertools import chain
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.exceptions import (
|
||||
NotDvcRepoError,
|
||||
@@ -300,30 +302,25 @@ class Repo(object):
|
||||
G_active = nx.DiGraph()
|
||||
stages = stages or self.stages(from_directory, check_dag=False)
|
||||
stages = [stage for stage in stages if stage]
|
||||
outs = []
|
||||
outs = {}
|
||||
|
||||
for stage in stages:
|
||||
for out in stage.outs:
|
||||
existing = []
|
||||
for o in outs:
|
||||
if o.path_info == out.path_info:
|
||||
existing.append(o.stage)
|
||||
|
||||
in_o_dir = out.path_info.isin(o.path_info)
|
||||
in_out_dir = o.path_info.isin(out.path_info)
|
||||
if in_o_dir or in_out_dir:
|
||||
raise OverlappingOutputPathsError(o, out)
|
||||
|
||||
if existing:
|
||||
stages = [stage.relpath, existing[0].relpath]
|
||||
if out.path_info in outs:
|
||||
stages = [stage.relpath, outs[out.path_info].stage.relpath]
|
||||
raise OutputDuplicationError(str(out), stages)
|
||||
outs[out.path_info] = out
|
||||
|
||||
outs.append(out)
|
||||
for stage in stages:
|
||||
for out in stage.outs:
|
||||
for p in out.path_info.parents:
|
||||
if p in outs:
|
||||
raise OverlappingOutputPathsError(outs[p], out)
|
||||
|
||||
for stage in stages:
|
||||
stage_path_info = PathInfo(stage.path)
|
||||
for out in outs:
|
||||
if stage_path_info.isin(out.path_info):
|
||||
for p in chain([stage_path_info], stage_path_info.parents):
|
||||
if p in outs:
|
||||
raise StagePathAsOutputError(stage.wdir, stage.relpath)
|
||||
|
||||
for stage in stages:
|
||||
@@ -335,19 +332,17 @@ class Repo(object):
|
||||
for dep in stage.deps:
|
||||
for out in outs:
|
||||
if (
|
||||
out.path_info != dep.path_info
|
||||
and not dep.path_info.isin(out.path_info)
|
||||
and not out.path_info.isin(dep.path_info)
|
||||
out == dep.path_info
|
||||
or dep.path_info.isin(out)
|
||||
or out.isin(dep.path_info)
|
||||
):
|
||||
continue
|
||||
|
||||
dep_stage = out.stage
|
||||
dep_node = relpath(dep_stage.path, self.root_dir)
|
||||
G.add_node(dep_node, stage=dep_stage)
|
||||
G.add_edge(node, dep_node)
|
||||
if not stage.locked:
|
||||
G_active.add_node(dep_node, stage=dep_stage)
|
||||
G_active.add_edge(node, dep_node)
|
||||
dep_stage = outs[out].stage
|
||||
dep_node = relpath(dep_stage.path, self.root_dir)
|
||||
G.add_node(dep_node, stage=dep_stage)
|
||||
G.add_edge(node, dep_node)
|
||||
if not stage.locked:
|
||||
G_active.add_node(dep_node, stage=dep_stage)
|
||||
G_active.add_edge(node, dep_node)
|
||||
|
||||
self._check_cyclic_graph(G)
|
||||
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import pytest
|
||||
|
||||
from dvc.path_info import URLInfo, CloudURLInfo
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo])
|
||||
def test_url_info_str(cls):
|
||||
u = cls("ssh://[email protected]:/test1/")
|
||||
assert u.url == "ssh://[email protected]/test1/"
|
||||
assert str(u) == u.url
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo])
|
||||
def test_url_info_eq(cls):
|
||||
u1 = cls("ssh://[email protected]:/test1/")
|
||||
u2 = cls("ssh://[email protected]/test1")
|
||||
assert u1 == u2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo])
|
||||
def test_url_info_parent(cls):
|
||||
u1 = cls("ssh://[email protected]:/test1/test2")
|
||||
p = u1.parent
|
||||
u3 = cls("ssh://[email protected]/test1")
|
||||
assert u3 == p
|
||||
assert str(p) == "ssh://[email protected]/test1"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo])
|
||||
def test_url_info_parents(cls):
|
||||
u1 = cls("ssh://[email protected]:/test1/test2/test3")
|
||||
assert list(u1.parents) == [
|
||||
cls("ssh://[email protected]/test1/test2"),
|
||||
cls("ssh://[email protected]/test1"),
|
||||
cls("ssh://[email protected]/"),
|
||||
]
|
||||
Reference in New Issue
Block a user