diff --git a/dvc/path_info.py b/dvc/path_info.py index 657d7762f..2c81cdc82 100644 --- a/dvc/path_info.py +++ b/dvc/path_info.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals, absolute_import import sys import os import posixpath + from funcy import cached_property from dvc.utils.compat import str, builtin_str, basestring, is_py2 @@ -100,6 +101,27 @@ class PosixPathInfo(PathInfo, pathlib.PurePosixPath): pass +class _URLPathParents(object): + def __init__(self, pathcls, scheme, netloc, path): + self._scheme = scheme + self._netloc = netloc + self._parents = path.parents + self._pathcls = pathcls + + def __len__(self): + return len(self._parents) + + def __getitem__(self, idx): + return self._pathcls.from_parts( + scheme=self._scheme, + netloc=self._netloc, + path=self._parents[idx].fspath, + ) + + def __repr__(self): + return "<{}.parents>".format(self._pathcls.__name__) + + class URLInfo(object): DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22} @@ -124,12 +146,7 @@ class URLInfo(object): @cached_property def url(self): p = self.parsed - netloc = p.hostname - if p.username: - netloc = p.username + "@" + netloc - if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme): - netloc += ":" + str(p.port) - return "{}://{}{}".format(p.scheme, netloc, p.path) + return "{}://{}{}".format(p.scheme, self.netloc, p.path) def __str__(self): return self.url @@ -140,7 +157,12 @@ class URLInfo(object): def __eq__(self, other): if isinstance(other, basestring): other = self.__class__(other) - return str(self) == str(other) + return ( + self.__class__ == other.__class__ + and self.scheme == other.scheme + and self.netloc == other.netloc + and self._path == other._path + ) def __hash__(self): return hash(self.url) @@ -158,6 +180,16 @@ class URLInfo(object): def __getattr__(self, name): return getattr(self.parsed, name) + @cached_property + def netloc(self): + p = self.parsed + netloc = p.hostname + if p.username: + netloc = p.username + "@" + netloc + if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme): + netloc += ":" + str(p.port) + return netloc + @property def port(self): return self.parsed.port or self.DEFAULT_PORTS.get(self.parsed.scheme) @@ -184,12 +216,20 @@ class URLInfo(object): @property def bucket(self): - return self.netloc + return self.parsed.netloc @property def parent(self): return self.from_parts( - scheme=self.scheme, netloc=self.netloc, path=self._path.parent + scheme=self.scheme, + netloc=self.parsed.netloc, + path=self._path.parent.fspath, + ) + + @property + def parents(self): + return _URLPathParents( + type(self), self.scheme, self.parsed.netloc, self._path ) def relative_to(self, other): diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index df0124495..f4ac80595 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals import os import logging +from itertools import chain + from dvc.config import Config from dvc.exceptions import ( NotDvcRepoError, @@ -300,30 +302,25 @@ class Repo(object): G_active = nx.DiGraph() stages = stages or self.stages(from_directory, check_dag=False) stages = [stage for stage in stages if stage] - outs = [] + outs = {} for stage in stages: for out in stage.outs: - existing = [] - for o in outs: - if o.path_info == out.path_info: - existing.append(o.stage) - - in_o_dir = out.path_info.isin(o.path_info) - in_out_dir = o.path_info.isin(out.path_info) - if in_o_dir or in_out_dir: - raise OverlappingOutputPathsError(o, out) - - if existing: - stages = [stage.relpath, existing[0].relpath] + if out.path_info in outs: + stages = [stage.relpath, outs[out.path_info].stage.relpath] raise OutputDuplicationError(str(out), stages) + outs[out.path_info] = out - outs.append(out) + for stage in stages: + for out in stage.outs: + for p in out.path_info.parents: + if p in outs: + raise OverlappingOutputPathsError(outs[p], out) for stage in stages: stage_path_info = PathInfo(stage.path) - for out in outs: - if stage_path_info.isin(out.path_info): + for p in chain([stage_path_info], stage_path_info.parents): + if p in outs: raise StagePathAsOutputError(stage.wdir, stage.relpath) for stage in stages: @@ -335,19 +332,17 @@ class Repo(object): for dep in stage.deps: for out in outs: if ( - out.path_info != dep.path_info - and not dep.path_info.isin(out.path_info) - and not out.path_info.isin(dep.path_info) + out == dep.path_info + or dep.path_info.isin(out) + or out.isin(dep.path_info) ): - continue - - dep_stage = out.stage - dep_node = relpath(dep_stage.path, self.root_dir) - G.add_node(dep_node, stage=dep_stage) - G.add_edge(node, dep_node) - if not stage.locked: - G_active.add_node(dep_node, stage=dep_stage) - G_active.add_edge(node, dep_node) + dep_stage = outs[out].stage + dep_node = relpath(dep_stage.path, self.root_dir) + G.add_node(dep_node, stage=dep_stage) + G.add_edge(node, dep_node) + if not stage.locked: + G_active.add_node(dep_node, stage=dep_stage) + G_active.add_edge(node, dep_node) self._check_cyclic_graph(G) diff --git a/tests/unit/test_path_info.py b/tests/unit/test_path_info.py new file mode 100644 index 000000000..46bd032b9 --- /dev/null +++ b/tests/unit/test_path_info.py @@ -0,0 +1,36 @@ +import pytest + +from dvc.path_info import URLInfo, CloudURLInfo + + +@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo]) +def test_url_info_str(cls): + u = cls("ssh://user@test.com:/test1/") + assert u.url == "ssh://user@test.com/test1/" + assert str(u) == u.url + + +@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo]) +def test_url_info_eq(cls): + u1 = cls("ssh://user@test.com:/test1/") + u2 = cls("ssh://user@test.com/test1") + assert u1 == u2 + + +@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo]) +def test_url_info_parent(cls): + u1 = cls("ssh://user@test.com:/test1/test2") + p = u1.parent + u3 = cls("ssh://user@test.com/test1") + assert u3 == p + assert str(p) == "ssh://user@test.com/test1" + + +@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo]) +def test_url_info_parents(cls): + u1 = cls("ssh://user@test.com:/test1/test2/test3") + assert list(u1.parents) == [ + cls("ssh://user@test.com/test1/test2"), + cls("ssh://user@test.com/test1"), + cls("ssh://user@test.com/"), + ]