From c38551b62c658101f5d0b6fc5810cff3dfb520f9 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 4 Jul 2019 14:27:48 -0700 Subject: [PATCH 1/8] path info: optimize isin --- dvc/path_info.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dvc/path_info.py b/dvc/path_info.py index 657d7762f..d16b452e4 100644 --- a/dvc/path_info.py +++ b/dvc/path_info.py @@ -57,7 +57,11 @@ class PathInfo(pathlib.PurePath): other = self.__class__(other) elif self.__class__ != other.__class__: return False - return any(p == other for p in self.parents) + for p in self.parents: + if p == other: + return True + if len(other.parts) >= len(p.parts): + return False # pathlib2 uses bytes internally in Python 2, and we use unicode everywhere # for paths in both pythons, thus we need this glue. From 2dfefbade335223bd2a84d4ac194bdb0f6c80677 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 4 Jul 2019 14:28:24 -0700 Subject: [PATCH 2/8] repo graph: optimize output checks --- dvc/repo/__init__.py | 58 ++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index df0124495..fe4c469a0 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -300,30 +300,32 @@ class Repo(object): G_active = nx.DiGraph() stages = stages or self.stages(from_directory, check_dag=False) stages = [stage for stage in stages if stage] - outs = [] + outs = dict() for stage in stages: for out in stage.outs: - existing = [] - for o in outs: - if o.path_info == out.path_info: - existing.append(o.stage) - - in_o_dir = out.path_info.isin(o.path_info) - in_out_dir = o.path_info.isin(out.path_info) - if in_o_dir or in_out_dir: - raise OverlappingOutputPathsError(o, out) - - if existing: - stages = [stage.relpath, existing[0].relpath] + if out.path_info in outs: + stages = [ + stage.relpath, + outs.get(out.path_info).stage.relpath, + ] raise OutputDuplicationError(str(out), stages) + outs[out.path_info] = out - outs.append(out) + for stage in stages: + for out in stage.outs: + for p in out.path_info.parents: + if p in outs: + raise OverlappingOutputPathsError( + outs.get(p), outs.get(out.path_info) + ) for stage in stages: stage_path_info = PathInfo(stage.path) - for out in outs: - if stage_path_info.isin(out.path_info): + if stage_path_info in outs: + raise StagePathAsOutputError(stage.wdir, stage.relpath) + for p in stage_path_info.parents: + if p in outs: raise StagePathAsOutputError(stage.wdir, stage.relpath) for stage in stages: @@ -333,21 +335,19 @@ class Repo(object): G_active.add_node(node, stage=stage) for dep in stage.deps: - for out in outs: + for out in outs.keys(): if ( - out.path_info != dep.path_info - and not dep.path_info.isin(out.path_info) - and not out.path_info.isin(dep.path_info) + out == dep.path_info + or dep.path_info.isin(out) + or out.isin(dep.path_info) ): - continue - - dep_stage = out.stage - dep_node = relpath(dep_stage.path, self.root_dir) - G.add_node(dep_node, stage=dep_stage) - G.add_edge(node, dep_node) - if not stage.locked: - G_active.add_node(dep_node, stage=dep_stage) - G_active.add_edge(node, dep_node) + dep_stage = outs.get(out).stage + dep_node = relpath(dep_stage.path, self.root_dir) + G.add_node(dep_node, stage=dep_stage) + G.add_edge(node, dep_node) + if not stage.locked: + G_active.add_node(dep_node, stage=dep_stage) + G_active.add_edge(node, dep_node) self._check_cyclic_graph(G) From ba57906b07f305757649384f72674129f6db401a Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 4 Jul 2019 15:04:55 -0700 Subject: [PATCH 3/8] deep source: address style issues --- dvc/repo/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index fe4c469a0..7f3fc2888 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -300,7 +300,7 @@ class Repo(object): G_active = nx.DiGraph() stages = stages or self.stages(from_directory, check_dag=False) stages = [stage for stage in stages if stage] - outs = dict() + outs = {} for stage in stages: for out in stage.outs: @@ -335,7 +335,7 @@ class Repo(object): G_active.add_node(node, stage=stage) for dep in stage.deps: - for out in outs.keys(): + for out in outs: if ( out == dep.path_info or dep.path_info.isin(out) From 9c87c9c6fe7da77bb3782c20670c058244214e19 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 4 Jul 2019 17:57:48 -0700 Subject: [PATCH 4/8] url info: fix comparators, add parents --- dvc/path_info.py | 64 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/dvc/path_info.py b/dvc/path_info.py index d16b452e4..12430f176 100644 --- a/dvc/path_info.py +++ b/dvc/path_info.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals, absolute_import import sys import os import posixpath +from collections import Sequence + from funcy import cached_property from dvc.utils.compat import str, builtin_str, basestring, is_py2 @@ -104,6 +106,27 @@ class PosixPathInfo(PathInfo, pathlib.PurePosixPath): pass +class _URLPathParents(Sequence): + def __init__(self, pathcls, scheme, netloc, path): + self._scheme = scheme + self._netloc = netloc + self._parents = path.parents + self._pathcls = pathcls + + def __len__(self): + return len(self._parents) + + def __getitem__(self, idx): + return self._pathcls.from_parts( + scheme=self._scheme, + netloc=self._netloc, + path=self._parents[idx].fspath, + ) + + def __repr__(self): + return "<{}.parents>".format(self._pathcls.__name__) + + class URLInfo(object): DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22} @@ -128,12 +151,7 @@ class URLInfo(object): @cached_property def url(self): p = self.parsed - netloc = p.hostname - if p.username: - netloc = p.username + "@" + netloc - if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme): - netloc += ":" + str(p.port) - return "{}://{}{}".format(p.scheme, netloc, p.path) + return "{}://{}{}".format(p.scheme, self.normalized_netloc, p.path) def __str__(self): return self.url @@ -144,7 +162,12 @@ class URLInfo(object): def __eq__(self, other): if isinstance(other, basestring): other = self.__class__(other) - return str(self) == str(other) + return ( + self.__class__ == other.__class__ + and self.scheme == other.scheme + and self.normalized_netloc == other.normalized_netloc + and self._path == other._path + ) def __hash__(self): return hash(self.url) @@ -162,6 +185,16 @@ class URLInfo(object): def __getattr__(self, name): return getattr(self.parsed, name) + @cached_property + def normalized_netloc(self): + p = self.parsed + netloc = p.hostname + if p.username: + netloc = p.username + "@" + netloc + if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme): + netloc += ":" + str(p.port) + return netloc + @property def port(self): return self.parsed.port or self.DEFAULT_PORTS.get(self.parsed.scheme) @@ -193,13 +226,24 @@ class URLInfo(object): @property def parent(self): return self.from_parts( - scheme=self.scheme, netloc=self.netloc, path=self._path.parent + scheme=self.scheme, + netloc=self.netloc, + path=self._path.parent.fspath, + ) + + @property + def parents(self): + return _URLPathParents( + type(self), self.scheme, self.netloc, self._path ) def relative_to(self, other): if isinstance(other, str): other = URLInfo(other) - if self.scheme != other.scheme or self.netloc != other.netloc: + if ( + self.scheme != other.scheme + or self.normalized_netloc != other.normalized_netloc + ): raise ValueError( "'{}' does not start with '{}'".format(self, other) ) @@ -212,7 +256,7 @@ class URLInfo(object): return False return ( self.scheme == other.scheme - and self.netloc == other.netloc + and self.normalized_netloc == other.normalized_netloc and self._path.isin(other._path) ) From 48d6cd9c2e462fc2e59decb6a02bae7d847e35ee Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 4 Jul 2019 19:04:46 -0700 Subject: [PATCH 5/8] path info: drop isin optimization, rename normalized netloc --- dvc/path_info.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/dvc/path_info.py b/dvc/path_info.py index 12430f176..c28477701 100644 --- a/dvc/path_info.py +++ b/dvc/path_info.py @@ -59,11 +59,7 @@ class PathInfo(pathlib.PurePath): other = self.__class__(other) elif self.__class__ != other.__class__: return False - for p in self.parents: - if p == other: - return True - if len(other.parts) >= len(p.parts): - return False + return any(p == other for p in self.parents) # pathlib2 uses bytes internally in Python 2, and we use unicode everywhere # for paths in both pythons, thus we need this glue. @@ -151,7 +147,7 @@ class URLInfo(object): @cached_property def url(self): p = self.parsed - return "{}://{}{}".format(p.scheme, self.normalized_netloc, p.path) + return "{}://{}{}".format(p.scheme, self.netloc, p.path) def __str__(self): return self.url @@ -165,7 +161,7 @@ class URLInfo(object): return ( self.__class__ == other.__class__ and self.scheme == other.scheme - and self.normalized_netloc == other.normalized_netloc + and self.netloc == other.netloc and self._path == other._path ) @@ -186,7 +182,7 @@ class URLInfo(object): return getattr(self.parsed, name) @cached_property - def normalized_netloc(self): + def netloc(self): p = self.parsed netloc = p.hostname if p.username: @@ -217,33 +213,30 @@ class URLInfo(object): @property def parts(self): - return (self.scheme, self.netloc) + self._path.parts + return (self.scheme, self.parsed.netloc) + self._path.parts @property def bucket(self): - return self.netloc + return self.parsed.netloc @property def parent(self): return self.from_parts( scheme=self.scheme, - netloc=self.netloc, + netloc=self.parsed.netloc, path=self._path.parent.fspath, ) @property def parents(self): return _URLPathParents( - type(self), self.scheme, self.netloc, self._path + type(self), self.scheme, self.parsed.netloc, self._path ) def relative_to(self, other): if isinstance(other, str): other = URLInfo(other) - if ( - self.scheme != other.scheme - or self.normalized_netloc != other.normalized_netloc - ): + if self.scheme != other.scheme or self.netloc != other.netloc: raise ValueError( "'{}' does not start with '{}'".format(self, other) ) @@ -256,7 +249,7 @@ class URLInfo(object): return False return ( self.scheme == other.scheme - and self.normalized_netloc == other.normalized_netloc + and self.netloc == other.netloc and self._path.isin(other._path) ) From 0abf530bef0b3cdbcf58571db4120393b10429f1 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 4 Jul 2019 19:34:43 -0700 Subject: [PATCH 6/8] tests: add tests for some previously failing calls in url info --- tests/unit/test_path_info.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/unit/test_path_info.py diff --git a/tests/unit/test_path_info.py b/tests/unit/test_path_info.py new file mode 100644 index 000000000..72479f9f1 --- /dev/null +++ b/tests/unit/test_path_info.py @@ -0,0 +1,30 @@ +from dvc.path_info import URLInfo + + +def test_url_info_str(): + u = URLInfo("ssh://user@test.com:/test1/") + assert u.url == "ssh://user@test.com/test1/" + assert str(u) == u.url + + +def test_url_info_eq(): + u1 = URLInfo("ssh://user@test.com:/test1/") + u2 = URLInfo("ssh://user@test.com/test1") + assert u1 == u2 + + +def test_url_info_parent(): + u1 = URLInfo("ssh://user@test.com:/test1/test2") + p = u1.parent + u3 = URLInfo("ssh://user@test.com/test1") + assert u3 == p + assert str(p) == "ssh://user@test.com/test1" + + +def test_url_info_parents(): + u1 = URLInfo("ssh://user@test.com:/test1/test2/test3") + parents = u1.parents + assert len(parents) == 3 + assert parents[0] == URLInfo("ssh://user@test.com/test1/test2") + assert parents[1] == URLInfo("ssh://user@test.com/test1") + assert parents[2] == URLInfo("ssh://user@test.com/") From 8fd18f5017b4ffa78c74de51d123cc4099584402 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 4 Jul 2019 23:35:58 -0700 Subject: [PATCH 7/8] url info and repo grap optimization: address review comments --- dvc/path_info.py | 2 +- dvc/repo/__init__.py | 17 ++++++---------- tests/unit/test_path_info.py | 38 +++++++++++++++++++++--------------- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/dvc/path_info.py b/dvc/path_info.py index c28477701..bd18b52bf 100644 --- a/dvc/path_info.py +++ b/dvc/path_info.py @@ -213,7 +213,7 @@ class URLInfo(object): @property def parts(self): - return (self.scheme, self.parsed.netloc) + self._path.parts + return (self.scheme, self.netloc) + self._path.parts @property def bucket(self): diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 7f3fc2888..f4ac80595 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals import os import logging +from itertools import chain + from dvc.config import Config from dvc.exceptions import ( NotDvcRepoError, @@ -305,10 +307,7 @@ class Repo(object): for stage in stages: for out in stage.outs: if out.path_info in outs: - stages = [ - stage.relpath, - outs.get(out.path_info).stage.relpath, - ] + stages = [stage.relpath, outs[out.path_info].stage.relpath] raise OutputDuplicationError(str(out), stages) outs[out.path_info] = out @@ -316,15 +315,11 @@ class Repo(object): for out in stage.outs: for p in out.path_info.parents: if p in outs: - raise OverlappingOutputPathsError( - outs.get(p), outs.get(out.path_info) - ) + raise OverlappingOutputPathsError(outs[p], out) for stage in stages: stage_path_info = PathInfo(stage.path) - if stage_path_info in outs: - raise StagePathAsOutputError(stage.wdir, stage.relpath) - for p in stage_path_info.parents: + for p in chain([stage_path_info], stage_path_info.parents): if p in outs: raise StagePathAsOutputError(stage.wdir, stage.relpath) @@ -341,7 +336,7 @@ class Repo(object): or dep.path_info.isin(out) or out.isin(dep.path_info) ): - dep_stage = outs.get(out).stage + dep_stage = outs[out].stage dep_node = relpath(dep_stage.path, self.root_dir) G.add_node(dep_node, stage=dep_stage) G.add_edge(node, dep_node) diff --git a/tests/unit/test_path_info.py b/tests/unit/test_path_info.py index 72479f9f1..46bd032b9 100644 --- a/tests/unit/test_path_info.py +++ b/tests/unit/test_path_info.py @@ -1,30 +1,36 @@ -from dvc.path_info import URLInfo +import pytest + +from dvc.path_info import URLInfo, CloudURLInfo -def test_url_info_str(): - u = URLInfo("ssh://user@test.com:/test1/") +@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo]) +def test_url_info_str(cls): + u = cls("ssh://user@test.com:/test1/") assert u.url == "ssh://user@test.com/test1/" assert str(u) == u.url -def test_url_info_eq(): - u1 = URLInfo("ssh://user@test.com:/test1/") - u2 = URLInfo("ssh://user@test.com/test1") +@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo]) +def test_url_info_eq(cls): + u1 = cls("ssh://user@test.com:/test1/") + u2 = cls("ssh://user@test.com/test1") assert u1 == u2 -def test_url_info_parent(): - u1 = URLInfo("ssh://user@test.com:/test1/test2") +@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo]) +def test_url_info_parent(cls): + u1 = cls("ssh://user@test.com:/test1/test2") p = u1.parent - u3 = URLInfo("ssh://user@test.com/test1") + u3 = cls("ssh://user@test.com/test1") assert u3 == p assert str(p) == "ssh://user@test.com/test1" -def test_url_info_parents(): - u1 = URLInfo("ssh://user@test.com:/test1/test2/test3") - parents = u1.parents - assert len(parents) == 3 - assert parents[0] == URLInfo("ssh://user@test.com/test1/test2") - assert parents[1] == URLInfo("ssh://user@test.com/test1") - assert parents[2] == URLInfo("ssh://user@test.com/") +@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo]) +def test_url_info_parents(cls): + u1 = cls("ssh://user@test.com:/test1/test2/test3") + assert list(u1.parents) == [ + cls("ssh://user@test.com/test1/test2"), + cls("ssh://user@test.com/test1"), + cls("ssh://user@test.com/"), + ] From bd79e9898b95bc975cb8981f3af91a5198cce942 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Fri, 5 Jul 2019 00:08:53 -0700 Subject: [PATCH 8/8] Sequence is deprecated in Python 3.8, remove subclassing --- dvc/path_info.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dvc/path_info.py b/dvc/path_info.py index bd18b52bf..2c81cdc82 100644 --- a/dvc/path_info.py +++ b/dvc/path_info.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals, absolute_import import sys import os import posixpath -from collections import Sequence from funcy import cached_property @@ -102,7 +101,7 @@ class PosixPathInfo(PathInfo, pathlib.PurePosixPath): pass -class _URLPathParents(Sequence): +class _URLPathParents(object): def __init__(self, pathcls, scheme, netloc, path): self._scheme = scheme self._netloc = netloc