Abstract access to file objects

Abstract access to filesystem and its implementation to access files and
particulary Stage objects in different git branches without `git
checkout` execution.

dvc/repo/__init__.py

@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str
-
 import os
 import dvc.prompt as prompt
 import dvc.logger as logger
@@ -29,6 +27,7 @@ class Repo(object):
     from dvc.repo.gc import gc
     from dvc.repo.commit import commit
     from dvc.repo.pkg import install_pkg
+    from dvc.repo.brancher import brancher
 
     def __init__(self, root_dir=None):
         from dvc.config import Config
@@ -39,6 +38,7 @@ class Repo(object):
         from dvc.data_cloud import DataCloud
         from dvc.updater import Updater
         from dvc.repo.metrics import Metrics
+        from dvc.repo.tree import WorkingTree
 
         root_dir = self.find_root(root_dir)
 
@@ -47,6 +47,8 @@ class Repo(object):
 
         self.config = Config(self.dvc_dir)
 
+        self.tree = WorkingTree()
+
         self.scm = SCM(self.root_dir, repo=self)
         self.lock = Lock(self.dvc_dir)
         # NOTE: storing state and link_state in the repository itself to avoid
@@ -133,7 +135,7 @@ class Repo(object):
             raise CyclicGraphError(cycles[0])
 
     def _get_pipeline(self, node):
-        pipelines = list(filter(lambda g: node in g.nodes(), self.pipelines()))
+        pipelines = [i for i in self.pipelines() if i.has_node(node)]
         assert len(pipelines) == 1
         return pipelines[0]
 
@@ -147,11 +149,10 @@ class Repo(object):
 
         node = os.path.relpath(stage.path, self.root_dir)
         G = self._get_pipeline(node)
-        stages = nx.get_node_attributes(G, "stage")
 
         ret = [stage]
         for n in nx.dfs_postorder_nodes(G, node):
-            ret.append(stages[n])
+            ret.append(G.node[n]["stage"])
 
         return ret
 
@@ -244,7 +245,7 @@ class Repo(object):
         cache["ssh"] = []
         cache["azure"] = []
 
-        for branch in self.scm.brancher(
+        for branch in self.brancher(
             all_branches=all_branches, all_tags=all_tags
         ):
             if target:
@@ -361,10 +362,10 @@ class Repo(object):
 
         stages = []
         outs = []
-        for root, dirs, files in os.walk(str(from_directory)):
+        for root, dirs, files in self.tree.walk(from_directory):
             for fname in files:
                 path = os.path.join(root, fname)
-                if not Stage.is_stage_file(path):
+                if not Stage.is_valid_filename(path):
                     continue
                 stage = Stage.load(self, path)
                 for out in stage.outs:
@@ -397,10 +398,14 @@ class Repo(object):
 
     def find_outs_by_path(self, path, outs=None, recursive=False):
         if not outs:
-            outs = [out for stage in self.stages() for out in stage.outs]
+            # there is no `from_directory=path` argument because some data
+            # files might be generated to an upper level, and so it is
+            # needed to look at all the files (project root_dir)
+            stages = self.stages()
+            outs = [out for stage in stages for out in stage.outs]
 
         abs_path = os.path.abspath(path)
-        is_dir = os.path.isdir(abs_path)
+        is_dir = self.tree.isdir(abs_path)
 
         def func(out):
             if out.path == abs_path:

dvc/repo/brancher.py

@@ -0,0 +1,56 @@
+def brancher(  # noqa: E302
+    self, branches=None, all_branches=False, tags=None, all_tags=False
+):
+    """Generator that iterates over specified revisions.
+
+    Args:
+        branches (list): a list of branches to iterate over.
+        all_branches (bool): iterate over all available branches.
+        tags (list): a list of tags to iterate over.
+        all_tags (bool): iterate over all available tags.
+
+    Yields:
+        str: the display name for the currently selected tree, it could be:
+            - a git revision identifier
+            - empty string it there is no branches to iterate over
+            - "Working Tree" if there are uncommited changes in the SCM repo
+    """
+    if not any([branches, all_branches, tags, all_tags]):
+        yield ""
+        return
+
+    saved_tree = self.tree
+    revs = []
+
+    scm = self.scm
+
+    if self.scm.is_dirty():
+        from dvc.repo.tree import WorkingTree
+
+        self.tree = WorkingTree()
+        yield "Working Tree"
+
+    if all_branches:
+        branches = scm.list_branches()
+
+    if all_tags:
+        tags = scm.list_tags()
+
+    if branches is None:
+        revs.extend([scm.active_branch()])
+    else:
+        revs.extend(branches)
+
+    if tags is not None:
+        revs.extend(tags)
+
+    # NOTE: it might be a good idea to wrap this loop in try/finally block
+    # to don't leave the tree on some unexpected branch after the
+    # `brancher()`, but this could cause problems on exception handling
+    # code which might expect the tree on which exception was raised to
+    # stay in place. This behavior is a subject to change.
+    for rev in revs:
+        self.tree = scm.get_tree(rev)
+        yield rev
+
+    self.tree = saved_tree

dvc/repo/init.py

@@ -4,7 +4,7 @@ import colorama
 
 import dvc.logger as logger
 from dvc.repo import Repo
-from dvc.scm import SCM, Base
+from dvc.scm import SCM, NoSCM
 from dvc.config import Config
 from dvc.exceptions import InitError
 
@@ -56,7 +56,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
     root_dir = os.path.abspath(root_dir)
     dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
     scm = SCM(root_dir)
-    if type(scm) == Base and not no_scm:
+    if isinstance(scm, NoSCM) and not no_scm:
         raise InitError(
             "{repo} is not tracked by any supported scm tool (e.g. git). "
             "Use '--no-scm' if you don't want to use any scm.".format(

dvc/repo/metrics/show.py

@@ -66,30 +66,23 @@ def _read_typed_metric(typ, xpath, fd):
     return ret
 
 
-def _read_metric(path, typ=None, xpath=None, branch=None):
-    ret = None
-    if not os.path.exists(path):
-        return ret
-
+def _read_metric(fd, typ=None, xpath=None, rel_path=None, branch=None):
     typ = typ.lower().strip() if typ else typ
-    xpath = xpath.strip() if xpath else xpath
     try:
-        with open(path, "r") as fd:
-            if not xpath:
-                ret = fd.read().strip()
-            else:
-                ret = _read_typed_metric(typ, xpath, fd)
+        if xpath:
+            return _read_typed_metric(typ, xpath.strip(), fd)
+        else:
+            return fd.read().strip()
     # Json path library has to be replaced or wrapped in
     # order to fix this too broad except clause.
     except Exception:
         logger.warning(
             "unable to read metric in '{}' in branch '{}'".format(
-                path, branch
+                rel_path, branch
             ),
             parse_exception=True,
         )
-
-    return ret
+        return None
 
 
 def _collect_metrics(self, path, recursive, typ, xpath, branch):
@@ -123,16 +116,33 @@ def _collect_metrics(self, path, recursive, typ, xpath, branch):
     return res
 
 
+def _read_metrics_filesystem(path, typ, xpath, rel_path, branch):
+    if not os.path.exists(path):
+        return None
+    with open(path, "r") as fd:
+        return _read_metric(
+            fd, typ=typ, xpath=xpath, rel_path=rel_path, branch=branch
+        )
+
+
 def _read_metrics(self, metrics, branch):
     res = {}
     for out, typ, xpath in metrics:
         assert out.scheme == "local"
         if out.use_cache:
-            path = self.cache.local.get(out.checksum)
+            metric = _read_metrics_filesystem(
+                self.cache.local.get(out.checksum),
+                typ=typ,
+                xpath=xpath,
+                rel_path=out.rel_path,
+                branch=branch,
+            )
         else:
-            path = out.path
+            fd = self.tree.open(out.path)
+            metric = _read_metric(
+                fd, typ=typ, xpath=xpath, rel_path=out.rel_path, branch=branch
+            )
 
-        metric = _read_metric(path, typ=typ, xpath=xpath, branch=branch)
         if not metric:
             continue
 
@@ -151,9 +161,8 @@ def show(
     recursive=False,
 ):
     res = {}
-    for branch in self.scm.brancher(
-        all_branches=all_branches, all_tags=all_tags
-    ):
+
+    for branch in self.brancher(all_branches=all_branches, all_tags=all_tags):
         entries = _collect_metrics(self, path, recursive, typ, xpath, branch)
         metrics = _read_metrics(self, entries, branch)
         if metrics:

dvc/repo/tree.py

@@ -0,0 +1,65 @@
+import os
+
+from dvc.utils.compat import open
+
+
+class BaseTree(object):
+    """Abstract class to represent access to files"""
+
+    def open(self, path):
+        """Open file and return a stream."""
+
+    def exists(self, path):
+        """Test whether a path exists."""
+
+    def isdir(self, path):
+        """Return true if the pathname refers to an existing directory."""
+
+    def isfile(self, path):
+        """Test whether a path is a regular file"""
+
+    def walk(self, top, topdown=True):
+        """Directory tree generator.
+
+        See `os.walk` for the docs. Differences:
+        - no support for symlinks
+        - it could raise exceptions, there is no onerror argument
+        """
+
+
+class WorkingTree(BaseTree):
+    """Proxies the repo file access methods to working tree files"""
+
+    def open(self, path):
+        """Open file and return a stream."""
+        return open(path, encoding="utf-8")
+
+    def exists(self, path):
+        """Test whether a path exists."""
+        return os.path.exists(path)
+
+    def isdir(self, path):
+        """Return true if the pathname refers to an existing directory."""
+        return os.path.isdir(path)
+
+    def isfile(self, path):
+        """Test whether a path is a regular file"""
+        return os.path.isfile(path)
+
+    def walk(self, top, topdown=True):
+        """Directory tree generator.
+
+        See `os.walk` for the docs. Differences:
+        - no support for symlinks
+        - it could raise exceptions, there is no onerror argument
+        """
+
+        def onerror(e):
+            raise e
+
+        for root, dirs, files in os.walk(
+            top, topdown=topdown, onerror=onerror
+        ):
+            if topdown:
+                dirs[:] = [i for i in dirs if i not in (".git", ".hg", ".dvc")]
+            yield os.path.normpath(root), dirs, files

dvc/scm/__init__.py

@@ -8,6 +8,12 @@ from dvc.scm.git import Git
 import dvc
 
 
+# just a sugar to point that this is an actual implementation for a dvc
+# project under no SCM control
+class NoSCM(Base):
+    pass
+
+
 def SCM(root_dir, repo=None):  # pylint: disable=invalid-name
     """Returns SCM instance that corresponds to a repo at the specified
     path.
@@ -22,7 +28,7 @@ def SCM(root_dir, repo=None):  # pylint: disable=invalid-name
     if Git.is_repo(root_dir) or Git.is_submodule(root_dir):
         return Git(root_dir, repo=repo)
 
-    return Base(root_dir, repo=repo)
+    return NoSCM(root_dir, repo=repo)
 
 
 def scm_context(method):

dvc/scm/base.py

@@ -81,47 +81,6 @@ class Base(object):
     def tag(self, tag):
         """Makes SCM create a tag with a specified name."""
 
-    def brancher(
-        self, branches=None, all_branches=False, tags=None, all_tags=False
-    ):
-        """Generator that iterates over specified revisions.
-
-        Args:
-            branches (list): a list of branches to iterate over.
-            all_branches (bool): iterate over all available branches.
-            tags (list): a list of tags to iterate over.
-            all_tags (bool): iterate over all available tags.
-
-        Yields:
-            str: the current revision.
-        """
-        if not branches and not all_branches and not tags and not all_tags:
-            yield ""
-            return
-
-        saved = self.active_branch()
-        revs = []
-
-        if all_branches:
-            branches = self.list_branches()
-
-        if all_tags:
-            tags = self.list_tags()
-
-        if branches is None:
-            revs.extend([saved])
-        else:
-            revs.extend(branches)
-
-        if tags is not None:
-            revs.extend(tags)
-
-        for rev in revs:
-            self.checkout(rev)
-            yield rev
-
-        self.checkout(saved)
-
     def untracked_files(self):  # pylint: disable=no-self-use
         """Returns a list of untracked files."""
         return []
@@ -130,6 +89,10 @@ class Base(object):
         """Returns whether or not a specified path is tracked."""
         return False
 
+    def is_dirty(self):
+        """Return whether the SCM contains uncommited changes."""
+        return False
+
     def active_branch(self):  # pylint: disable=no-self-use
         """Returns current branch in the repo."""
         return ""

dvc/scm/git/__init__.py

@@ -2,11 +2,9 @@
 
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, open
-
 import os
 
-import dvc.logger as logger
+from dvc.utils.compat import str, open
 from dvc.utils import fix_env
 from dvc.scm.base import (
     Base,
@@ -14,6 +12,8 @@ from dvc.scm.base import (
     FileNotInRepoError,
     FileNotInTargetSubdirError,
 )
+from dvc.scm.git.tree import GitTree
+import dvc.logger as logger
 
 
 class Git(Base):
@@ -122,7 +122,7 @@ class Git(Base):
         content = entry
         if ignore_list:
             content = "\n" + content
-        with open(gitignore, "a") as fobj:
+        with open(gitignore, "a", encoding="utf-8") as fobj:
             fobj.write(content)
 
     def ignore_remove(self, path):
@@ -176,7 +176,13 @@ class Git(Base):
         return [os.path.join(self.git.working_dir, fname) for fname in files]
 
     def is_tracked(self, path):
-        return len(self.git.git.ls_files(path)) != 0
+        # it is equivalent to `bool(self.git.git.ls_files(path))` by
+        # functionality, but ls_files fails on unicode filenames
+        path = os.path.relpath(path, self.root_dir)
+        return path in [i[0] for i in self.git.index.entries]
+
+    def is_dirty(self):
+        return self.git.is_dirty()
 
     def active_branch(self):
         return self.git.active_branch.name
@@ -226,3 +232,6 @@ class Git(Base):
         basename = os.path.basename(path)
         path_parts = os.path.normpath(path).split(os.path.sep)
         return basename == self.ignore_file or Git.GIT_DIR in path_parts
+
+    def get_tree(self, rev):
+        return GitTree(self.git, rev)

dvc/scm/git/tree.py

@@ -0,0 +1,114 @@
+import errno
+import os
+
+from dvc.utils.compat import StringIO
+
+from dvc.repo.tree import BaseTree
+
+
+# see git-fast-import(1)
+GIT_MODE_DIR = 0o40000
+GIT_MODE_FILE = 0o644
+
+
+class GitTree(BaseTree):
+    """Proxies the repo file access methods to Git objects"""
+
+    def __init__(self, git, rev):
+        """Create GitTree instance
+
+        Args:
+            git (dvc.scm.Git):
+            branch:
+        """
+        self.git = git
+        self.rev = rev
+
+    def open(self, path):
+
+        relpath = os.path.relpath(path, self.git.working_dir)
+
+        obj = self.git_object_by_path(path)
+        if obj is None:
+            msg = "No such file in branch '{}'".format(self.rev)
+            raise IOError(errno.ENOENT, msg, relpath)
+        if obj.mode == GIT_MODE_DIR:
+            raise IOError(errno.EISDIR, "Is a directory", relpath)
+
+        # GitPython's obj.data_stream is a fragile thing, it is better to
+        # read it immediately, also it needs to be to decoded if we follow
+        # the `open()` behavior (since data_stream.read() returns bytes,
+        # and `open` with default "r" mode returns str)
+        return StringIO(obj.data_stream.read().decode("utf-8"))
+
+    def exists(self, path):
+        return self.git_object_by_path(path) is not None
+
+    def isdir(self, path):
+        obj = self.git_object_by_path(path)
+        if obj is None:
+            return False
+        return obj.mode == GIT_MODE_DIR
+
+    def isfile(self, path):
+        obj = self.git_object_by_path(path)
+        if obj is None:
+            return False
+        # according to git-fast-import(1) file mode could be 644 or 755
+        return obj.mode & GIT_MODE_FILE == GIT_MODE_FILE
+
+    @staticmethod
+    def _is_tree_and_contains(obj, path):
+        if obj.mode != GIT_MODE_DIR:
+            return False
+        # see https://github.com/gitpython-developers/GitPython/issues/851
+        # `return (i in tree)` doesn't work so here is a workaround:
+        for i in obj:
+            if i.name == path:
+                return True
+        return False
+
+    def git_object_by_path(self, path):
+        path = os.path.relpath(os.path.realpath(path), self.git.working_dir)
+        assert path.split(os.sep, 1)[0] != ".."
+        tree = self.git.tree(self.rev)
+        if not path or path == ".":
+            return tree
+        for i in path.split(os.sep):
+            if not self._is_tree_and_contains(tree, i):
+                # there is no tree for specified path
+                return None
+            tree = tree[i]
+        return tree
+
+    def _walk(self, tree, topdown=True):
+
+        dirs, nondirs = [], []
+        for i in tree:
+            if i.mode == GIT_MODE_DIR:
+                dirs.append(i.name)
+            else:
+                nondirs.append(i.name)
+
+        if topdown:
+            yield os.path.normpath(tree.path), dirs, nondirs
+        for i in dirs:
+            for x in self._walk(tree[i], topdown=True):
+                yield x
+        if not topdown:
+            yield os.path.normpath(tree.path), dirs, nondirs
+
+    def walk(self, top, topdown=True):
+        """Directory tree generator.
+
+        See `os.walk` for the docs. Differences:
+        - no support for symlinks
+        - it could raise exceptions, there is no onerror argument
+        """
+
+        tree = self.git_object_by_path(top)
+        if tree is None:
+            raise IOError(errno.ENOENT, "No such file")
+
+        for x in self._walk(tree, topdown):
+            yield x

dvc/stage.py

@@ -14,7 +14,7 @@ import dvc.logger as logger
 import dvc.dependency as dependency
 import dvc.output as output
 from dvc.exceptions import DvcException
-from dvc.utils import dict_md5, fix_env, load_stage_file
+from dvc.utils import dict_md5, fix_env, load_stage_file_fobj
 
 
 class StageCmdFailedError(DvcException):
@@ -171,7 +171,10 @@ class Stage(object):
     @staticmethod
     def is_valid_filename(path):
         return (
-            path.endswith(Stage.STAGE_FILE_SUFFIX)
+            # path.endswith doesn't work for encoded unicode filenames on
+            # Python 2 and since Stage.STAGE_FILE_SUFFIX is ascii then it is
+            # not needed to decode the path from py2's str
+            path[-len(Stage.STAGE_FILE_SUFFIX) :] == Stage.STAGE_FILE_SUFFIX
             or os.path.basename(path) == Stage.STAGE_FILE
         )
 
@@ -488,19 +491,27 @@ class Stage(object):
             )
 
     @staticmethod
-    def _check_file_exists(fname):
-        if not os.path.exists(fname):
+    def _check_file_exists(repo, fname):
+        if not repo.tree.exists(fname):
             raise StageFileDoesNotExistError(fname)
 
     @staticmethod
-    def load(repo, fname):
-        Stage._check_file_exists(fname)
-        Stage._check_dvc_filename(fname)
-
-        if not Stage.is_stage_file(fname):
+    def _check_isfile(repo, fname):
+        if not repo.tree.isfile(fname):
             raise StageFileIsNotDvcFileError(fname)
 
-        d = load_stage_file(fname)
+    @staticmethod
+    def load(repo, fname):
+
+        # it raises the proper exceptions by priority:
+        # 1. when the file doesn't exists
+        # 2. filename is not a dvc filename
+        # 3. path doesn't represent a regular file
+        Stage._check_file_exists(repo, fname)
+        Stage._check_dvc_filename(fname)
+        Stage._check_isfile(repo, fname)
+
+        d = load_stage_file_fobj(repo.tree.open(fname), fname)
 
         Stage.validate(d, fname=os.path.relpath(fname))
         path = os.path.abspath(fname)

dvc/utils/__init__.py

@@ -232,13 +232,17 @@ def current_timestamp():
 
 
 def load_stage_file(path):
+    with open(path, "r") as fobj:
+        return load_stage_file_fobj(fobj, path)
+
+
+def load_stage_file_fobj(fobj, path):
     from dvc.exceptions import StageFileCorruptedError
 
-    with open(path, "r") as fobj:
-        try:
-            return yaml.safe_load(fobj) or {}
-        except ScannerError:
-            raise StageFileCorruptedError(path)
+    try:
+        return yaml.safe_load(fobj) or {}
+    except ScannerError:
+        raise StageFileCorruptedError(path)
 
 
 def walk_files(directory):

tests/basic_env.py

@@ -1,3 +1,7 @@
+# encoding: utf-8
+
+from __future__ import unicode_literals
+
 import os
 import shutil
 import uuid
@@ -10,6 +14,7 @@ from unittest import TestCase
 import dvc.logger as logger
 from dvc.command.remote import CmdRemoteAdd
 from dvc.repo import Repo as DvcRepo
+from dvc.utils.compat import open, str
 
 
 class TestDirFixture(object):
@@ -40,6 +45,8 @@ class TestDirFixture(object):
         "import sys\nimport shutil\n"
         "shutil.copyfile(sys.argv[1], sys.argv[2])"
     )
+    UNICODE = "тест"
+    UNICODE_CONTENTS = "проверка"
 
     def __init__(self, root_dir=None):
         if root_dir:
@@ -61,8 +68,12 @@ class TestDirFixture(object):
         if len(dname) > 0 and not os.path.isdir(dname):
             os.makedirs(dname)
 
-        with open(name, "a") as f:
-            f.write(contents)
+        with open(name, "a", encoding="utf-8") as f:
+            f.write(
+                contents
+                if isinstance(contents, str)
+                else contents.decode("utf-8")
+            )
 
     @staticmethod
     def mkdtemp():
@@ -79,6 +90,7 @@ class TestDirFixture(object):
         os.mkdir(self.DATA_SUB_DIR)
         self.create(self.DATA, self.DATA_CONTENTS)
         self.create(self.DATA_SUB, self.DATA_SUB_CONTENTS)
+        self.create(self.UNICODE, self.UNICODE_CONTENTS)
 
     def tearDown(self):
         self._popd()
@@ -126,6 +138,7 @@ class TestDvcFixture(TestGitFixture):
     def setUp(self):
         super(TestDvcFixture, self).setUp()
         self.dvc = DvcRepo.init(self._root_dir)
+        self.dvc.scm.commit("init dvc")
         logger.be_verbose()
 
 

tests/test_repo.py

@@ -0,0 +1,46 @@
+from tests.basic_env import TestDvc
+
+from dvc.scm.git import GitTree
+from dvc.repo.tree import WorkingTree
+
+
+class TestCollect(TestDvc):
+    def setUp(self):
+        super(TestCollect, self).setUp()
+        self.dvc.add(self.FOO)
+        self.dvc.run(
+            deps=[self.FOO],
+            outs=[self.BAR],
+            cmd="python code.py {} {}".format(self.FOO, self.BAR),
+        )
+        self.dvc.scm.add([".gitignore", self.FOO + ".dvc", self.BAR + ".dvc"])
+        self.dvc.scm.commit("foo.dvc and bar.dvc")
+        self.dvc.scm.checkout("new_branch", True)
+        self.dvc.run(
+            deps=[self.BAR],
+            outs=["buzz"],
+            cmd="python code.py {} {}".format(self.BAR, "buzz"),
+        )
+        self.dvc.scm.add([".gitignore", "buzz.dvc"])
+        self.dvc.scm.commit("add buzz")
+        self.dvc.scm.checkout("master")
+
+    def _check(self, branch, target, with_deps, expected):
+        if branch:
+            self.dvc.tree = GitTree(self.dvc.scm.git, branch)
+        else:
+            self.dvc.tree = WorkingTree()
+        result = self.dvc.collect(target + ".dvc", with_deps=with_deps)
+        self.assertEqual(
+            [[j.rel_path for j in i.outs] for i in result], expected
+        )
+        return result
+
+    def test(self):
+        self._check("", self.BAR, True, [[self.FOO], [self.BAR]])
+        self._check("master", self.BAR, True, [[self.FOO], [self.BAR]])
+        self._check(
+            "new_branch", "buzz", True, [[self.FOO], [self.BAR], ["buzz"]]
+        )
+        result = self._check("new_branch", "buzz", False, [["buzz"]])
+        self.assertEqual([i.rel_path for i in result[0].deps], ["bar"])

tests/test_scm.py

@@ -1,10 +1,11 @@
 from __future__ import unicode_literals
 
 import os
+
 from git import Repo
 
 from dvc.utils.compat import str  # noqa: F401
-from dvc.scm import SCM, Base, Git
+from dvc.scm import SCM, NoSCM, Git
 from dvc.scm.base import FileNotInTargetSubdirError
 
 from tests.basic_env import TestDir, TestGit, TestGitSubmodule
@@ -13,7 +14,7 @@ from tests.utils import get_gitignore_content
 
 class TestSCM(TestDir):
     def test_none(self):
-        self.assertIsInstance(SCM(self._root_dir), Base)
+        self.assertIsInstance(SCM(self._root_dir), NoSCM)
 
     def test_git(self):
         Repo.init(os.curdir)
@@ -28,7 +29,22 @@ class TestSCMGit(TestGit):
         G = Git(self._root_dir)
         G.add(["foo"])
         G.commit("add")
-        self.assertTrue("foo" in self.git.git.ls_files())
+        self.assertIn("foo", self.git.git.ls_files())
+
+    def test_is_tracked(self):
+        foo = os.path.abspath(self.FOO)
+        G = Git(self._root_dir)
+        G.add([self.FOO, self.UNICODE])
+        self.assertTrue(G.is_tracked(foo))
+        self.assertTrue(G.is_tracked(self.FOO))
+        self.assertTrue(G.is_tracked(self.UNICODE))
+        G.commit("add")
+        self.assertTrue(G.is_tracked(foo))
+        self.assertTrue(G.is_tracked(self.FOO))
+        G.git.index.remove([self.FOO], working_tree=True)
+        self.assertFalse(G.is_tracked(foo))
+        self.assertFalse(G.is_tracked(self.FOO))
+        self.assertFalse(G.is_tracked("not-existing-file"))
 
 
 class TestSCMGitSubmodule(TestGitSubmodule):

tests/test_tree.py

@@ -0,0 +1,150 @@
+# encoding: utf-8
+
+from __future__ import unicode_literals
+
+from os.path import join
+
+from dvc.scm import SCM
+from dvc.scm.git import GitTree
+from dvc.repo.tree import WorkingTree
+
+from tests.basic_env import TestDir, TestGit
+
+
+class TestWorkingTree(TestDir):
+    def setUp(self):
+        super(TestWorkingTree, self).setUp()
+        self.tree = WorkingTree()
+
+    def test_open(self):
+        self.assertEqual(self.tree.open(self.FOO).read(), self.FOO_CONTENTS)
+        self.assertEqual(
+            self.tree.open(self.UNICODE).read(), self.UNICODE_CONTENTS
+        )
+
+    def test_exists(self):
+        self.assertTrue(self.tree.exists(self.FOO))
+        self.assertTrue(self.tree.exists(self.UNICODE))
+        self.assertFalse(self.tree.exists("not-existing-file"))
+
+    def test_isdir(self):
+        self.assertTrue(self.tree.isdir(self.DATA_DIR))
+        self.assertFalse(self.tree.isdir(self.FOO))
+        self.assertFalse(self.tree.isdir("not-existing-file"))
+
+    def test_isfile(self):
+        self.assertTrue(self.tree.isfile(self.FOO))
+        self.assertFalse(self.tree.isfile(self.DATA_DIR))
+        self.assertFalse(self.tree.isfile("not-existing-file"))
+
+
+class TestGitTree(TestGit):
+    def setUp(self):
+        super(TestGitTree, self).setUp()
+        self.scm = SCM(self._root_dir)
+        self.tree = GitTree(self.git, "master")
+
+    def test_open(self):
+        self.scm.add([self.FOO, self.UNICODE, self.DATA_DIR])
+        self.scm.commit("add")
+        self.assertEqual(self.tree.open(self.FOO).read(), self.FOO_CONTENTS)
+        self.assertEqual(
+            self.tree.open(self.UNICODE).read(), self.UNICODE_CONTENTS
+        )
+        with self.assertRaises(IOError):
+            self.tree.open("not-existing-file")
+        with self.assertRaises(IOError):
+            self.tree.open(self.DATA_DIR)
+
+    def test_exists(self):
+        self.assertFalse(self.tree.exists(self.FOO))
+        self.assertFalse(self.tree.exists(self.UNICODE))
+        self.assertFalse(self.tree.exists(self.DATA_DIR))
+        self.scm.add([self.FOO, self.UNICODE, self.DATA])
+        self.scm.commit("add")
+        self.assertTrue(self.tree.exists(self.FOO))
+        self.assertTrue(self.tree.exists(self.UNICODE))
+        self.assertTrue(self.tree.exists(self.DATA_DIR))
+        self.assertFalse(self.tree.exists("non-existing-file"))
+
+    def test_isdir(self):
+        self.scm.add([self.FOO, self.DATA_DIR])
+        self.scm.commit("add")
+        self.assertTrue(self.tree.isdir(self.DATA_DIR))
+        self.assertFalse(self.tree.isdir(self.FOO))
+        self.assertFalse(self.tree.isdir("non-existing-file"))
+
+    def test_isfile(self):
+        self.scm.add([self.FOO, self.DATA_DIR])
+        self.scm.commit("add")
+        self.assertTrue(self.tree.isfile(self.FOO))
+        self.assertFalse(self.tree.isfile(self.DATA_DIR))
+        self.assertFalse(self.tree.isfile("not-existing-file"))
+
+
+class AssertWalkEqualMixin(object):
+    def assertWalkEqual(self, actual, expected, msg=None):
+        def convert_to_sets(walk_results):
+            return [
+                (root, set(dirs), set(nondirs))
+                for root, dirs, nondirs in walk_results
+            ]
+
+        self.assertEqual(
+            convert_to_sets(actual), convert_to_sets(expected), msg=msg
+        )
+
+
+class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir):
+    def test(self):
+        tree = WorkingTree()
+        self.assertWalkEqual(
+            tree.walk("."),
+            [
+                (".", ["data_dir"], ["code.py", "bar", "тест", "foo"]),
+                (join("data_dir"), ["data_sub_dir"], ["data"]),
+                (join("data_dir", "data_sub_dir"), [], ["data_sub"]),
+            ],
+        )
+
+    def test_subdir(self):
+        tree = WorkingTree()
+        self.assertWalkEqual(
+            tree.walk(join("data_dir", "data_sub_dir")),
+            [(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
+        )
+
+
+class TestWalkInGit(AssertWalkEqualMixin, TestGit):
+    def test_nobranch(self):
+        tree = WorkingTree()
+        self.assertWalkEqual(
+            tree.walk("."),
+            [
+                (".", ["data_dir"], ["bar", "тест", "code.py", "foo"]),
+                ("data_dir", ["data_sub_dir"], ["data"]),
+                (join("data_dir", "data_sub_dir"), [], ["data_sub"]),
+            ],
+        )
+        self.assertWalkEqual(
+            tree.walk(join("data_dir", "data_sub_dir")),
+            [(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
+        )
+
+    def test_branch(self):
+        scm = SCM(self._root_dir)
+        scm.add([self.DATA_SUB_DIR])
+        scm.commit("add data_dir/data_sub_dir/data_sub")
+        tree = GitTree(self.git, "master")
+        self.assertWalkEqual(
+            tree.walk("."),
+            [
+                (".", ["data_dir"], ["code.py"]),
+                ("data_dir", ["data_sub_dir"], []),
+                (join("data_dir", "data_sub_dir"), [], ["data_sub"]),
+            ],
+        )
+        self.assertWalkEqual(
+            tree.walk(join("data_dir", "data_sub_dir")),
+            [(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
+        )

tests/unit/scm/test_scm.py

@@ -1,5 +1,5 @@
 from dvc.repo import Repo
-from dvc.scm import scm_context, Base
+from dvc.scm import scm_context, NoSCM
 from unittest import TestCase
 import mock
 
@@ -7,7 +7,7 @@ import mock
 class TestScmContext(TestCase):
     def setUp(self):
         self.repo_mock = mock.Mock(spec=Repo)
-        self.scm_mock = mock.Mock(spec=Base)
+        self.scm_mock = mock.Mock(spec=NoSCM)
         self.repo_mock.scm = self.scm_mock
 
     def test_should_successfully_perform_method(self):