Files
dvc/dvc/scm/tree.py
T
Ruslan Kuprieiev d77af8196e optimize local status (#3867)
* remote: remove unpacked dir

We've done a lot of optimizations lately, which made unpacked dir trick
obsoleted.

* local: save us a stat call in is_protected

* local: speedup checksum_to_path ~x5.5

* tree: walk: don't use os.path.join

It is ~5.5 times slower than joining by hand.

* dvcignore: optimize matching x10

On a repo with a dvcignore with 1 pattern and a directory with 400K
files, `dvc status` now takes ~8 sec instead of ~30 sec.

To achieve that, we make some assumptions about the paths formats that
we are dealing with, so we could use simpler logic instead of using
very slow `relpath`, `abspath` etc on every entry in a directory.

It is also clear that CleanTree behavior is inconsistent (even tests
expect very different outputs from it), so we will need to look into
this later.
2020-05-25 03:20:11 +03:00

96 lines
2.5 KiB
Python

import os
import stat
class BaseTree:
"""Abstract class to represent access to files"""
@property
def tree_root(self):
pass
def open(self, path, mode="r", encoding="utf-8"):
"""Open file and return a stream."""
def exists(self, path):
"""Test whether a path exists."""
def isdir(self, path):
"""Return true if the pathname refers to an existing directory."""
def isfile(self, path):
"""Test whether a path is a regular file"""
def walk(self, top, topdown=True):
"""Directory tree generator.
See `os.walk` for the docs. Differences:
- no support for symlinks
- it could raise exceptions, there is no onerror argument
"""
def walk_files(self, top):
for root, _, files in self.walk(top):
for file in files:
# NOTE: os.path.join is ~5.5 times slower
yield f"{root}{os.sep}{file}"
class WorkingTree(BaseTree):
"""Proxies the repo file access methods to working tree files"""
def __init__(self, repo_root=os.getcwd()):
self.repo_root = repo_root
@property
def tree_root(self):
return self.repo_root
def open(self, path, mode="r", encoding="utf-8"):
"""Open file and return a stream."""
if "b" in mode:
encoding = None
return open(path, mode=mode, encoding=encoding)
def exists(self, path):
"""Test whether a path exists."""
return os.path.lexists(path)
def isdir(self, path):
"""Return true if the pathname refers to an existing directory."""
return os.path.isdir(path)
def isfile(self, path):
"""Test whether a path is a regular file"""
return os.path.isfile(path)
def walk(self, top, topdown=True):
"""Directory tree generator.
See `os.walk` for the docs. Differences:
- no support for symlinks
- it could raise exceptions, there is no onerror argument
"""
def onerror(e):
raise e
for root, dirs, files in os.walk(
top, topdown=topdown, onerror=onerror
):
yield os.path.normpath(root), dirs, files
def isexec(self, path):
mode = os.stat(path).st_mode
return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
@staticmethod
def stat(path):
return os.stat(path)
def is_working_tree(tree):
return isinstance(tree, WorkingTree) or isinstance(
getattr(tree, "tree", None), WorkingTree
)