mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-21 07:18:27 +01:00
* remote: remove unpacked dir We've done a lot of optimizations lately, which made unpacked dir trick obsoleted. * local: save us a stat call in is_protected * local: speedup checksum_to_path ~x5.5 * tree: walk: don't use os.path.join It is ~5.5 times slower than joining by hand. * dvcignore: optimize matching x10 On a repo with a dvcignore with 1 pattern and a directory with 400K files, `dvc status` now takes ~8 sec instead of ~30 sec. To achieve that, we make some assumptions about the paths formats that we are dealing with, so we could use simpler logic instead of using very slow `relpath`, `abspath` etc on every entry in a directory. It is also clear that CleanTree behavior is inconsistent (even tests expect very different outputs from it), so we will need to look into this later.
96 lines
2.5 KiB
Python
96 lines
2.5 KiB
Python
import os
|
|
import stat
|
|
|
|
|
|
class BaseTree:
|
|
"""Abstract class to represent access to files"""
|
|
|
|
@property
|
|
def tree_root(self):
|
|
pass
|
|
|
|
def open(self, path, mode="r", encoding="utf-8"):
|
|
"""Open file and return a stream."""
|
|
|
|
def exists(self, path):
|
|
"""Test whether a path exists."""
|
|
|
|
def isdir(self, path):
|
|
"""Return true if the pathname refers to an existing directory."""
|
|
|
|
def isfile(self, path):
|
|
"""Test whether a path is a regular file"""
|
|
|
|
def walk(self, top, topdown=True):
|
|
"""Directory tree generator.
|
|
|
|
See `os.walk` for the docs. Differences:
|
|
- no support for symlinks
|
|
- it could raise exceptions, there is no onerror argument
|
|
"""
|
|
|
|
def walk_files(self, top):
|
|
for root, _, files in self.walk(top):
|
|
for file in files:
|
|
# NOTE: os.path.join is ~5.5 times slower
|
|
yield f"{root}{os.sep}{file}"
|
|
|
|
|
|
class WorkingTree(BaseTree):
|
|
"""Proxies the repo file access methods to working tree files"""
|
|
|
|
def __init__(self, repo_root=os.getcwd()):
|
|
self.repo_root = repo_root
|
|
|
|
@property
|
|
def tree_root(self):
|
|
return self.repo_root
|
|
|
|
def open(self, path, mode="r", encoding="utf-8"):
|
|
"""Open file and return a stream."""
|
|
if "b" in mode:
|
|
encoding = None
|
|
return open(path, mode=mode, encoding=encoding)
|
|
|
|
def exists(self, path):
|
|
"""Test whether a path exists."""
|
|
return os.path.lexists(path)
|
|
|
|
def isdir(self, path):
|
|
"""Return true if the pathname refers to an existing directory."""
|
|
return os.path.isdir(path)
|
|
|
|
def isfile(self, path):
|
|
"""Test whether a path is a regular file"""
|
|
return os.path.isfile(path)
|
|
|
|
def walk(self, top, topdown=True):
|
|
"""Directory tree generator.
|
|
|
|
See `os.walk` for the docs. Differences:
|
|
- no support for symlinks
|
|
- it could raise exceptions, there is no onerror argument
|
|
"""
|
|
|
|
def onerror(e):
|
|
raise e
|
|
|
|
for root, dirs, files in os.walk(
|
|
top, topdown=topdown, onerror=onerror
|
|
):
|
|
yield os.path.normpath(root), dirs, files
|
|
|
|
def isexec(self, path):
|
|
mode = os.stat(path).st_mode
|
|
return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
|
|
|
@staticmethod
|
|
def stat(path):
|
|
return os.stat(path)
|
|
|
|
|
|
def is_working_tree(tree):
|
|
return isinstance(tree, WorkingTree) or isinstance(
|
|
getattr(tree, "tree", None), WorkingTree
|
|
)
|