mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
Ruslan Kuprieiev
d77af8196e
* remote: remove unpacked dir We've done a lot of optimizations lately, which made unpacked dir trick obsoleted. * local: save us a stat call in is_protected * local: speedup checksum_to_path ~x5.5 * tree: walk: don't use os.path.join It is ~5.5 times slower than joining by hand. * dvcignore: optimize matching x10 On a repo with a dvcignore with 1 pattern and a directory with 400K files, `dvc status` now takes ~8 sec instead of ~30 sec. To achieve that, we make some assumptions about the paths formats that we are dealing with, so we could use simpler logic instead of using very slow `relpath`, `abspath` etc on every entry in a directory. It is also clear that CleanTree behavior is inconsistent (even tests expect very different outputs from it), so we will need to look into this later.
102 lines
2.8 KiB
Python
102 lines
2.8 KiB
Python
import os
|
|
|
|
import pytest
|
|
from mock import MagicMock, mock_open, patch
|
|
|
|
from dvc.ignore import DvcIgnoreDirs, DvcIgnorePatterns
|
|
|
|
|
|
def mock_dvcignore(dvcignore_path, patterns):
|
|
tree = MagicMock()
|
|
with patch.object(tree, "open", mock_open(read_data="\n".join(patterns))):
|
|
ignore_patterns = DvcIgnorePatterns(dvcignore_path, tree)
|
|
|
|
return ignore_patterns
|
|
|
|
|
|
def test_ignore_from_file_should_filter_dirs_and_files():
|
|
dvcignore_path = os.path.join(
|
|
os.path.sep, "full", "path", "to", "ignore", "file", ".dvcignore"
|
|
)
|
|
|
|
patterns = ["dir_to_ignore", "file_to_ignore"]
|
|
|
|
root = os.path.dirname(dvcignore_path)
|
|
dirs = ["dir1", "dir2", "dir_to_ignore"]
|
|
files = ["file1", "file2", "file_to_ignore"]
|
|
|
|
ignore = mock_dvcignore(dvcignore_path, patterns)
|
|
new_dirs, new_files = ignore(root, dirs, files)
|
|
|
|
assert {"dir1", "dir2"} == set(new_dirs)
|
|
assert {"file1", "file2"} == set(new_files)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"file_to_ignore_relpath, patterns, expected_match",
|
|
[
|
|
("to_ignore", ["to_ignore"], True),
|
|
("to_ignore.txt", ["to_ignore*"], True),
|
|
(
|
|
os.path.join("rel", "p", "p2", "to_ignore"),
|
|
["rel/**/to_ignore"],
|
|
True,
|
|
),
|
|
(
|
|
os.path.join(
|
|
os.path.sep,
|
|
"full",
|
|
"path",
|
|
"to",
|
|
"ignore",
|
|
"file",
|
|
"to_ignore",
|
|
),
|
|
["to_ignore"],
|
|
True,
|
|
),
|
|
("to_ignore.txt", ["/*.txt"], True),
|
|
(
|
|
os.path.join("rel", "path", "path2", "to_ignore"),
|
|
["rel/*/to_ignore"],
|
|
False,
|
|
),
|
|
(os.path.join("path", "to_ignore.txt"), ["/*.txt"], False),
|
|
(
|
|
os.path.join("rel", "path", "path2", "dont_ignore"),
|
|
["rel/**/to_ignore"],
|
|
False,
|
|
),
|
|
("dont_ignore.txt", ["dont_ignore"], False),
|
|
("dont_ignore.txt", ["dont*", "!dont_ignore.txt"], False),
|
|
],
|
|
)
|
|
def test_match_ignore_from_file(
|
|
file_to_ignore_relpath, patterns, expected_match
|
|
):
|
|
|
|
dvcignore_path = os.path.join(
|
|
os.path.sep, "full", "path", "to", "ignore", "file", ".dvcignore"
|
|
)
|
|
dvcignore_dirname = os.path.dirname(dvcignore_path)
|
|
|
|
ignore_file = mock_dvcignore(dvcignore_path, patterns)
|
|
|
|
assert (
|
|
ignore_file.matches(dvcignore_dirname, file_to_ignore_relpath)
|
|
== expected_match
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("omit_dir", [".git", ".hg", ".dvc"])
|
|
def test_should_ignore_dir(omit_dir):
|
|
ignore = DvcIgnoreDirs([".git", ".hg", ".dvc"])
|
|
|
|
root = os.path.join(os.path.sep, "walk", "dir", "root")
|
|
dirs = [omit_dir, "dir1", "dir2"]
|
|
files = []
|
|
|
|
new_dirs, _ = ignore(root, dirs, files)
|
|
|
|
assert set(new_dirs) == {"dir1", "dir2"}
|