From c9c36c20494babedcff8982e44f6da9718865b9a Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Sun, 29 Sep 2019 23:16:21 -0500 Subject: [PATCH 1/8] git: escape filename before ignoring it Fix #2043 --- dvc/scm/git/__init__.py | 11 +++++++++-- tests/func/test_add.py | 11 +++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 55bef9beb..4678634b2 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -149,8 +149,9 @@ class Git(Base): self.ignored_paths.append(path) - @staticmethod - def _add_entry_to_gitignore(entry, gitignore): + def _add_entry_to_gitignore(self, entry, gitignore): + entry = self._escape(entry) + with open(gitignore, "a+", encoding="utf-8") as fobj: fobj.seek(0, os.SEEK_END) if fobj.tell() == 0: @@ -162,6 +163,12 @@ class Git(Base): prefix = "" if last == "\n" else "\n" fobj.write("{}{}\n".format(prefix, entry)) + @staticmethod + def _escape(entry): + meta_characters = r"[]!*#" + + return "".join("\\" + x if x in meta_characters else x for x in entry) + def ignore_remove(self, path): entry, gitignore = self._get_gitignore(path) diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 82e474d05..b1d639e49 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -634,3 +634,14 @@ def test_should_protect_on_repeated_add(link, dvc_repo, repo_dir): dvc_repo.add(repo_dir.FOO) assert not os.access(repo_dir.FOO, os.W_OK) + + +def test_escape_gitignore_entries(git, dvc_repo, repo_dir): + fname = "file!with*weird#naming_[1].txt" + ignored_fname = r"/file\!with\*weird\#naming_\[1\].txt" + + os.rename(repo_dir.FOO, fname) + + dvc_repo.add(fname) + + assert ignored_fname in get_gitignore_content() From a9201f8237eca489e3934a4b9d32328d3b3e1b6e Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 30 Sep 2019 11:21:39 -0500 Subject: [PATCH 2/8] git: add a note referencing gitignore pattern --- dvc/scm/git/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 4678634b2..02382cad4 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -165,6 +165,7 @@ class Git(Base): @staticmethod def _escape(entry): + # Reference: https://git-scm.com/docs/gitignore#_pattern_format meta_characters = r"[]!*#" return "".join("\\" + x if x in meta_characters else x for x in entry) From 80cb1c919a2c627c18728ed4c35bc95a1eb312c7 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 30 Sep 2019 11:27:53 -0500 Subject: [PATCH 3/8] git: escape question mark character --- dvc/scm/git/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 02382cad4..11b91e339 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -166,7 +166,7 @@ class Git(Base): @staticmethod def _escape(entry): # Reference: https://git-scm.com/docs/gitignore#_pattern_format - meta_characters = r"[]!*#" + meta_characters = r"[]!*#?" return "".join("\\" + x if x in meta_characters else x for x in entry) From 9496f34f459a2316400a1135316c0bb290bcbddb Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Mon, 30 Sep 2019 11:28:38 -0500 Subject: [PATCH 4/8] tests: Windows specific filename for gitignore --- tests/func/test_add.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/func/test_add.py b/tests/func/test_add.py index b1d639e49..f1e293641 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -637,8 +637,14 @@ def test_should_protect_on_repeated_add(link, dvc_repo, repo_dir): def test_escape_gitignore_entries(git, dvc_repo, repo_dir): - fname = "file!with*weird#naming_[1].txt" - ignored_fname = r"/file\!with\*weird\#naming_\[1\].txt" + fname = "file!with*weird#naming_[1].t?t" + ignored_fname = r"/file\!with\*weird\#naming_\[1\].t\?t" + + if os.name == "nt": + # Some characters are not supported by Windows in the filename + # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + fname = "file!with_weird#naming_[1].txt" + ignored_fname = r"/file\!with_weird\#naming_\[1\].txt" os.rename(repo_dir.FOO, fname) From a47ea235614e7ebf366dd0f5b54015f17b25cf91 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Thu, 3 Oct 2019 12:10:52 -0500 Subject: [PATCH 5/8] git: use escape method from GitWildMatchPattern --- dvc/scm/git/__init__.py | 10 ++-------- setup.py | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 11b91e339..d9d52fe92 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -6,6 +6,7 @@ import os import logging from funcy import cached_property +from pathspec.patterns import GitWildMatchPattern from dvc.exceptions import GitHookAlreadyExistsError from dvc.utils.compat import str, open @@ -150,7 +151,7 @@ class Git(Base): self.ignored_paths.append(path) def _add_entry_to_gitignore(self, entry, gitignore): - entry = self._escape(entry) + entry = GitWildMatchPattern.escape(entry) with open(gitignore, "a+", encoding="utf-8") as fobj: fobj.seek(0, os.SEEK_END) @@ -163,13 +164,6 @@ class Git(Base): prefix = "" if last == "\n" else "\n" fobj.write("{}{}\n".format(prefix, entry)) - @staticmethod - def _escape(entry): - # Reference: https://git-scm.com/docs/gitignore#_pattern_format - meta_characters = r"[]!*#?" - - return "".join("\\" + x if x in meta_characters else x for x in entry) - def ignore_remove(self, path): entry, gitignore = self._get_gitignore(path) diff --git a/setup.py b/setup.py index 0da1837cf..a57b84c4c 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,7 @@ install_requires = [ "humanize>=0.5.1", "ruamel.yaml>=0.16.1", "funcy>=1.12", - "pathspec>=0.5.9", + "pathspec@git+https://github.com/cpburnz/python-path-specification@master", "shortuuid>=0.5.0", "tqdm>=4.35.0", "packaging>=19.0", From 1f6d8964b5cf4b190ea9751bda991d0dfc5c1a0a Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Thu, 3 Oct 2019 12:25:12 -0500 Subject: [PATCH 6/8] setup: pathspec@master -> pathspec@1d321d --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a57b84c4c..da36023a5 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,8 @@ install_requires = [ "humanize>=0.5.1", "ruamel.yaml>=0.16.1", "funcy>=1.12", - "pathspec@git+https://github.com/cpburnz/python-path-specification@master", + # pathspec: Waiting for a release (0.6.0) including commit 1d321d + "pathspec@git+https://github.com/cpburnz/python-path-specification@1d321d", "shortuuid>=0.5.0", "tqdm>=4.35.0", "packaging>=19.0", From e6ccf32189e4f33d49b06177d53b403c4d4fe72d Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Thu, 3 Oct 2019 17:26:29 -0500 Subject: [PATCH 7/8] scripts: upgrade pip before installing dependencies --- scripts/build_posix.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/build_posix.sh b/scripts/build_posix.sh index b61a408f8..81c5eabaa 100755 --- a/scripts/build_posix.sh +++ b/scripts/build_posix.sh @@ -78,6 +78,9 @@ install_dependencies() gem install --no-document fpm + print_info "Uprgading pip..." + pip install --upgrade pip + print_info "Installing requirements..." pip install .[all] psutil From cfecb75c4cb4e34293d53feb1b61900016a8f071 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Thu, 3 Oct 2019 21:55:21 -0500 Subject: [PATCH 8/8] setup: pathspec@1d321d -> pathspec>=0.6.0 --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index da36023a5..63b4a33fc 100644 --- a/setup.py +++ b/setup.py @@ -63,8 +63,7 @@ install_requires = [ "humanize>=0.5.1", "ruamel.yaml>=0.16.1", "funcy>=1.12", - # pathspec: Waiting for a release (0.6.0) including commit 1d321d - "pathspec@git+https://github.com/cpburnz/python-path-specification@1d321d", + "pathspec>=0.6.0", "shortuuid>=0.5.0", "tqdm>=4.35.0", "packaging>=19.0",