From 53414a06182b98e900688a11314be82cdef1acf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Rowlands=20=28=EB=B3=80=EA=B8=B0=ED=98=B8=29?= Date: Tue, 4 May 2021 15:14:55 +0900 Subject: [PATCH] dvcfile/stage: check if dvcfile would be gitignored on stage creation (#5907) * dvcfile: add check for gitignored when validating filenames * update tests * update exception using in tests * review fixes - clarify method name - use FileIsGitIgnored --- dvc/dvcfile.py | 37 ++++++++++++++++++++++++++----------- dvc/stage/__init__.py | 4 ++-- tests/func/test_add.py | 13 ++++++++++++- tests/unit/test_dvcfile.py | 2 +- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py index 2ddb45078..116594518 100644 --- a/dvc/dvcfile.py +++ b/dvc/dvcfile.py @@ -37,8 +37,12 @@ PIPELINE_LOCK = "dvc.lock" class FileIsGitIgnored(DvcException): - def __init__(self, path): - super().__init__(f"'{path}' is git-ignored.") + def __init__(self, path, pipeline_file=False): + super().__init__( + "{}'{}' is git-ignored.".format( + "bad DVC file name " if pipeline_file else "", path, + ) + ) class LockfileCorruptedError(DvcException): @@ -66,7 +70,19 @@ def is_lock_file(path): return os.path.basename(path) == PIPELINE_LOCK -def check_dvc_filename(path): +def is_git_ignored(repo, path): + from dvc.fs.local import LocalFileSystem + from dvc.scm.base import NoSCMError + + try: + return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored( + path + ) + except NoSCMError: + return False + + +def check_dvcfile_path(repo, path): if not is_valid_filename(path): raise StageFileBadNameError( "bad DVC file name '{}'. DVC files should be named " @@ -75,6 +91,9 @@ def check_dvc_filename(path): ) ) + if is_git_ignored(repo, path): + raise FileIsGitIgnored(relpath(path), True) + class FileMixin: SCHEMA = None @@ -108,15 +127,11 @@ class FileMixin: return self.repo.fs.exists(self.path) def _is_git_ignored(self): - from dvc.fs.local import LocalFileSystem - - return isinstance( - self.repo.fs, LocalFileSystem - ) and self.repo.scm.is_ignored(self.path) + return is_git_ignored(self.repo, self.path) def _verify_filename(self): if self.verify: - check_dvc_filename(self.path) + check_dvcfile_path(self.repo, self.path) def _check_gitignored(self): if self._is_git_ignored(): @@ -181,7 +196,7 @@ class SingleStageFile(FileMixin): assert not isinstance(stage, PipelineStage) if self.verify: - check_dvc_filename(self.path) + check_dvcfile_path(self.repo, self.path) logger.debug( "Saving information to '{file}'.".format(file=relpath(self.path)) ) @@ -218,7 +233,7 @@ class PipelineFile(FileMixin): assert isinstance(stage, PipelineStage) if self.verify: - check_dvc_filename(self.path) + check_dvcfile_path(self.repo, self.path) if update_pipeline and not stage.is_data_source: self._dump_pipeline_file(stage) diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 9f6081349..4f89236f4 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -72,11 +72,11 @@ class RawData: def create_stage(cls, repo, path, external=False, **kwargs): - from dvc.dvcfile import check_dvc_filename + from dvc.dvcfile import check_dvcfile_path wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir) path = os.path.abspath(path) - check_dvc_filename(path) + check_dvcfile_path(repo, path) check_stage_path(repo, wdir, is_wdir=kwargs.get("wdir")) check_stage_path(repo, os.path.dirname(path)) diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 4127b4ae4..78ebf4dbc 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -823,7 +823,7 @@ def test_add_from_data_dir(tmp_dir, scm, dvc): tmp_dir.gen({"dir": {"file2": "file2 content"}}) with pytest.raises(OverlappingOutputPathsError) as e: - dvc.add(os.path.join("dir", "file2")) + dvc.add(os.path.join("dir", "file2"), fname="file2.dvc") assert str(e.value) == ( "Cannot add '{out}', because it is overlapping with other DVC " "tracked output: 'dir'.\n" @@ -1175,3 +1175,14 @@ def test_add_to_cache_from_remote(tmp_dir, dvc, workspace): foo.unlink() dvc.checkout(str(foo)) assert foo.read_text() == "foo" + + +def test_add_ignored(tmp_dir, scm, dvc): + from dvc.dvcfile import FileIsGitIgnored + + tmp_dir.gen({"dir": {"subdir": {"file": "content"}}, ".gitignore": "dir/"}) + with pytest.raises(FileIsGitIgnored) as exc: + dvc.add(targets=[os.path.join("dir", "subdir")]) + assert str(exc.value) == ("bad DVC file name '{}' is git-ignored.").format( + os.path.join("dir", "subdir.dvc") + ) diff --git a/tests/unit/test_dvcfile.py b/tests/unit/test_dvcfile.py index f0017c517..b9b5b5cbc 100644 --- a/tests/unit/test_dvcfile.py +++ b/tests/unit/test_dvcfile.py @@ -122,4 +122,4 @@ def test_try_loading_dvcfile_that_is_gitignored(tmp_dir, dvc, scm, file): with pytest.raises(FileIsGitIgnored) as exc_info: dvcfile._load() - assert str(exc_info.value) == f"'{file}' is git-ignored." + assert str(exc_info.value) == f"bad DVC file name '{file}' is git-ignored."