Decoupling dvcignore and fs (#5812)

* Decoupling dvcignore and fs

[x] remove dvcignore from fs objects
[x] attach dvcignore to repo
[x] add ignore check to functions(add, checkout, output, pipeline, etc)
[x] pass all tests
[x] final check

* Solve all of the arugments error.

* add dvcignore to dvc list

* Solve some test_ignore cases

* add dvcignore to checkout.

1. add dvcignore to 'checkout', 'add'
2. add ignore_filter to fs

* Solve two tests

* Get rid of ignore_filter

* Add ignore examine to download

* Solve a test issue

* Solve dvcignore and subrepo problems

* Solve sub repo error

* Revert some changes

* pass all tests

* Remove debug codes

* Some final exams

* dvcignore trie tree only update when needed

* reuse of the dir result in path walk.

* Some change after code review.

* Recover something from false `--force` push

* add abspath trans to dvcginore

* pass pylint

* pylint related

* Update dvc/fs/repo.py

Co-authored-by: Ruslan Kuprieiev <[email protected]>

* Changes related to reviews.

* dvcignore call api change

from dvcignore.walk_files(fs.walk(path_info)) to dvcignore.walk_files(fs, path_info)

* Get rid of PathInfo in test_ignore.py

* fs.walk follow the arguments of os.walk

* Dvcignore as parameters

* pass dvcignore as arguments to checkouts

* Decoupling state and repo

* Update dvc/dvcfile.py

Co-authored-by: Paweł Redzyński <[email protected]>

* Update dvc/objects/stage.py

Co-authored-by: Paweł Redzyński <[email protected]>

* Solve some small problems

* Add tests to verify that dependency now not ignore

fix#5256

* Update dvc/ignore.py

* Update dvc/fs/git.py

* Update dvc/repo/__init__.py

* Some small improvements

* pass black

* Refactor is_ignored

* Update dvc/output/base.py

* repofs: use dvcignore in exists/isdir/isfile

Per https://github.com/iterative/dvc/pull/5812/files#r630604524

Co-authored-by: myname <[email protected]>
Co-authored-by: Ruslan Kuprieiev <[email protected]>
Co-authored-by: Paweł Redzyński <[email protected]>
Co-authored-by: Ruslan Kuprieiev <[email protected]>

tests/func/test_fs.py

@@ -144,9 +144,13 @@ class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir):
 
 class TestWalkInGit(AssertWalkEqualMixin, TestGit):
     def test_nobranch(self):
-        fs = LocalFileSystem(None, {"url": self._root_dir}, use_dvcignore=True)
+        fs = LocalFileSystem(None, {"url": self._root_dir})
+        walk_result = []
+        for root, dirs, files in fs.walk("."):
+            dirs[:] = [i for i in dirs if i != ".git"]
+            walk_result.append((root, dirs, files))
         self.assertWalkEqual(
-            fs.walk("."),
+            walk_result,
             [
                 (".", ["data_dir"], ["bar", "тест", "code.py", "foo"]),
                 (join("data_dir"), ["data_sub_dir"], ["data"]),
@@ -196,13 +200,11 @@ def test_cleanfs_subrepo(tmp_dir, dvc, scm, monkeypatch):
 
     path = PathInfo(subrepo_dir)
 
-    assert dvc.fs.use_dvcignore
-    assert not dvc.fs.exists(path / "foo")
-    assert not dvc.fs.isfile(path / "foo")
-    assert not dvc.fs.exists(path / "dir")
-    assert not dvc.fs.isdir(path / "dir")
+    assert dvc.fs.exists(path / "foo")
+    assert dvc.fs.isfile(path / "foo")
+    assert dvc.fs.exists(path / "dir")
+    assert dvc.fs.isdir(path / "dir")
 
-    assert subrepo.fs.use_dvcignore
     assert subrepo.fs.exists(path / "foo")
     assert subrepo.fs.isfile(path / "foo")
     assert subrepo.fs.exists(path / "dir")
@@ -219,17 +221,13 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
     scm.commit("Add subrepo")
 
     dvc_fs = dvc.fs
-    dvc_fs._reset()
-    scm_fs = scm.get_fs("HEAD", use_dvcignore=True)
+    dvc._reset()
+    scm_fs = scm.get_fs("HEAD")
     path = os.fspath(tmp_dir)
     get_dirs = itemgetter(1)
 
-    assert get_dirs(next(dvc_fs.walk(path))) == []
-    assert get_dirs(next(scm_fs.walk(path))) == []
-
-    kw = {"ignore_subrepos": False}
-    assert get_dirs(next(dvc_fs.walk(path, **kw))) == ["subdir"]
-    assert get_dirs(next(scm_fs.walk(path, **kw))) == ["subdir"]
+    assert set(get_dirs(next(dvc_fs.walk(path)))) == {".dvc", "subdir", ".git"}
+    assert set(get_dirs(next(scm_fs.walk(path)))) == {".dvc", "subdir"}
 
 
 @pytest.mark.parametrize(