diff --git a/dvc/analytics.py b/dvc/analytics.py index 18aea6428..7c0a69310 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -16,7 +16,8 @@ from dvc.exceptions import NotDvcRepoError from dvc.lock import Lock, LockError from dvc.repo import Repo from dvc.scm import SCM -from dvc.utils import env2bool, is_binary, makedirs +from dvc.utils import env2bool, is_binary +from dvc.utils.fs import makedirs logger = logging.getLogger(__name__) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 1c316416b..36409645e 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -24,8 +24,8 @@ from dvc.path_info import PathInfo, URLInfo from dvc.progress import Tqdm from dvc.remote.slow_link_detection import slow_link_guard from dvc.state import StateNoop -from dvc.utils import makedirs, relpath, tmp_fname -from dvc.utils.fs import move +from dvc.utils import relpath, tmp_fname +from dvc.utils.fs import move, makedirs from dvc.utils.http import open_url logger = logging.getLogger(__name__) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 1117ca993..4cfe48d57 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -23,13 +23,11 @@ from dvc.scm.tree import is_working_tree from dvc.system import System from dvc.utils import copyfile from dvc.utils import file_md5 -from dvc.utils import makedirs from dvc.utils import relpath from dvc.utils import tmp_fname from dvc.utils import walk_files from dvc.compat import fspath_py35 -from dvc.utils.fs import move -from dvc.utils.fs import remove +from dvc.utils.fs import move, makedirs, remove logger = logging.getLogger(__name__) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index af68180aa..9cef82e11 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -72,7 +72,7 @@ class Repo(object): from dvc.repo.metrics import Metrics from dvc.scm.tree import WorkingTree from dvc.repo.tag import Tag - from dvc.utils import makedirs + from dvc.utils.fs import makedirs root_dir = self.find_root(root_dir) diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index cca9ebebe..63d51387a 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -135,23 +135,6 @@ def copyfile(src, dest, no_progress_bar=False, name=None): pbar.update(len(buf)) -def makedirs(path, exist_ok=False, mode=None): - path = fspath_py35(path) - - if mode is None: - os.makedirs(path, exist_ok=exist_ok) - return - - # utilize umask to set proper permissions since Python 3.7 the `mode` - # `makedirs` argument no longer affects the file permission bits of - # newly-created intermediate-level directories. - umask = os.umask(0o777 - mode) - try: - os.makedirs(path, exist_ok=exist_ok) - finally: - os.umask(umask) - - def _split(list_to_split, chunk_size): return [ list_to_split[i : i + chunk_size] diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index 21e0118ef..426baa410 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -152,3 +152,20 @@ def path_isin(child, parent): parent = os.path.join(normalize_path(parent), "") child = normalize_path(child) return child != parent and child.startswith(parent) + + +def makedirs(path, exist_ok=False, mode=None): + path = fspath_py35(path) + + if mode is None: + os.makedirs(path, exist_ok=exist_ok) + return + + # utilize umask to set proper permissions since Python 3.7 the `mode` + # `makedirs` argument no longer affects the file permission bits of + # newly-created intermediate-level directories. + umask = os.umask(0o777 - mode) + try: + os.makedirs(path, exist_ok=exist_ok) + finally: + os.umask(umask) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index cd001c002..c531e4b8c 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -50,7 +50,7 @@ from contextlib import contextmanager import pytest from funcy.py3 import lmap, retry -from dvc.utils import makedirs +from dvc.utils.fs import makedirs from dvc.compat import fspath, fspath_py35 diff --git a/tests/func/test_fs.py b/tests/func/test_fs.py new file mode 100644 index 000000000..be5dcb3ac --- /dev/null +++ b/tests/func/test_fs.py @@ -0,0 +1,20 @@ +import os +import stat + +import pytest + +from dvc.utils.fs import makedirs + + +@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.") +def test_makedirs_permissions(tmp_dir): + dir_mode = 0o755 + intermediate_dir = "тестовая-директория" + test_dir = os.path.join(intermediate_dir, "data") + + assert not os.path.exists(intermediate_dir) + + makedirs(test_dir, mode=dir_mode) + + assert stat.S_IMODE(os.stat(test_dir).st_mode) == dir_mode + assert stat.S_IMODE(os.stat(intermediate_dir).st_mode) == dir_mode diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 99fd23840..cb2d05076 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -9,7 +9,7 @@ from dvc.exceptions import UrlNotDvcRepoError from dvc.repo.get import GetDVCFileError, PathMissingError from dvc.repo import Repo from dvc.system import System -from dvc.utils import makedirs +from dvc.utils.fs import makedirs from dvc.compat import fspath from tests.utils import trees_equal diff --git a/tests/func/test_get_url.py b/tests/func/test_get_url.py index 3657dd713..ae287f903 100644 --- a/tests/func/test_get_url.py +++ b/tests/func/test_get_url.py @@ -4,7 +4,7 @@ import os import pytest from dvc.repo import Repo -from dvc.utils import makedirs +from dvc.utils.fs import makedirs def test_get_file(repo_dir): diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 551ae38a8..c3a2c7b00 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -12,7 +12,7 @@ from dvc.exceptions import PathMissingError from dvc.exceptions import NoOutputInExternalRepoError from dvc.stage import Stage from dvc.system import System -from dvc.utils import makedirs +from dvc.utils.fs import makedirs from dvc.compat import fspath from tests.utils import trees_equal diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py index de212da73..7d73cff79 100644 --- a/tests/func/test_import_url.py +++ b/tests/func/test_import_url.py @@ -7,7 +7,7 @@ from mock import patch import dvc from dvc.main import main -from dvc.utils import makedirs +from dvc.utils.fs import makedirs from tests.basic_env import TestDvc from tests.utils import spy diff --git a/tests/func/test_utils.py b/tests/func/test_utils.py index 70ef3dee8..07747092c 100644 --- a/tests/func/test_utils.py +++ b/tests/func/test_utils.py @@ -1,8 +1,5 @@ # encoding: utf-8 import os -import stat - -import pytest from dvc import utils @@ -62,17 +59,3 @@ def test_boxify(): ) assert expected == utils.boxify("message") - - -@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.") -def test_makedirs_permissions(tmp_dir): - dir_mode = 0o755 - intermediate_dir = "тестовая-директория" - test_dir = os.path.join(intermediate_dir, "data") - - assert not os.path.exists(intermediate_dir) - - utils.makedirs(test_dir, mode=dir_mode) - - assert stat.S_IMODE(os.stat(test_dir).st_mode) == dir_mode - assert stat.S_IMODE(os.stat(intermediate_dir).st_mode) == dir_mode diff --git a/tests/unit/utils/test_fs.py b/tests/unit/utils/test_fs.py index 2c11d287c..169482e51 100644 --- a/tests/unit/utils/test_fs.py +++ b/tests/unit/utils/test_fs.py @@ -16,6 +16,7 @@ from dvc.utils.fs import get_inode from dvc.utils.fs import get_mtime_and_size from dvc.utils.fs import move from dvc.utils.fs import path_isin, remove +from dvc.utils.fs import makedirs from tests.basic_env import TestDir from tests.utils import spy @@ -202,3 +203,16 @@ def test_path_isin_with_absolute_path(): child = os.path.join(parent, "to", "folder") assert path_isin(child, parent) + + +def test_makedirs(repo_dir): + path = os.path.join(repo_dir.root_dir, "directory") + path_info = PathInfo( + os.path.join(repo_dir.root_dir, "another", "directory") + ) + + makedirs(path) + assert os.path.isdir(path) + + makedirs(path_info) + assert os.path.isdir(path_info.fspath) diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py index 15b2c541f..18f68fb02 100644 --- a/tests/unit/utils/test_utils.py +++ b/tests/unit/utils/test_utils.py @@ -8,7 +8,6 @@ from dvc.path_info import PathInfo from dvc.utils import copyfile from dvc.utils import file_md5 from dvc.utils import fix_env -from dvc.utils import makedirs from dvc.utils import relpath from dvc.utils import to_chunks from dvc.utils import tmp_fname @@ -115,19 +114,6 @@ def test_copyfile(path, repo_dir): assert filecmp.cmp(src_info.fspath, dest_info.fspath, shallow=False) -def test_makedirs(repo_dir): - path = os.path.join(repo_dir.root_dir, "directory") - path_info = PathInfo( - os.path.join(repo_dir.root_dir, "another", "directory") - ) - - makedirs(path) - assert os.path.isdir(path) - - makedirs(path_info) - assert os.path.isdir(path_info.fspath) - - def test_tmp_fname(): file_path = os.path.join("path", "to", "file") file_path_info = PathInfo(file_path)