mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
267 lines
8.2 KiB
Python
267 lines
8.2 KiB
Python
# encoding: utf-8
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
import os
|
|
import shutil
|
|
import uuid
|
|
import tempfile
|
|
import logging
|
|
import warnings
|
|
|
|
from git import Repo
|
|
from git.exc import GitCommandNotFound
|
|
from unittest import TestCase
|
|
import pytest
|
|
|
|
from dvc.command.remote import CmdRemoteAdd
|
|
from dvc.repo import Repo as DvcRepo
|
|
from dvc.utils.compat import open, str
|
|
|
|
|
|
logger = logging.getLogger("dvc")
|
|
|
|
|
|
class TestDirFixture(object):
|
|
DATA_DIR = "data_dir"
|
|
DATA_SUB_DIR = os.path.join(DATA_DIR, "data_sub_dir")
|
|
DATA = os.path.join(DATA_DIR, "data")
|
|
DATA_SUB = os.path.join(DATA_SUB_DIR, "data_sub")
|
|
DATA_CONTENTS = DATA
|
|
DATA_SUB_CONTENTS = DATA_SUB
|
|
FOO = "foo"
|
|
FOO_CONTENTS = FOO
|
|
BAR = "bar"
|
|
# NOTE: len(FOO_CONTENTS) must be != len(BAR_CONTENTS)
|
|
#
|
|
# Our state database relies on file mtime and file size to determine
|
|
# that a file has changed. Usually, mtime is enough by itself but on
|
|
# some filesystems like APFS on macOS mtime resolution is so bad,
|
|
# that our tests can overwrite a file in that time window without dvc
|
|
# being able to detect that, thus causing tests to fail. Usually,
|
|
# in tests, we replace foo with bar, so we need to make sure that when we
|
|
# modify a file in our tests, its content length changes.
|
|
BAR_CONTENTS = BAR + "r"
|
|
CODE = "code.py"
|
|
CODE_CONTENTS = (
|
|
"import sys\nimport shutil\n"
|
|
"shutil.copyfile(sys.argv[1], sys.argv[2])"
|
|
)
|
|
UNICODE = "тест"
|
|
UNICODE_CONTENTS = "проверка"
|
|
|
|
def __init__(self, root_dir=None):
|
|
if root_dir:
|
|
os.mkdir(root_dir)
|
|
else:
|
|
root_dir = self.mkdtemp()
|
|
self._root_dir = os.path.realpath(root_dir)
|
|
|
|
def _pushd(self, d):
|
|
if not hasattr(self, "_saved_dir"):
|
|
self._saved_dir = os.path.realpath(os.curdir)
|
|
os.chdir(d)
|
|
|
|
def _popd(self):
|
|
os.chdir(self._saved_dir)
|
|
self._saved_dir = None
|
|
|
|
def create(self, name, contents):
|
|
dname = os.path.dirname(name)
|
|
if len(dname) > 0 and not os.path.isdir(dname):
|
|
os.makedirs(dname)
|
|
|
|
with open(name, "a", encoding="utf-8") as f:
|
|
f.write(
|
|
contents
|
|
if isinstance(contents, str)
|
|
else contents.decode("utf-8")
|
|
)
|
|
|
|
@staticmethod
|
|
def mkdtemp():
|
|
prefix = "dvc-test.{}.".format(os.getpid())
|
|
suffix = ".{}".format(uuid.uuid4())
|
|
return tempfile.mkdtemp(prefix=prefix, suffix=suffix)
|
|
|
|
def setUp(self):
|
|
self._pushd(self._root_dir)
|
|
self.create(self.FOO, self.FOO_CONTENTS)
|
|
self.create(self.BAR, self.BAR_CONTENTS)
|
|
self.create(self.CODE, self.CODE_CONTENTS)
|
|
os.mkdir(self.DATA_DIR)
|
|
os.mkdir(self.DATA_SUB_DIR)
|
|
self.create(self.DATA, self.DATA_CONTENTS)
|
|
self.create(self.DATA_SUB, self.DATA_SUB_CONTENTS)
|
|
self.create(self.UNICODE, self.UNICODE_CONTENTS)
|
|
|
|
def tearDown(self):
|
|
self._popd()
|
|
try:
|
|
shutil.rmtree(self._root_dir)
|
|
except OSError as exc:
|
|
# We ignore this under Windows with a warning because it happened
|
|
# to be really hard to trace all not properly closed files.
|
|
#
|
|
# Best guess so far is that gitpython is the culprit:
|
|
# it opens files and uses __del__ to close them, which can happen
|
|
# late in current pythons. TestGitFixture and TestDvcFixture try
|
|
# to close that and it works on most of the tests, but not all.
|
|
# Repos and thus git repos are created all over the dvc ;)
|
|
if os.name == "nt" and exc.winerror == 32:
|
|
warnings.warn("Failed to remove test dir: " + str(exc))
|
|
else:
|
|
raise
|
|
|
|
|
|
class TestGitFixture(TestDirFixture):
|
|
N_RETRIES = 5
|
|
|
|
def setUp(self):
|
|
super(TestGitFixture, self).setUp()
|
|
# NOTE: handles EAGAIN error on BSD systems (osx in our case).
|
|
# Otherwise when running tests you might get this exception:
|
|
#
|
|
# GitCommandNotFound: Cmd('git') not found due to:
|
|
# OSError('[Errno 35] Resource temporarily unavailable')
|
|
retries = self.N_RETRIES
|
|
while retries:
|
|
try:
|
|
self.git = Repo.init()
|
|
except GitCommandNotFound:
|
|
retries -= 1
|
|
continue
|
|
break
|
|
|
|
self.git.index.add([self.CODE])
|
|
self.git.index.commit("add code")
|
|
|
|
def tearDown(self):
|
|
self.git.close()
|
|
|
|
|
|
class TestGitSubmoduleFixture(TestGitFixture):
|
|
def __init__(self, root_dir=None):
|
|
super(TestGitSubmoduleFixture, self).__init__(root_dir)
|
|
|
|
def setUp(self):
|
|
super(TestGitSubmoduleFixture, self).setUp()
|
|
subrepo = Repo.init()
|
|
subrepo_path = "subrepo"
|
|
self.git.create_submodule(subrepo_path, subrepo_path, subrepo.git_dir)
|
|
self._pushd(subrepo_path)
|
|
|
|
|
|
class TestDvcFixture(TestGitFixture):
|
|
def __init__(self, root_dir=None):
|
|
super(TestDvcFixture, self).__init__(root_dir)
|
|
|
|
def setUp(self):
|
|
super(TestDvcFixture, self).setUp()
|
|
self.dvc = DvcRepo.init(self._root_dir)
|
|
self.dvc.scm.commit("init dvc")
|
|
|
|
def tearDown(self):
|
|
self.dvc.scm.git.close()
|
|
|
|
|
|
class TestDvcGitInitializedFixture(TestDvcFixture):
|
|
def __init__(self, root_dir=None):
|
|
super(TestDvcGitInitializedFixture, self).__init__(root_dir)
|
|
|
|
def setUp(self):
|
|
super(TestDvcGitInitializedFixture, self).setUp()
|
|
self.git.init()
|
|
|
|
|
|
class TestDvcDataFileFixture(TestDvcGitInitializedFixture):
|
|
DATA_DVC_FILE = "data.dvc"
|
|
DATA_DIR_DVC_FILE = "data_sub_dir.dvc"
|
|
REMOTE = "myremote2"
|
|
|
|
def __init__(self, root_dir=None, cache_dir=None):
|
|
super(TestDvcDataFileFixture, self).__init__(root_dir)
|
|
self.cache_dir = cache_dir
|
|
|
|
def setUp(self):
|
|
super(TestDvcDataFileFixture, self).setUp()
|
|
|
|
self.dvc.add(self.DATA)
|
|
self.dvc.add(self.DATA_SUB_DIR)
|
|
|
|
if self.cache_dir:
|
|
shutil.copytree(self.dvc.cache.local.cache_dir, self.cache_dir)
|
|
|
|
class MockConfig:
|
|
system = None
|
|
glob = None
|
|
local = None
|
|
default = True
|
|
name = self.REMOTE
|
|
url = self.cache_dir
|
|
|
|
cmd = CmdRemoteAdd(MockConfig())
|
|
cmd.run()
|
|
|
|
self.git.index.add(
|
|
[
|
|
os.path.join(self.DATA_DIR, self.DATA_DVC_FILE),
|
|
os.path.join(self.DATA_DIR, self.DATA_DIR_DVC_FILE),
|
|
os.path.join(self.DATA_DIR, ".gitignore"),
|
|
".dvc/config",
|
|
self.FOO,
|
|
self.BAR,
|
|
]
|
|
)
|
|
self.git.index.commit("Hello world commit")
|
|
|
|
# Return to the dir we started to not confuse parent fixture
|
|
os.chdir(self._saved_dir)
|
|
|
|
|
|
# NOTE: Inheritance order in the classes below is important.
|
|
|
|
|
|
class TestDir(TestDirFixture, TestCase):
|
|
def __init__(self, methodName, root_dir=None):
|
|
TestDirFixture.__init__(self, root_dir)
|
|
TestCase.__init__(self, methodName)
|
|
|
|
|
|
class TestGit(TestGitFixture, TestCase):
|
|
def __init__(self, methodName, root_dir=None):
|
|
TestGitFixture.__init__(self, root_dir)
|
|
TestCase.__init__(self, methodName)
|
|
|
|
|
|
class TestGitSubmodule(TestGitSubmoduleFixture, TestCase):
|
|
def __init__(self, methodName, root_dir=None):
|
|
TestGitSubmoduleFixture.__init__(self, root_dir)
|
|
TestCase.__init__(self, methodName)
|
|
|
|
|
|
class TestDvc(TestDvcFixture, TestCase):
|
|
def __init__(self, methodName, root_dir=None):
|
|
TestDvcFixture.__init__(self, root_dir)
|
|
TestCase.__init__(self, methodName)
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def inject_fixtures(self, caplog):
|
|
self._caplog = caplog
|
|
|
|
|
|
class TestDvcPkg(TestDvcFixture, TestCase):
|
|
GIT_PKG = "git_pkg"
|
|
CACHE_DIR = "mycache"
|
|
|
|
def __init__(self, methodName, root_dir=None):
|
|
TestDvcFixture.__init__(self, root_dir)
|
|
TestCase.__init__(self, methodName)
|
|
|
|
self.pkg_dir = os.path.join(self._root_dir, self.GIT_PKG)
|
|
cache_dir = os.path.join(self._root_dir, self.CACHE_DIR)
|
|
self.pkg_fixture = TestDvcDataFileFixture(
|
|
root_dir=self.pkg_dir, cache_dir=cache_dir
|
|
)
|
|
self.pkg_fixture.setUp()
|