From 487eae3f0719d5ce46081e7555a0d836dca49891 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 10 Jan 2018 16:09:18 +0200 Subject: [PATCH] test: improve test coverage Signed-off-by: Ruslan Kuprieiev --- dvc/data_cloud.py | 34 +++++++++---------------------- dvc/project.py | 2 +- tests/__init__.py | 5 +++++ tests/test_add.py | 13 ++++++++++++ tests/test_cmd_lock.py | 33 ++++++++++++++++++++++++++++++ tests/test_data_cloud.py | 43 +++++++++++++++++++++++++++++++++++++--- tests/test_gc.py | 25 ++++++++++++++++------- tests/test_init.py | 24 ++++++++++++++++++++++ tests/test_remove.py | 14 +++++++++++++ tests/test_repro.py | 14 +++++++++++++ tests/test_run.py | 39 ++++++++++++++++++++++++++++++++++++ 11 files changed, 210 insertions(+), 36 deletions(-) create mode 100644 tests/test_cmd_lock.py create mode 100644 tests/test_init.py diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 42b10bf65..be5374709 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -10,25 +10,9 @@ import filecmp import math from boto.s3.connection import S3Connection -try: - import httplib -except ImportError: - # Python3 workaround for ResumableDownloadHandler. - # See https://github.com/boto/boto/pull/3755. - import sys - import http.client as httplib - sys.modules['httplib'] = httplib from boto.s3.resumable_download_handler import ResumableDownloadHandler from google.cloud import storage as gc -try: - from urlparse import urlparse -except ImportError: - # Python 3 - # pylint: disable=no-name-in-module, import-error - from urllib.parse import urlparse - -import dvc from dvc.cloud.instance_manager import CloudSettings from dvc.logger import Logger from dvc.exceptions import DvcException @@ -234,20 +218,21 @@ class DataCloudLOCAL(DataCloudBase): return (local_exists, remote_exists, diff) + class DataCloudHTTP(DataCloudBase): """ Driver for http cloud. """ - def push(self, item): + def push(self, path): raise Exception('Not implemented yet') - def pull(self, item): + def pull(self, path): raise Exception('Not implemented yet') - def remove(self, item): + def remove(self, path): raise Exception('Not implemented yet') - def status(self, item): + def status(self, path): raise Exception('Not implemented yet') @staticmethod @@ -327,12 +312,12 @@ class DataCloudHTTP(DataCloudBase): Logger.debug('Checksum matches') return True - def import_data(self, url, item): + def import_data(self, url, path): """ Download single file from url. """ - tmp_file = self.tmp_file(item.data.dvc) + tmp_file = self.tmp_file(path) downloaded, header = self._downloaded_size(tmp_file) req = requests.get(url, stream=True, headers=header) @@ -347,10 +332,9 @@ class DataCloudHTTP(DataCloudBase): if not self._verify_md5(req, tmp_file): return None - os.rename(tmp_file, item.data.dvc) - item.move_data_to_cache() + os.rename(tmp_file, path) - return item + return path class DataCloudAWS(DataCloudBase): diff --git a/dvc/project.py b/dvc/project.py index 8cf5f5084..5c5f9cad6 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -73,7 +73,7 @@ class Project(object): self.cloud = DataCloud(self.config._config) @staticmethod - def init(root_dir): + def init(root_dir=os.curdir): """ Initiate dvc project in directory. diff --git a/tests/__init__.py b/tests/__init__.py index e69de29bb..bc0468a6d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,5 @@ +import os +# Increasing fd ulimit for tests +if os.name == 'nt': + import win32file + win32file._setmaxstdio(2048) diff --git a/tests/test_add.py b/tests/test_add.py index 4914b4245..f2778a278 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -2,11 +2,13 @@ import os import shutil import filecmp +from dvc.main import main from dvc.data_cloud import file_md5 from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError from dvc.stage import OutputAlreadyTrackedError from dvc.project import StageNotFoundError +from dvc.command.add import CmdAdd from tests.basic_env import TestDvc @@ -55,3 +57,14 @@ class TestAddTrackedFile(TestDvc): with self.assertRaises(OutputAlreadyTrackedError) as cx: self.dvc.add(fname) + + +class TestCmdAdd(TestDvc): + def test(self): + ret = main(['add', + self.FOO]) + self.assertEqual(ret, 0) + + ret = main(['add', + 'non-existing-file']) + self.assertNotEqual(ret, 0) diff --git a/tests/test_cmd_lock.py b/tests/test_cmd_lock.py new file mode 100644 index 000000000..7d2a5e10d --- /dev/null +++ b/tests/test_cmd_lock.py @@ -0,0 +1,33 @@ +from dvc.main import main +from dvc.stage import Stage + +from tests.test_repro import TestRepro + + +class TestCmdLock(TestRepro): + def test_lock(self): + ret = main(['lock', + self.file1_stage]) + stage = Stage.load(self.dvc, self.file1_stage) + self.assertEqual(ret, 0) + self.assertTrue(stage.locked) + + def test_unlock(self): + ret = main(['lock', + '-u', + self.file1_stage]) + stage = Stage.load(self.dvc, self.file1_stage) + self.assertEqual(ret, 0) + self.assertFalse(stage.locked) + + def test_twice(self): + self.test_lock() + self.test_lock() + + self.test_unlock() + self.test_unlock() + + def test_non_existing(self): + ret = main(['lock', + 'non-existing-file']) + self.assertNotEqual(ret, 0) diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index 4cd41ddc2..e08d3514d 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -2,7 +2,7 @@ import os import uuid from dvc.config import ConfigError -from dvc.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL +from dvc.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL, DataCloudHTTP, DataCloudGCP from dvc.data_cloud import STATUS_UNKNOWN, STATUS_OK, STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED from dvc.cloud.instance_manager import CloudSettings @@ -87,8 +87,34 @@ class TestDataCloudAWS(TestDataCloudBase): self.cloud = DataCloudAWS(cloud_settings) def test(self): - if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \ - os.getenv("TRAVIS_SECURE_ENV_VARS") == "true": + if self._should_test(): + self._test_cloud() + + +class TestDataCloudGCP(TestDataCloudBase): + TEST_REPO_GCP = 'dvc-test/myrepo' + TEST_REPO_GCP_PROJECT='dvc-project' + + def _should_test(self): + if os.getenv("TRAVIS") == "true" or os.getenv("APPVEYOR") == "True": + return False + return True + + def _setup_cloud(self): + if not self._should_test(): + return + + # Cleanup + os.system("gsutil rm -rf gs://{}/".format(self.TEST_REPO_GCP)) + + # Setup cloud + config = {'StoragePath': self.TEST_REPO_GCP, + 'Region': self.TEST_REPO_GCP_PROJECT} + cloud_settings = CloudSettings(None, config) + self.cloud = DataCloudGCP(cloud_settings) + + def test(self): + if self._should_test(): self._test_cloud() @@ -105,3 +131,14 @@ class TestDataCloudLOCAL(TestDataCloudBase): self.assertTrue(os.path.isdir(self.dname)) path = os.path.join(self.dname, self.FOO) self.assertTrue(os.path.isfile(path)) + + +class TestDataCloudHTTP(TestDvc): + def test(self): + cloud = DataCloudHTTP(CloudSettings(None, None)) + path = 'out' + url = 'https://github.com/dataversioncontrol/dvc/LICENSE' + ret = cloud.import_data(url, path) + self.assertEqual(ret, path) + self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.isfile(path)) diff --git a/tests/test_gc.py b/tests/test_gc.py index c61ec9993..ef06fe75a 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -2,31 +2,42 @@ import os import shutil import filecmp +from dvc.main import main from dvc.data_cloud import file_md5 from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError from dvc.project import StageNotFoundError +from dvc.command.gc import CmdGC from tests.basic_env import TestDvc class TestGC(TestDvc): - def test_gc(self): - stage = self.dvc.add(self.FOO) - good_cache = self.dvc.cache.all() + def setUp(self): + super(TestGC, self).setUp() - bad_cache = [] + stage = self.dvc.add(self.FOO) + self.good_cache = self.dvc.cache.all() + + self.bad_cache = [] for i in ['1', '2', '3']: path = os.path.join(self.dvc.cache.cache_dir, i) self.create(path, i) - bad_cache.append(path) + self.bad_cache.append(path) + def test_api(self): self.dvc.gc() + self._test_gc() + def test_cli(self): + ret = main(['gc']) + self._test_gc() + + def _test_gc(self): self.assertTrue(os.path.isdir(self.dvc.cache.cache_dir)) - for c in bad_cache: + for c in self.bad_cache: self.assertFalse(os.path.exists(c)) - for c in good_cache: + for c in self.good_cache: self.assertTrue(os.path.exists(c)) self.assertTrue(os.path.isfile(c)) diff --git a/tests/test_init.py b/tests/test_init.py new file mode 100644 index 000000000..4f286f06e --- /dev/null +++ b/tests/test_init.py @@ -0,0 +1,24 @@ +import os + +from dvc.main import main +from dvc.command.init import CmdInit +from dvc.project import Project + +from tests.basic_env import TestGit + + +class TestInit(TestGit): + def _test_init(self): + self.assertTrue(os.path.exists(Project.DVC_DIR)) + self.assertTrue(os.path.isdir(Project.DVC_DIR)) + + def test_api(self): + Project.init() + + self._test_init() + + def test_cli(self): + ret = main(['init']) + self.assertEqual(ret, 0) + + self._test_init() diff --git a/tests/test_remove.py b/tests/test_remove.py index d0fac4d2c..591c99bd9 100644 --- a/tests/test_remove.py +++ b/tests/test_remove.py @@ -1,7 +1,9 @@ import os +from dvc.main import main from dvc.stage import Stage, OutputOutsideOfRepoError from dvc.project import StageNotFoundError +from dvc.command.remove import CmdRemove from tests.basic_env import TestDvc @@ -26,3 +28,15 @@ class TestRemoveFileOutsideOfRepo(TestDvc): def test(self): with self.assertRaises(OutputOutsideOfRepoError) as cx: self.dvc.remove(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO)) + + +class TestCmdRemove(TestDvc): + def test(self): + self.dvc.add(self.FOO) + ret = main(['remove', + self.FOO]) + self.assertEqual(ret, 0) + + ret = main(['remove', + 'non-existing-file']) + self.assertNotEqual(ret, 0) diff --git a/tests/test_repro.py b/tests/test_repro.py index 43dbd3fe0..4bad3a21b 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -3,6 +3,9 @@ import stat import shutil import filecmp +from dvc.main import main +from dvc.command.repro import CmdRepro + from tests.basic_env import TestDvc @@ -70,3 +73,14 @@ class TestReproPhony(TestReproChangedData): self.dvc.reproduce(stage.path) self.assertTrue(filecmp.cmp(self.file1, self.BAR)) + + +class TestCmdRepro(TestRepro): + def test(self): + ret = main(['repro', + self.file1_stage]) + self.assertEqual(ret, 0) + + ret = main(['repro', + 'non-existing-file']) + self.assertNotEqual(ret, 0) diff --git a/tests/test_run.py b/tests/test_run.py index 20dd33ab0..7a4957546 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -1,7 +1,10 @@ import os import filecmp +from dvc.main import main from dvc.data_cloud import file_md5 +from dvc.stage import Stage +from dvc.command.run import CmdRun from tests.basic_env import TestDvc @@ -52,3 +55,39 @@ class TestRunNoExec(TestDvc): self.dvc.run(cmd='python {} {} {}'.format(self.CODE, self.FOO, 'out'), no_exec=True) self.assertFalse(os.path.exists('out')) + + +class TestCmdRun(TestDvc): + def test_run(self): + ret = main(['run', + '-d', self.FOO, + '-d', self.CODE, + '-o', 'out', + '-f', 'out.dvc', + 'python', self.CODE, self.FOO, 'out']) + self.assertEqual(ret, 0) + self.assertTrue(os.path.isfile('out')) + self.assertTrue(os.path.isfile('out.dvc')) + self.assertTrue(filecmp.cmp(self.FOO, 'out')) + + def test_run_bad_command(self): + ret = main(['run', + 'non-existing-command']) + self.assertNotEqual(ret, 0) + + def test_stage_file_name(self): + fname = 'path/to/file' + outs = [fname, 'dummy'] + dvcfile = os.path.basename(fname) + Stage.STAGE_FILE_SUFFIX + + ret = CmdRun.stage_file_name(None, [], []) + self.assertEqual(ret, Stage.STAGE_FILE) + + ret = CmdRun.stage_file_name(None, [], outs) + self.assertEqual(ret, dvcfile) + + ret = CmdRun.stage_file_name(None, outs, []) + self.assertEqual(ret, dvcfile) + + ret = CmdRun.stage_file_name(fname, [], []) + self.assertEqual(ret, fname)