Merge pull request #406 from efiop/master

test: improve test coverage

dvc/data_cloud.py

@@ -10,25 +10,9 @@ import filecmp
 import math
 
 from boto.s3.connection import S3Connection
-try:
-    import httplib
-except ImportError:
-    # Python3 workaround for ResumableDownloadHandler.
-    # See https://github.com/boto/boto/pull/3755.
-    import sys
-    import http.client as httplib
-    sys.modules['httplib'] = httplib
 from boto.s3.resumable_download_handler import ResumableDownloadHandler
 from google.cloud import storage as gc
 
-try:
-    from urlparse import urlparse
-except ImportError:
-    # Python 3
-    # pylint: disable=no-name-in-module, import-error
-    from urllib.parse import urlparse
-
-import dvc
 from dvc.cloud.instance_manager import CloudSettings
 from dvc.logger import Logger
 from dvc.exceptions import DvcException
@@ -234,20 +218,21 @@ class DataCloudLOCAL(DataCloudBase):
 
         return (local_exists, remote_exists, diff)
 
+
 class DataCloudHTTP(DataCloudBase):
     """
     Driver for http cloud.
     """
-    def push(self, item):
+    def push(self, path):
         raise Exception('Not implemented yet')
 
-    def pull(self, item):
+    def pull(self, path):
         raise Exception('Not implemented yet')
 
-    def remove(self, item):
+    def remove(self, path):
         raise Exception('Not implemented yet')
 
-    def status(self, item):
+    def status(self, path):
         raise Exception('Not implemented yet')
 
     @staticmethod
@@ -327,12 +312,12 @@ class DataCloudHTTP(DataCloudBase):
         Logger.debug('Checksum matches')
         return True
 
-    def import_data(self, url, item):
+    def import_data(self, url, path):
         """
         Download single file from url.
         """
 
-        tmp_file = self.tmp_file(item.data.dvc)
+        tmp_file = self.tmp_file(path)
 
         downloaded, header = self._downloaded_size(tmp_file)
         req = requests.get(url, stream=True, headers=header)
@@ -347,10 +332,9 @@ class DataCloudHTTP(DataCloudBase):
         if not self._verify_md5(req, tmp_file):
             return None
 
-        os.rename(tmp_file, item.data.dvc)
-        item.move_data_to_cache()
+        os.rename(tmp_file, path)
 
-        return item
+        return path
 
 
 class DataCloudAWS(DataCloudBase):

dvc/project.py

@@ -73,7 +73,7 @@ class Project(object):
         self.cloud = DataCloud(self.config._config)
 
     @staticmethod
-    def init(root_dir):
+    def init(root_dir=os.curdir):
         """
         Initiate dvc project in directory.
 

tests/__init__.py

@@ -0,0 +1,5 @@
+import os
+# Increasing fd ulimit for tests
+if os.name == 'nt':
+    import win32file
+    win32file._setmaxstdio(2048)

tests/test_add.py

@@ -2,11 +2,13 @@ import os
 import shutil
 import filecmp
 
+from dvc.main import main
 from dvc.data_cloud import file_md5
 from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError
 from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError
 from dvc.stage import OutputAlreadyTrackedError
 from dvc.project import StageNotFoundError
+from dvc.command.add import CmdAdd
 
 from tests.basic_env import TestDvc
 
@@ -55,3 +57,14 @@ class TestAddTrackedFile(TestDvc):
 
         with self.assertRaises(OutputAlreadyTrackedError) as cx:
             self.dvc.add(fname)
+
+
+class TestCmdAdd(TestDvc):
+    def test(self):
+        ret = main(['add',
+                    self.FOO])
+        self.assertEqual(ret, 0)
+
+        ret = main(['add',
+                    'non-existing-file'])
+        self.assertNotEqual(ret, 0)

tests/test_cmd_lock.py

@@ -0,0 +1,33 @@
+from dvc.main import main
+from dvc.stage import Stage
+
+from tests.test_repro import TestRepro
+
+
+class TestCmdLock(TestRepro):
+    def test_lock(self):
+        ret = main(['lock',
+                    self.file1_stage])
+        stage = Stage.load(self.dvc, self.file1_stage)
+        self.assertEqual(ret, 0)
+        self.assertTrue(stage.locked)
+
+    def test_unlock(self):
+        ret = main(['lock',
+                    '-u',
+                    self.file1_stage])
+        stage = Stage.load(self.dvc, self.file1_stage)
+        self.assertEqual(ret, 0)
+        self.assertFalse(stage.locked)
+
+    def test_twice(self):
+        self.test_lock()
+        self.test_lock()
+
+        self.test_unlock()
+        self.test_unlock()
+
+    def test_non_existing(self):
+        ret = main(['lock',
+                    'non-existing-file'])
+        self.assertNotEqual(ret, 0)

tests/test_data_cloud.py

@@ -2,7 +2,7 @@ import os
 import uuid
 
 from dvc.config import ConfigError
-from dvc.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL
+from dvc.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL, DataCloudHTTP, DataCloudGCP
 from dvc.data_cloud import STATUS_UNKNOWN, STATUS_OK, STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED
 from dvc.cloud.instance_manager import CloudSettings
 
@@ -87,8 +87,34 @@ class TestDataCloudAWS(TestDataCloudBase):
         self.cloud = DataCloudAWS(cloud_settings)
 
     def test(self):
-        if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \
-           os.getenv("TRAVIS_SECURE_ENV_VARS") == "true":
+        if self._should_test():
+            self._test_cloud()
+
+
+class TestDataCloudGCP(TestDataCloudBase):
+    TEST_REPO_GCP = 'dvc-test/myrepo'
+    TEST_REPO_GCP_PROJECT='dvc-project'
+
+    def _should_test(self):
+        if os.getenv("TRAVIS") == "true" or os.getenv("APPVEYOR") == "True":
+            return False
+        return True
+
+    def _setup_cloud(self):
+        if not self._should_test():
+            return
+
+        # Cleanup
+        os.system("gsutil rm -rf gs://{}/".format(self.TEST_REPO_GCP))
+
+        # Setup cloud
+        config = {'StoragePath': self.TEST_REPO_GCP,
+                  'Region': self.TEST_REPO_GCP_PROJECT}
+        cloud_settings = CloudSettings(None, config)
+        self.cloud = DataCloudGCP(cloud_settings)
+
+    def test(self):
+        if self._should_test():
             self._test_cloud()
 
 
@@ -105,3 +131,14 @@ class TestDataCloudLOCAL(TestDataCloudBase):
         self.assertTrue(os.path.isdir(self.dname))
         path = os.path.join(self.dname, self.FOO)
         self.assertTrue(os.path.isfile(path))
+
+
+class TestDataCloudHTTP(TestDvc):
+    def test(self):
+        cloud = DataCloudHTTP(CloudSettings(None, None))
+        path = 'out'
+        url = 'https://github.com/dataversioncontrol/dvc/LICENSE'
+        ret = cloud.import_data(url, path)
+        self.assertEqual(ret, path)
+        self.assertTrue(os.path.exists(path))
+        self.assertTrue(os.path.isfile(path))

tests/test_gc.py

@@ -2,31 +2,42 @@ import os
 import shutil
 import filecmp
 
+from dvc.main import main
 from dvc.data_cloud import file_md5
 from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError
 from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError
 from dvc.project import StageNotFoundError
+from dvc.command.gc import CmdGC
 
 from tests.basic_env import TestDvc
 
 
 class TestGC(TestDvc):
-    def test_gc(self):
-        stage = self.dvc.add(self.FOO)
-        good_cache = self.dvc.cache.all()
+    def setUp(self):
+        super(TestGC, self).setUp()
 
-        bad_cache = []
+        stage = self.dvc.add(self.FOO)
+        self.good_cache = self.dvc.cache.all()
+
+        self.bad_cache = []
         for i in ['1', '2', '3']:
             path = os.path.join(self.dvc.cache.cache_dir, i)
             self.create(path, i)
-            bad_cache.append(path)
+            self.bad_cache.append(path)
 
+    def test_api(self):
         self.dvc.gc()
+        self._test_gc()
 
+    def test_cli(self):
+        ret = main(['gc'])
+        self._test_gc()
+
+    def _test_gc(self):
         self.assertTrue(os.path.isdir(self.dvc.cache.cache_dir))
-        for c in bad_cache:
+        for c in self.bad_cache:
             self.assertFalse(os.path.exists(c))
 
-        for c in good_cache:
+        for c in self.good_cache:
             self.assertTrue(os.path.exists(c))
             self.assertTrue(os.path.isfile(c))

tests/test_init.py

@@ -0,0 +1,24 @@
+import os
+
+from dvc.main import main
+from dvc.command.init import CmdInit
+from dvc.project import Project
+
+from tests.basic_env import TestGit
+
+
+class TestInit(TestGit):
+    def _test_init(self):
+        self.assertTrue(os.path.exists(Project.DVC_DIR))
+        self.assertTrue(os.path.isdir(Project.DVC_DIR))
+
+    def test_api(self):
+        Project.init()
+        
+        self._test_init()
+
+    def test_cli(self):
+        ret = main(['init'])
+        self.assertEqual(ret, 0)
+
+        self._test_init()

tests/test_remove.py

@@ -1,7 +1,9 @@
 import os
 
+from dvc.main import main
 from dvc.stage import Stage, OutputOutsideOfRepoError
 from dvc.project import StageNotFoundError
+from dvc.command.remove import CmdRemove
 
 from tests.basic_env import TestDvc
 
@@ -26,3 +28,15 @@ class TestRemoveFileOutsideOfRepo(TestDvc):
     def test(self):
         with self.assertRaises(OutputOutsideOfRepoError) as cx:
             self.dvc.remove(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO))
+
+
+class TestCmdRemove(TestDvc):
+    def test(self):
+        self.dvc.add(self.FOO)
+        ret = main(['remove',
+                    self.FOO])
+        self.assertEqual(ret, 0)
+
+        ret = main(['remove',
+                    'non-existing-file'])
+        self.assertNotEqual(ret, 0)

tests/test_repro.py

@@ -3,6 +3,9 @@ import stat
 import shutil
 import filecmp
 
+from dvc.main import main
+from dvc.command.repro import CmdRepro
+
 from tests.basic_env import TestDvc
 
 
@@ -70,3 +73,14 @@ class TestReproPhony(TestReproChangedData):
         self.dvc.reproduce(stage.path)
 
         self.assertTrue(filecmp.cmp(self.file1, self.BAR))
+
+
+class TestCmdRepro(TestRepro):
+    def test(self):
+        ret = main(['repro',
+                    self.file1_stage])
+        self.assertEqual(ret, 0)
+
+        ret = main(['repro',
+                    'non-existing-file'])
+        self.assertNotEqual(ret, 0)

tests/test_run.py

@@ -1,7 +1,10 @@
 import os
 import filecmp
 
+from dvc.main import main
 from dvc.data_cloud import file_md5
+from dvc.stage import Stage
+from dvc.command.run import CmdRun
 
 from tests.basic_env import TestDvc
 
@@ -52,3 +55,39 @@ class TestRunNoExec(TestDvc):
         self.dvc.run(cmd='python {} {} {}'.format(self.CODE, self.FOO, 'out'),
                      no_exec=True)
         self.assertFalse(os.path.exists('out'))
+
+
+class TestCmdRun(TestDvc):
+    def test_run(self):
+        ret = main(['run',
+                    '-d', self.FOO,
+                    '-d', self.CODE,
+                    '-o', 'out',
+                    '-f', 'out.dvc',
+                    'python', self.CODE, self.FOO, 'out'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile('out'))
+        self.assertTrue(os.path.isfile('out.dvc'))
+        self.assertTrue(filecmp.cmp(self.FOO, 'out'))
+
+    def test_run_bad_command(self):
+        ret = main(['run',
+                    'non-existing-command'])
+        self.assertNotEqual(ret, 0)
+
+    def test_stage_file_name(self):
+        fname = 'path/to/file'
+        outs = [fname, 'dummy']
+        dvcfile = os.path.basename(fname) + Stage.STAGE_FILE_SUFFIX
+
+        ret = CmdRun.stage_file_name(None, [], [])
+        self.assertEqual(ret, Stage.STAGE_FILE)
+
+        ret = CmdRun.stage_file_name(None, [], outs)
+        self.assertEqual(ret, dvcfile)
+
+        ret = CmdRun.stage_file_name(None, outs, [])
+        self.assertEqual(ret, dvcfile)
+
+        ret = CmdRun.stage_file_name(fname, [], [])
+        self.assertEqual(ret, fname)