From 96c87d5643f4d8a8a1fa1b0dfc984fda47c001de Mon Sep 17 00:00:00 2001 From: Saugat Pachhai Date: Sat, 4 Jan 2020 19:27:06 +0545 Subject: [PATCH 1/8] test: merge _should_oss_azure and _get_oss_url inside OSS class --- tests/func/test_data_cloud.py | 11 +++++----- tests/remotes.py | 41 ++++++++++++++++------------------- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index c9eee21b5..2d8be3538 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -34,10 +34,10 @@ from tests.remotes import ( _should_test_aws, _should_test_gcp, _should_test_hdfs, - _should_test_oss, _should_test_ssh, Azure, GDrive, + OSS, TEST_CONFIG, TEST_SECTION, TEST_GCP_CREDS_FILE, @@ -48,7 +48,6 @@ from tests.remotes import ( get_gcp_url, get_hdfs_url, get_local_url, - get_oss_url, get_ssh_url, get_ssh_url_mocked, ) @@ -273,10 +272,10 @@ class TestRemoteAZURE(TestDataCloudBase): class TestRemoteOSS(TestDataCloudBase): def _should_test(self): - return _should_test_oss() + return OSS.should_test() def _get_url(self): - return get_oss_url() + return OSS.get_url() def _get_cloud_class(self): return RemoteOSS @@ -545,10 +544,10 @@ class TestRemoteAZURECLI(TestDataCloudCLIBase): class TestRemoteOSSCLI(TestDataCloudCLIBase): def _should_test(self): - return _should_test_oss() + return OSS.should_test() def _test(self): - url = get_oss_url() + url = OSS.get_url() self.main(["remote", "add", TEST_REMOTE, url]) diff --git a/tests/remotes.py b/tests/remotes.py index f2623b2d4..88093f30f 100644 --- a/tests/remotes.py +++ b/tests/remotes.py @@ -77,18 +77,6 @@ def _should_test_gcp(): return True -def _should_test_oss(): - do_test = env2bool("DVC_TEST_OSS", undefined=None) - if do_test is not None: - return do_test - - return ( - os.getenv("OSS_ENDPOINT") - and os.getenv("OSS_ACCESS_KEY_ID") - and os.getenv("OSS_ACCESS_KEY_SECRET") - ) - - def _should_test_ssh(): do_test = env2bool("DVC_TEST_SSH", undefined=None) if do_test is not None: @@ -184,14 +172,6 @@ def get_gcp_url(): return "gs://" + get_gcp_storagepath() -def get_oss_storagepath(): - return "{}/{}".format(TEST_OSS_REPO_BUCKET, (uuid.uuid4())) - - -def get_oss_url(): - return "oss://{}".format(get_oss_storagepath()) - - # NOTE: staticmethod is only needed in Python 2 class Local: should_test = staticmethod(lambda: True) @@ -270,8 +250,25 @@ class Azure: class OSS: - should_test = staticmethod(_should_test_oss) - get_url = staticmethod(get_oss_url) + @staticmethod + def should_test(): + do_test = env2bool("DVC_TEST_OSS", undefined=None) + if do_test is not None: + return do_test + + return ( + os.getenv("OSS_ENDPOINT") + and os.getenv("OSS_ACCESS_KEY_ID") + and os.getenv("OSS_ACCESS_KEY_SECRET") + ) + + @staticmethod + def get_storagepath(): + return "{}/{}".format(TEST_OSS_REPO_BUCKET, (uuid.uuid4())) + + @staticmethod + def get_url(): + return "oss://{}".format(OSS.get_storagepath()) class SSH: From bb72ce8a5097933ea74218e59109b0a18281669d Mon Sep 17 00:00:00 2001 From: pawel Date: Mon, 6 Jan 2020 15:49:26 +0100 Subject: [PATCH 2/8] analytics: tests: migrate to dir helpers --- tests/unit/test_analytics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 59cd19628..3179aeff0 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -83,8 +83,8 @@ def test_send(mock_post, tmp_path): ({"analytics": "false", "unknown": "broken"}, False), ], ) -def test_is_enabled(dvc_repo, config, result, monkeypatch, tmp_global_config): - configobj = dvc_repo.config._repo_config +def test_is_enabled(dvc, config, result, monkeypatch, tmp_global_config): + configobj = dvc.config._repo_config configobj["core"] = config configobj.write() From d2859a311d6af6798833c2c70b618d7a872e6eb2 Mon Sep 17 00:00:00 2001 From: pawel Date: Mon, 6 Jan 2020 16:02:04 +0100 Subject: [PATCH 3/8] cache: test: migrate to dir helpers --- tests/func/test_cache.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/func/test_cache.py b/tests/func/test_cache.py index eb44bbacd..a1496dcc7 100644 --- a/tests/func/test_cache.py +++ b/tests/func/test_cache.py @@ -188,16 +188,16 @@ class TestShouldCacheBeReflinkOrCopyByDefault(TestDvc): "protected,dir_mode,file_mode", [(False, 0o775, 0o664), (True, 0o775, 0o444)], ) -def test_shared_cache(repo_dir, dvc_repo, protected, dir_mode, file_mode): - assert main(["config", "cache.shared", "group"]) == 0 +def test_shared_cache(tmp_dir, dvc, protected, dir_mode, file_mode): + dvc.config.set("cache", "shared", "group") + dvc.config.set("cache", "protected", str(protected)) + dvc.cache = Cache(dvc) - if protected: - assert main(["config", "cache.protected", "true"]) == 0 + tmp_dir.dvc_gen( + {"file": "file content", "dir": {"file2": "file 2 " "content"}} + ) - assert main(["add", repo_dir.FOO]) == 0 - assert main(["add", repo_dir.DATA_DIR]) == 0 - - for root, dnames, fnames in os.walk(dvc_repo.cache.local.cache_dir): + for root, dnames, fnames in os.walk(dvc.cache.local.cache_dir): for dname in dnames: path = os.path.join(root, dname) assert stat.S_IMODE(os.stat(path).st_mode) == dir_mode From c2b3921af3216d8e2eb6a1d9355e659223ade2f0 Mon Sep 17 00:00:00 2001 From: pawel Date: Mon, 6 Jan 2020 16:06:52 +0100 Subject: [PATCH 4/8] dependency: test: migrate to dir helpers --- tests/func/test_dependency.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/func/test_dependency.py b/tests/func/test_dependency.py index 8d591e9ed..5affd4bbb 100644 --- a/tests/func/test_dependency.py +++ b/tests/func/test_dependency.py @@ -18,5 +18,5 @@ def _get_dep(dvc, path): @pytest.mark.parametrize("url,scheme", TESTS) -def test_scheme(dvc_repo, url, scheme): - assert type(_get_dep(dvc_repo, url)) == DEP_MAP[scheme] +def test_scheme(dvc, url, scheme): + assert type(_get_dep(dvc, url)) == DEP_MAP[scheme] From a620658884ecc67113d6e86a3641ffe0c2e723e8 Mon Sep 17 00:00:00 2001 From: pawel Date: Mon, 6 Jan 2020 16:18:01 +0100 Subject: [PATCH 5/8] http/ imp tests - migrate do dir helpers --- tests/unit/command/test_imp.py | 2 +- tests/unit/command/test_imp_url.py | 4 ++-- tests/unit/remote/test_http.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/command/test_imp.py b/tests/unit/command/test_imp.py index 5775849f1..e5fee4de1 100644 --- a/tests/unit/command/test_imp.py +++ b/tests/unit/command/test_imp.py @@ -2,7 +2,7 @@ from dvc.cli import parse_args from dvc.command.imp import CmdImport -def test_import(mocker, dvc_repo): +def test_import(mocker, dvc): cli_args = parse_args( ["import", "repo_url", "src", "--out", "out", "--rev", "version"] ) diff --git a/tests/unit/command/test_imp_url.py b/tests/unit/command/test_imp_url.py index 8858188f6..2ce69dfab 100644 --- a/tests/unit/command/test_imp_url.py +++ b/tests/unit/command/test_imp_url.py @@ -5,7 +5,7 @@ from dvc.command.imp_url import CmdImportUrl from dvc.exceptions import DvcException -def test_import_url(mocker, dvc_repo): +def test_import_url(mocker, dvc): cli_args = parse_args(["import-url", "src", "out", "--file", "file"]) assert cli_args.func == CmdImportUrl @@ -17,7 +17,7 @@ def test_import_url(mocker, dvc_repo): m.assert_called_once_with("src", out="out", fname="file") -def test_failed_import_url(mocker, caplog, dvc_repo): +def test_failed_import_url(mocker, caplog, dvc): cli_args = parse_args(["import-url", "http://somesite.com/file_name"]) assert cli_args.func == CmdImportUrl diff --git a/tests/unit/remote/test_http.py b/tests/unit/remote/test_http.py index dbd54bcd1..20c8854fc 100644 --- a/tests/unit/remote/test_http.py +++ b/tests/unit/remote/test_http.py @@ -7,7 +7,7 @@ from dvc.remote.http import RemoteHTTP from tests.utils.httpd import StaticFileServer -def test_no_traverse_compatibility(dvc_repo): +def test_no_traverse_compatibility(dvc): config = { "url": "http://example.com/", "path_info": "file.html", @@ -15,15 +15,15 @@ def test_no_traverse_compatibility(dvc_repo): } with pytest.raises(ConfigError): - RemoteHTTP(dvc_repo, config) + RemoteHTTP(dvc, config) -def test_download_fails_on_error_code(dvc_repo): +def test_download_fails_on_error_code(dvc): with StaticFileServer() as httpd: url = "http://localhost:{}/".format(httpd.server_port) config = {"url": url} - remote = RemoteHTTP(dvc_repo, config) + remote = RemoteHTTP(dvc, config) with pytest.raises(HTTPError): remote._download(URLInfo(url) / "missing.txt", "missing.txt") From e1176bf42dc654da37d63087fd9f745d11f2f2fc Mon Sep 17 00:00:00 2001 From: pawel Date: Mon, 6 Jan 2020 16:40:15 +0100 Subject: [PATCH 6/8] imp/imp_url cmd test: remove unnecessary dvc fixture --- tests/unit/command/test_imp.py | 2 +- tests/unit/command/test_imp_url.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/command/test_imp.py b/tests/unit/command/test_imp.py index e5fee4de1..f149ee5dc 100644 --- a/tests/unit/command/test_imp.py +++ b/tests/unit/command/test_imp.py @@ -2,7 +2,7 @@ from dvc.cli import parse_args from dvc.command.imp import CmdImport -def test_import(mocker, dvc): +def test_import(mocker): cli_args = parse_args( ["import", "repo_url", "src", "--out", "out", "--rev", "version"] ) diff --git a/tests/unit/command/test_imp_url.py b/tests/unit/command/test_imp_url.py index 2ce69dfab..df8e20ee9 100644 --- a/tests/unit/command/test_imp_url.py +++ b/tests/unit/command/test_imp_url.py @@ -5,7 +5,7 @@ from dvc.command.imp_url import CmdImportUrl from dvc.exceptions import DvcException -def test_import_url(mocker, dvc): +def test_import_url(mocker): cli_args = parse_args(["import-url", "src", "out", "--file", "file"]) assert cli_args.func == CmdImportUrl @@ -17,7 +17,7 @@ def test_import_url(mocker, dvc): m.assert_called_once_with("src", out="out", fname="file") -def test_failed_import_url(mocker, caplog, dvc): +def test_failed_import_url(mocker, caplog): cli_args = parse_args(["import-url", "http://somesite.com/file_name"]) assert cli_args.func == CmdImportUrl From 46379c8591ede22acfb3c82c6afb70ff68ab8d64 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 6 Jan 2020 18:58:10 +0200 Subject: [PATCH 7/8] config: update command for setting default remote --- dvc/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/config.py b/dvc/config.py index 61ceb6756..ad2070939 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -27,7 +27,7 @@ class NoRemoteError(ConfigError): def __init__(self, command): msg = ( "no remote specified. Setup default remote with\n" - " dvc config core.remote \n" + " dvc remote default \n" "or use:\n" " dvc {} -r \n".format(command) ) From cf37fe2c56b69b1b7210fc00c50951b1bf590ec8 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 6 Jan 2020 20:16:22 +0200 Subject: [PATCH 8/8] travis: deploy snap only on master --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 51b72be6c..039097ddb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -98,6 +98,7 @@ jobs: tags: true repo: iterative/dvc - name: Snapcraft snap + stage: deploy addons: snaps: - name: snapcraft @@ -121,6 +122,8 @@ jobs: snap: dvc_*.snap channel: ${SOURCE_TAG:+stable}${SOURCE_TAG:-edge} skip_cleanup: true + on: + branch: master before_install: - bash ./scripts/ci/before_install.sh