From a768fd13ff8ad93444b9f5879963e227b90389b9 Mon Sep 17 00:00:00 2001 From: n3hrox Date: Sat, 26 Oct 2019 22:14:14 +0200 Subject: [PATCH 01/64] logger: output footer once for multithreaded push/pull errors --- dvc/remote/base.py | 8 ++------ dvc/remote/local.py | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index f9bd7ef76..30ec6ed6b 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -515,9 +515,7 @@ class RemoteBASE(object): no_progress_bar=no_progress_bar, ) except Exception: - msg = "failed to upload '{}' to '{}'" - logger.exception(msg.format(from_info, to_info)) - return 1 # 1 fail + return 1, from_info, to_info, "upload" # 1 fail return 0 @@ -555,9 +553,7 @@ class RemoteBASE(object): from_info, tmp_file, name=name, no_progress_bar=no_progress_bar ) except Exception: - msg = "failed to download '{}' to '{}'" - logger.exception(msg.format(from_info, to_info)) - return 1 # 1 fail + return 1, from_info, to_info, "download" # 1 fail move(tmp_file, to_info, mode=file_mode) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index dff433756..f9cdb5cdf 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -358,11 +358,24 @@ class RemoteLOCAL(RemoteBASE): if len(plans[0]) == 0: return 0 + fails = 0 + msgs = "" if jobs > 1: with TqdmThreadPoolExecutor(max_workers=jobs) as executor: - fails = sum(executor.map(func, *plans)) + failures = executor.map(func, *plans) else: - fails = sum(map(func, *plans)) + failures = list(map(func, *plans)) + + for it in failures: + if not isinstance(it, tuple): + continue + fails += it[0] + from_info, to_info = it[1], it[2] + operation = it[3] + msgs += f"Failed to {operation} '{from_info}' to '{to_info}'\n" + + if msgs: + logger.exception(msgs) if fails: if download: From 2e12af095eb0df7034eacb2e554b98bc5437ee68 Mon Sep 17 00:00:00 2001 From: n3hrox Date: Sat, 26 Oct 2019 22:21:58 +0200 Subject: [PATCH 02/64] Fix improper variable name --- dvc/remote/local.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index f9cdb5cdf..fc11aee28 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -366,12 +366,12 @@ class RemoteLOCAL(RemoteBASE): else: failures = list(map(func, *plans)) - for it in failures: - if not isinstance(it, tuple): + for failure in failures: + if not isinstance(failure, tuple): continue - fails += it[0] - from_info, to_info = it[1], it[2] - operation = it[3] + fails += failure[0] + from_info, to_info = failure[1], failure[2] + operation = failure[3] msgs += f"Failed to {operation} '{from_info}' to '{to_info}'\n" if msgs: From 64f78c164dbfb6a1f73a16ffcd0ef172f4614828 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 26 Oct 2019 23:35:22 +0300 Subject: [PATCH 03/64] Revert "Added path.lstrip for / in imports (#2651)" (#2675) This reverts commit bfc0a3596f9ea633dc7037de07be958b7f322525. --- dvc/repo/get.py | 1 - dvc/repo/imp.py | 1 - 2 files changed, 2 deletions(-) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 258f6d884..9096216c9 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -21,7 +21,6 @@ logger = logging.getLogger(__name__) @staticmethod def get(url, path, out=None, rev=None): out = resolve_output(path, out) - path = path.lstrip("/") if Stage.is_valid_filename(out): raise GetDVCFileError() diff --git a/dvc/repo/imp.py b/dvc/repo/imp.py index 270d4caa6..68ead389b 100644 --- a/dvc/repo/imp.py +++ b/dvc/repo/imp.py @@ -2,6 +2,5 @@ def imp(self, url, path, out=None, rev=None): erepo = {"url": url} if rev is not None: erepo["rev"] = rev - path = path.lstrip("/") return self.imp_url(path, out=out, erepo=erepo, locked=True) From b9c7d88fccd379f5830b2f01cdea8d011f950afb Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 26 Oct 2019 23:36:13 +0300 Subject: [PATCH 04/64] dvc: bump to 0.66.1 --- dvc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/version.py b/dvc/version.py index 4fc71199c..95b3643ad 100644 --- a/dvc/version.py +++ b/dvc/version.py @@ -7,7 +7,7 @@ import os import subprocess -_BASE_VERSION = "0.66.0" +_BASE_VERSION = "0.66.1" def _generate_version(base_version): From 98bafa996834b8c7b606db3a806918ef517e7a26 Mon Sep 17 00:00:00 2001 From: n3hrox Date: Sat, 26 Oct 2019 23:01:34 +0200 Subject: [PATCH 05/64] Change f-string to old format --- dvc/remote/local.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index fc11aee28..c66a26be3 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -372,7 +372,9 @@ class RemoteLOCAL(RemoteBASE): fails += failure[0] from_info, to_info = failure[1], failure[2] operation = failure[3] - msgs += f"Failed to {operation} '{from_info}' to '{to_info}'\n" + msgs += "Failed to {} '{}' to '{}'\n".format( + operation, from_info, to_info + ) if msgs: logger.exception(msgs) From 5f644745c7b19a02b26aead1dbda62b92c976d3e Mon Sep 17 00:00:00 2001 From: Aman Sharma Date: Mon, 28 Oct 2019 20:13:47 +0530 Subject: [PATCH 06/64] Write test for `get_inode` (#2673) --- tests/unit/utils/test_fs.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/utils/test_fs.py b/tests/unit/utils/test_fs.py index 30a96b63f..97c1ad29b 100644 --- a/tests/unit/utils/test_fs.py +++ b/tests/unit/utils/test_fs.py @@ -14,6 +14,7 @@ from dvc.utils.fs import ( contains_symlink_up_to, BasePathNotInCheckedPathException, get_parent_dirs_up_to, + get_inode, ) from mock import patch from tests.basic_env import TestDir @@ -149,3 +150,9 @@ def test_relpath_windows_different_drives(): rel_info = relpath(info1, info2) assert isinstance(rel_info, str) assert rel_info == path1 + + +def test_get_inode(repo_dir): + path = repo_dir.FOO + path_info = PathInfo(path) + assert get_inode(path) == get_inode(path_info) From 5d6918a04ce59076caa4a69ff501ef4b25fdefd2 Mon Sep 17 00:00:00 2001 From: Sagun Karanjit Date: Mon, 28 Oct 2019 21:12:34 +0545 Subject: [PATCH 07/64] Remove --all-commits options from usage output (#2682) * Comment out --all-commits optional arg * Suppress --all-commits optional arg --- dvc/command/gc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/command/gc.py b/dvc/command/gc.py index 699f88304..45d774f6d 100644 --- a/dvc/command/gc.py +++ b/dvc/command/gc.py @@ -83,7 +83,7 @@ def add_parser(subparsers, parent_parser): "--all-commits", action="store_true", default=False, - help="Keep data files for all commits.", + help=argparse.SUPPRESS, ) gc_parser.add_argument( "-c", From a48481b01fd607b36f9ea5b643af24c938254f60 Mon Sep 17 00:00:00 2001 From: Jorge Orpinel Date: Mon, 28 Oct 2019 14:00:53 -0600 Subject: [PATCH 08/64] dvc: change ` for ' in dynamic parts of output messages (#2644) * dvc: change ` for ' in dinamic parts of DVC outputs per https://github.com/iterative/dvc/pull/2644#issuecomment-545151398 * dvc: use ' for dynamic output and ` for DVC names per https://github.com/iterative/dvc/pull/2644#discussion_r338417698 --- dvc/command/destroy.py | 2 +- dvc/command/imp_url.py | 2 +- dvc/command/init.py | 2 +- dvc/command/remove.py | 2 +- dvc/command/run.py | 6 +++--- dvc/exceptions.py | 6 +++--- dvc/remote/slow_link_detection.py | 2 +- dvc/repo/init.py | 4 ++-- dvc/scm/git/__init__.py | 8 ++++---- dvc/stage.py | 12 ++++++------ dvc/utils/__init__.py | 2 +- tests/unit/command/test_imp_url.py | 4 ++-- 12 files changed, 26 insertions(+), 26 deletions(-) diff --git a/dvc/command/destroy.py b/dvc/command/destroy.py index ca739de4a..3dc251387 100644 --- a/dvc/command/destroy.py +++ b/dvc/command/destroy.py @@ -24,7 +24,7 @@ class CmdDestroy(CmdBase): if not self.args.force and not prompt.confirm(statement): raise DvcException( "cannot destroy without a confirmation from the user." - " Use '-f' to force." + " Use `-f` to force." ) self.repo.destroy() diff --git a/dvc/command/imp_url.py b/dvc/command/imp_url.py index 012ea80c0..1b0b1cf87 100644 --- a/dvc/command/imp_url.py +++ b/dvc/command/imp_url.py @@ -19,7 +19,7 @@ class CmdImportUrl(CmdBase): except DvcException: logger.exception( "failed to import {}. You could also try downloading " - "it manually and adding it with `dvc add` command.".format( + "it manually, and adding it with `dvc add`.".format( self.args.url ) ) diff --git a/dvc/command/init.py b/dvc/command/init.py index 0e9bdf83f..075e3be4c 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -53,7 +53,7 @@ def add_parser(subparsers, parent_parser): action="store_true", default=False, help=( - "Overwrite existing '.dvc' directory. " + "Overwrite existing '.dvc/' directory. " "This operation removes local cache." ), ) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index 814adcfae..a00448029 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -28,7 +28,7 @@ class CmdRemove(CmdBase): raise DvcException( "Cannot purge without a confirmation from the user." - " Use '-f' to force." + " Use `-f` to force." ) def run(self): diff --git a/dvc/command/run.py b/dvc/command/run.py index b0761435f..c51f10bab 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -27,9 +27,9 @@ class CmdRun(CmdBase): ] ): # pragma: no cover logger.error( - "too few arguments. Specify at least one: '-d', '-o', '-O'," - " '-m', '-M', '--outs-persist', '--outs-persist-no-cache'," - " 'command'." + "too few arguments. Specify at least one: `-d`, `-o`, `-O`, " + "`-m`, `-M`, `--outs-persist`, `--outs-persist-no-cache`, " + "`command`." ) return 1 diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 2309bcdbf..16b8b837b 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -179,7 +179,7 @@ class ConfirmRemoveError(DvcException): def __init__(self, path): super(ConfirmRemoveError, self).__init__( "unable to remove '{}' without a confirmation from the user. Use " - "'-f' to force.".format(path) + "`-f` to force.".format(path) ) @@ -209,7 +209,7 @@ class NoMetricsError(DvcException): def __init__(self): super(NoMetricsError, self).__init__( "no metric files in this repository. " - "Use 'dvc metrics add' to add a metric file to track." + "Use `dvc metrics add` to add a metric file to track." ) @@ -247,7 +247,7 @@ class OverlappingOutputPathsError(DvcException): class CheckoutErrorSuggestGit(DvcException): def __init__(self, target, cause): super(CheckoutErrorSuggestGit, self).__init__( - "Did you mean 'git checkout {}'?".format(target), cause=cause + "Did you mean `git checkout {}`?".format(target), cause=cause ) diff --git a/dvc/remote/slow_link_detection.py b/dvc/remote/slow_link_detection.py index 0dd2285d6..bffa6e9e8 100644 --- a/dvc/remote/slow_link_detection.py +++ b/dvc/remote/slow_link_detection.py @@ -18,7 +18,7 @@ this.message = ( "See {blue}https://dvc.org/doc/commands-reference/config#cache{reset} " "for more information.\n" "To disable this message, run:\n" - "'dvc config cache.slow_link_warning false'".format( + "`dvc config cache.slow_link_warning false`".format( blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET ) ) diff --git a/dvc/repo/init.py b/dvc/repo/init.py index 19dc20d1b..e2f6b4702 100644 --- a/dvc/repo/init.py +++ b/dvc/repo/init.py @@ -65,7 +65,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False): if isinstance(scm, NoSCM) and not no_scm: raise InitError( "{repo} is not tracked by any supported scm tool (e.g. git). " - "Use '--no-scm' if you don't want to use any scm.".format( + "Use `--no-scm` if you don't want to use any scm.".format( repo=root_dir ) ) @@ -73,7 +73,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False): if os.path.isdir(dvc_dir): if not force: raise InitError( - "'{repo}' exists. Use '-f' to force.".format( + "'{repo}' exists. Use `-f` to force.".format( repo=relpath(dvc_dir) ) ) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 05bb2faf1..fd4e7ad10 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -200,10 +200,10 @@ class Git(Base): self.repo.index.add(paths) except AssertionError: msg = ( - "failed to add '{}' to git. You can add those files" - " manually using 'git add'." - " See 'https://github.com/iterative/dvc/issues/610'" - " for more details.".format(str(paths)) + "failed to add '{}' to git. You can add those files " + "manually using `git add`. See " + "https://github.com/iterative/dvc/issues/610 for more " + "details.".format(str(paths)) ) logger.exception(msg) diff --git a/dvc/stage.py b/dvc/stage.py index 4d3b7d3d0..94ec95c21 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -242,7 +242,7 @@ class Stage(object): if self.is_callback: logger.warning( - "DVC-file '{fname}' is a 'callback' stage " + "DVC-file '{fname}' is a \"callback\" stage " "(has a command and no dependencies) and thus always " "considered as changed.".format(fname=self.relpath) ) @@ -451,8 +451,8 @@ class Stage(object): if fname is not None and os.path.basename(fname) != fname: raise StageFileBadNameError( "DVC-file name '{fname}' may not contain subdirectories" - " if '-c|--cwd' (deprecated) is specified. Use '-w|--wdir'" - " along with '-f' to specify DVC-file path with working" + " if `-c|--cwd` (deprecated) is specified. Use `-w|--wdir`" + " along with `-f` to specify DVC-file path with working" " directory.".format(fname=fname) ) wdir = cwd @@ -571,8 +571,8 @@ class Stage(object): def _check_dvc_filename(fname): if not Stage.is_valid_filename(fname): raise StageFileBadNameError( - "bad DVC-file name '{}'. DVC-files should be named" - " 'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format( + "bad DVC-file name '{}'. DVC-files should be named " + "'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format( relpath(fname), os.path.basename(fname) ) ) @@ -736,7 +736,7 @@ class Stage(object): if not force and not prompt.confirm(msg): raise StageCommitError( "unable to commit changed '{}'. Use `-f|--force` to " - "force.`".format(self.relpath) + "force.".format(self.relpath) ) self.save() diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 538b7a97e..b0cb6b3ff 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -232,7 +232,7 @@ def _to_chunks_by_chunks_number(list_to_split, num_chunks): def to_chunks(list_to_split, num_chunks=None, chunk_size=None): if (num_chunks and chunk_size) or (not num_chunks and not chunk_size): raise ValueError( - "One and only one of 'num_chunks', 'chunk_size' must be defined" + "Only one among `num_chunks` or `chunk_size` must be defined." ) if chunk_size: return _split(list_to_split, chunk_size) diff --git a/tests/unit/command/test_imp_url.py b/tests/unit/command/test_imp_url.py index 3ec0efbaa..50cb6e5bd 100644 --- a/tests/unit/command/test_imp_url.py +++ b/tests/unit/command/test_imp_url.py @@ -29,7 +29,7 @@ def test_failed_import_url(mocker, caplog, dvc_repo): assert cmd.run() == 1 expected_error = ( "failed to import http://somesite.com/file_name. " - "You could also try downloading it manually and " - "adding it with `dvc add` command." + "You could also try downloading it manually, and " + "adding it with `dvc add`." ) assert expected_error in caplog.text From 877e38551eca7b8d88155b4ae97dd1237c99b0d0 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 29 Oct 2019 20:54:56 +0200 Subject: [PATCH 09/64] lock: add a workaround for a flufl.lock bug (#2688) This bug doesn't affect us during regular dvc CLI commands, but is not visualy pleasing during API usage. Fixes #2573 --- dvc/lock.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dvc/lock.py b/dvc/lock.py index 8859ff9e7..d3bc83d19 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -87,6 +87,17 @@ if is_py3: self._tmp_dir, filename + ".lock" ) + # Fix for __del__ bug in flufl.lock [1] which is causing errors on + # Python shutdown [2]. + # [1] https://gitlab.com/warsaw/flufl.lock/issues/7 + # [2] https://github.com/iterative/dvc/issues/2573 + def __del__(self): + try: + if self._owned: + self.finalize() + except ImportError: + pass + else: import zc.lockfile From 77fcd4969557f91c9019e75f9b0c51ed3874a0f8 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Wed, 30 Oct 2019 15:43:18 +0100 Subject: [PATCH 10/64] dvc: fix tqdm dep (#2694) tqdm==4.35.0 was throwing KeyError: 'percentage'. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 47bd6c46b..1780637a6 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ install_requires = [ "funcy>=1.12", "pathspec>=0.6.0", "shortuuid>=0.5.0", - "tqdm>=4.35.0,<5", + "tqdm>=4.36.1,<5", "packaging>=19.0", "win-unicode-console>=0.5; sys_platform == 'win32'", ] From 2ec0aa0c37c9a885c177fdeaefa1c361adff2337 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Wed, 30 Oct 2019 08:18:07 -0700 Subject: [PATCH 11/64] failing while pulling imported directories (#2692) * pull: fix-2684 - failing while pulling imported directories * pull: fix-2684 - fix code indentation * minor imported directories pull test update Co-Authored-By: Ruslan Kuprieiev --- dvc/output/base.py | 2 +- dvc/repo/fetch.py | 26 +++++++++++++------------- tests/func/test_import.py | 18 ++++++++++++++++++ 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/dvc/output/base.py b/dvc/output/base.py index f940d3f52..8fa3ad367 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -334,7 +334,7 @@ class OutputBase(object): self.remote.unprotect(self.path_info) def _collect_used_dir_cache(self, remote=None, force=False, jobs=None): - """Get a list of `info`s retaled to the given directory. + """Get a list of `info`s related to the given directory. - Pull the directory entry from the remote cache if it was changed. diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 702946163..b69c8c04b 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -76,20 +76,20 @@ def _fetch_external(self, repo_url, repo_rev, files): cache_dir = self.cache.local.cache_dir try: with external_repo(repo_url, repo_rev, cache_dir=cache_dir) as repo: - cache = NamedCache() - for name in files: - try: - out = repo.find_out_by_relpath(name) - except OutputNotFoundError: - failed += 1 - logger.exception( - "failed to fetch data for '{}'".format(name) - ) - continue - else: - cache.update(out.get_used_cache()) - with repo.state: + cache = NamedCache() + for name in files: + try: + out = repo.find_out_by_relpath(name) + except OutputNotFoundError: + failed += 1 + logger.exception( + "failed to fetch data for '{}'".format(name) + ) + continue + else: + cache.update(out.get_used_cache()) + try: return repo.cloud.pull(cache), failed except DownloadError as exc: diff --git a/tests/func/test_import.py b/tests/func/test_import.py index fc00ee7b1..4fe8857aa 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import os import filecmp +import shutil import pytest from mock import patch @@ -67,6 +68,23 @@ def test_pull_imported_stage(dvc_repo, erepo): assert os.path.isfile(dst_cache) +def test_pull_imported_directory_stage(dvc_repo, erepo): + src = erepo.DATA_DIR + dst = erepo.DATA_DIR + "_imported" + stage_file = dst + ".dvc" + + dvc_repo.imp(erepo.root_dir, src, dst) + + shutil.rmtree(dst) + shutil.rmtree(dvc_repo.cache.local.cache_dir) + + dvc_repo.pull([stage_file]) + + assert os.path.exists(dst) + assert os.path.isdir(dst) + trees_equal(src, dst) + + def test_download_error_pulling_imported_stage(dvc_repo, erepo): src = erepo.FOO dst = erepo.FOO + "_imported" From e623c4faf782ee6f0d774552e8f786f6b8387e95 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 30 Oct 2019 17:20:05 +0200 Subject: [PATCH 12/64] dvc: bump to 0.66.2 --- dvc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/version.py b/dvc/version.py index 95b3643ad..ab922547f 100644 --- a/dvc/version.py +++ b/dvc/version.py @@ -7,7 +7,7 @@ import os import subprocess -_BASE_VERSION = "0.66.1" +_BASE_VERSION = "0.66.2" def _generate_version(base_version): From 732ed904ceec8c0689827046e4d2e4bb4bb80296 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Wed, 30 Oct 2019 11:42:33 -0600 Subject: [PATCH 13/64] tests: same ETag across buckets --- tests/unit/remote/test_s3.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unit/remote/test_s3.py b/tests/unit/remote/test_s3.py index 7861fb5a9..36b9d1e23 100644 --- a/tests/unit/remote/test_s3.py +++ b/tests/unit/remote/test_s3.py @@ -83,3 +83,20 @@ def test_walk_files(remote): ] assert list(remote.walk_files(remote.path_info / "data")) == files + + +def test_copy_preserve_etag_across_buckets(remote): + s3 = remote.s3 + s3.create_bucket(Bucket="another") + + another = RemoteS3(None, {"url": "s3://another", "region": "us-east-1"}) + + from_info = remote.path_info / "foo" + to_info = another.path_info / "foo" + + remote.copy(from_info, to_info) + + from_etag = RemoteS3.get_etag(s3, "bucket", "foo") + to_etag = RemoteS3.get_etag(s3, "another", "foo") + + assert from_etag == to_etag From b9a17752e3e989aff0ec4312af270ed49754e680 Mon Sep 17 00:00:00 2001 From: Aman Sharma Date: Thu, 31 Oct 2019 00:23:18 +0530 Subject: [PATCH 14/64] Test acceptable path types of `get_mtime_and_size` --- tests/unit/utils/test_fs.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unit/utils/test_fs.py b/tests/unit/utils/test_fs.py index 97c1ad29b..5c8849903 100644 --- a/tests/unit/utils/test_fs.py +++ b/tests/unit/utils/test_fs.py @@ -39,6 +39,15 @@ class TestMtimeAndSize(TestDir): self.assertIs(type(dir_size), str) self.assertEqual(dir_size, str(actual_dir_size)) + def test_path_object_and_str_are_valid_types(self): + dvcignore = DvcIgnoreFilter(self.root_dir) + file_time, file_size = get_mtime_and_size(self.DATA, dvcignore) + file_object_time, file_object_size = get_mtime_and_size( + PathInfo(self.DATA), dvcignore + ) + self.assertEqual(file_time, file_object_time) + self.assertEqual(file_size, file_object_size) + class TestContainsLink(TestCase): def test_should_raise_exception_on_base_path_not_in_path(self): From 98750c95c506a2ad5cc3e86f5014a0a4ca3313cb Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 31 Oct 2019 09:58:07 -0700 Subject: [PATCH 15/64] get-url: fix NPE when src non local and dst not absolute --- dvc/output/local.py | 3 ++- dvc/repo/get_url.py | 3 ++- tests/func/test_get_url.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/dvc/output/local.py b/dvc/output/local.py index 777ac3756..ea0cb9d18 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -26,9 +26,10 @@ class OutputLOCAL(OutputBase): # so we should expect both posix and windows style paths. # PathInfo accepts both, i.e. / works everywhere, \ only on win. # - # FIXME: if we have Windows path containig / or posix one with \ + # FIXME: if we have Windows path containing / or posix one with \ # then we have #2059 bug and can't really handle that. p = self.REMOTE.path_cls(path) + if not p.is_absolute(): p = self.stage.wdir / p diff --git a/dvc/repo/get_url.py b/dvc/repo/get_url.py index ff191e149..8ba187865 100644 --- a/dvc/repo/get_url.py +++ b/dvc/repo/get_url.py @@ -12,7 +12,8 @@ def get_url(url, out=None): if os.path.exists(url): url = os.path.abspath(url) - out = os.path.abspath(out) + + out = os.path.abspath(out) dep, = dependency.loads_from(None, [url]) out, = output.loads_from(None, [out], use_cache=False) diff --git a/tests/func/test_get_url.py b/tests/func/test_get_url.py index eae2b556e..fab7a4db6 100644 --- a/tests/func/test_get_url.py +++ b/tests/func/test_get_url.py @@ -1,13 +1,19 @@ from __future__ import unicode_literals import os +import boto3 import filecmp import pytest +from moto import mock_s3 + +from dvc.remote import RemoteS3 from dvc.repo import Repo from dvc.utils import makedirs +from tests.func.test_data_cloud import get_aws_url + def test_get_file(repo_dir): src = repo_dir.FOO @@ -32,3 +38,27 @@ def test_get_url_to_dir(dname, repo_dir): assert os.path.isdir(dname) assert filecmp.cmp(repo_dir.DATA, dst, shallow=False) + + +@mock_s3 +@pytest.mark.parametrize("dst", [".", "./from"]) +def test_get_url_from_non_local_path_to_dir_and_file(repo_dir, dst): + file_name = "from" + file_content = "data" + base_info = RemoteS3.path_cls(get_aws_url()) + from_info = base_info / file_name + + s3 = boto3.client("s3") + s3.create_bucket(Bucket=from_info.bucket) + s3.put_object( + Bucket=from_info.bucket, Key=from_info.path, Body=file_content + ) + + Repo.get_url(from_info.url, dst) + + result_path = os.path.join(dst, file_name) if os.path.isdir(dst) else dst + + assert os.path.exists(result_path) + assert os.path.isfile(result_path) + with open(result_path, "r") as fd: + assert fd.read() == file_content From 31551e6e2e4534047c0817111422af3489d95ca5 Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Tue, 29 Oct 2019 19:13:17 +0000 Subject: [PATCH 16/64] CLI: change return output for 'checkout' and 'pull' commands Change messages output after return of some commands: * Remove warning for 'dvc checkout' & commands calling it under the hood * Add an informational message output for 'dvc pull' command Fixes #2685. --- dvc/repo/checkout.py | 6 ------ dvc/repo/pull.py | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 6f8abde42..f753c1419 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -50,12 +50,6 @@ def _checkout( total=total, unit="file", desc="Checkout", disable=total == 0 ) as pbar: for stage in stages: - if stage.locked: - logger.warning( - "DVC-file '{path}' is locked. Its dependencies are" - " not going to be checked out.".format(path=stage.relpath) - ) - failed.extend( stage.checkout(force=force, progress_callback=pbar.update_desc) ) diff --git a/dvc/repo/pull.py b/dvc/repo/pull.py index 7c9258824..6fbce8851 100644 --- a/dvc/repo/pull.py +++ b/dvc/repo/pull.py @@ -1,6 +1,11 @@ from __future__ import unicode_literals -from . import locked +import logging + +from dvc.repo import locked + + +logger = logging.getLogger(__name__) @locked @@ -20,11 +25,16 @@ def pull( jobs, remote=remote, all_branches=all_branches, - all_tags=all_tags, with_deps=with_deps, + all_tags=all_tags, recursive=recursive, ) self._checkout( - targets=targets, with_deps=with_deps, force=force, recursive=recursive + targets=targets, + with_deps=with_deps, + force=force, + recursive=recursive, + ) + logger.info("Data retrieved successfully from DVC remote storage.") return processed_files_count From 0dd4d2361eb8684b67790558303f54c0af0910ae Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 1 Nov 2019 16:40:06 +0200 Subject: [PATCH 17/64] fix formatting --- dvc/repo/pull.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dvc/repo/pull.py b/dvc/repo/pull.py index 6fbce8851..9624fbc4c 100644 --- a/dvc/repo/pull.py +++ b/dvc/repo/pull.py @@ -30,11 +30,7 @@ def pull( recursive=recursive, ) self._checkout( - targets=targets, - with_deps=with_deps, - force=force, - recursive=recursive, - + targets=targets, with_deps=with_deps, force=force, recursive=recursive ) logger.info("Data retrieved successfully from DVC remote storage.") return processed_files_count From 5c8a312002430ff8bafd9d7bee1e10e8784ed333 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 1 Nov 2019 16:41:22 +0200 Subject: [PATCH 18/64] fix order --- dvc/repo/pull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/repo/pull.py b/dvc/repo/pull.py index 9624fbc4c..cd052b651 100644 --- a/dvc/repo/pull.py +++ b/dvc/repo/pull.py @@ -25,8 +25,8 @@ def pull( jobs, remote=remote, all_branches=all_branches, - with_deps=with_deps, all_tags=all_tags, + with_deps=with_deps, recursive=recursive, ) self._checkout( From c999b5d270e90b646fbfdb3cb147c172d07af60e Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 1 Nov 2019 16:45:41 +0200 Subject: [PATCH 19/64] remove info message --- dvc/repo/pull.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dvc/repo/pull.py b/dvc/repo/pull.py index cd052b651..822897e29 100644 --- a/dvc/repo/pull.py +++ b/dvc/repo/pull.py @@ -32,5 +32,4 @@ def pull( self._checkout( targets=targets, with_deps=with_deps, force=force, recursive=recursive ) - logger.info("Data retrieved successfully from DVC remote storage.") return processed_files_count From 3bf810a832e0241d2c2c0674621eff1b1008a7a2 Mon Sep 17 00:00:00 2001 From: n3hrox Date: Fri, 1 Nov 2019 16:36:31 +0100 Subject: [PATCH 20/64] move footer from logger to main --- dvc/logger.py | 16 +--------------- dvc/main.py | 14 ++++++++++++++ dvc/remote/base.py | 8 ++++++-- dvc/remote/local.py | 19 ++----------------- 4 files changed, 23 insertions(+), 34 deletions(-) diff --git a/dvc/logger.py b/dvc/logger.py index 71f9d9147..075b1b680 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -48,16 +48,6 @@ class ColorFormatter(logging.Formatter): "CRITICAL": colorama.Fore.RED, } - footer = ( - "{yellow}Having any troubles?{nc}" - " Hit us up at {blue}https://dvc.org/support{nc}," - " we are always happy to help!" - ).format( - blue=colorama.Fore.BLUE, - nc=colorama.Fore.RESET, - yellow=colorama.Fore.YELLOW, - ) - def format(self, record): if record.levelname == "INFO": return record.msg @@ -66,10 +56,7 @@ class ColorFormatter(logging.Formatter): exception, stack_trace = self._parse_exc(record.exc_info) return ( - "{color}{levelname}{nc}: {description}" - "{stack_trace}\n" - "\n" - "{footer}" + "{color}{levelname}{nc}: {description}" "{stack_trace}\n" ).format( color=self.color_code.get(record.levelname, ""), nc=colorama.Fore.RESET, @@ -77,7 +64,6 @@ class ColorFormatter(logging.Formatter): description=self._description(record.msg, exception), msg=record.msg, stack_trace=stack_trace, - footer=self.footer, ) return "{color}{levelname}{nc}: {msg}".format( diff --git a/dvc/main.py b/dvc/main.py index f8b94b8d3..d21a51592 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import logging +import colorama from dvc.cli import parse_args from dvc.lock import LockError @@ -16,6 +17,16 @@ from dvc.utils.compat import is_py2 logger = logging.getLogger("dvc") +footer = ( + "\n{yellow}Having any troubles?{nc}" + " Hit us up at {blue}https://dvc.org/support{nc}," + " we are always happy to help!" +).format( + blue=colorama.Fore.BLUE, + nc=colorama.Fore.RESET, + yellow=colorama.Fore.YELLOW, +) + def main(argv=None): """Run dvc CLI command. @@ -75,6 +86,9 @@ def main(argv=None): # so won't be reused by any other subsequent run anyway. clean_repos() + if ret != 0: + logger.info(footer) + Analytics().send_cmd(cmd, args, ret) return ret diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 30ec6ed6b..cead3293a 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -515,7 +515,9 @@ class RemoteBASE(object): no_progress_bar=no_progress_bar, ) except Exception: - return 1, from_info, to_info, "upload" # 1 fail + msg = "failed to upload '{}' to '{}'" + logger.exception(msg.format(from_info, to_info)) + return 1 # 1 fail return 0 @@ -553,7 +555,9 @@ class RemoteBASE(object): from_info, tmp_file, name=name, no_progress_bar=no_progress_bar ) except Exception: - return 1, from_info, to_info, "download" # 1 fail + msg = "failed to upload '{}' to '{}'" + logger.exception(msg.format(from_info, to_info)) + return 1 # 1 fail move(tmp_file, to_info, mode=file_mode) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index c66a26be3..dff433756 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -358,26 +358,11 @@ class RemoteLOCAL(RemoteBASE): if len(plans[0]) == 0: return 0 - fails = 0 - msgs = "" if jobs > 1: with TqdmThreadPoolExecutor(max_workers=jobs) as executor: - failures = executor.map(func, *plans) + fails = sum(executor.map(func, *plans)) else: - failures = list(map(func, *plans)) - - for failure in failures: - if not isinstance(failure, tuple): - continue - fails += failure[0] - from_info, to_info = failure[1], failure[2] - operation = failure[3] - msgs += "Failed to {} '{}' to '{}'\n".format( - operation, from_info, to_info - ) - - if msgs: - logger.exception(msgs) + fails = sum(map(func, *plans)) if fails: if download: From 8ae1165a9f659b8b258e0e6737b221fa00d3fa24 Mon Sep 17 00:00:00 2001 From: n3hrox Date: Fri, 1 Nov 2019 16:38:23 +0100 Subject: [PATCH 21/64] fix typo upload -> download --- dvc/remote/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index cead3293a..f9bd7ef76 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -555,7 +555,7 @@ class RemoteBASE(object): from_info, tmp_file, name=name, no_progress_bar=no_progress_bar ) except Exception: - msg = "failed to upload '{}' to '{}'" + msg = "failed to download '{}' to '{}'" logger.exception(msg.format(from_info, to_info)) return 1 # 1 fail From b83564d80e0b03aae4b7aebd813480078c7485ba Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Fri, 1 Nov 2019 16:51:11 +0100 Subject: [PATCH 22/64] perf: refactor URLInfo to not use ParseResult inside This permites URLInfo construction from parts without building/reparsing string, which makes it around 20% faster. Code is also simpler and a bit shorter now. Other improvements: - added URLInfo.replace(path=...) - URLInfo._path now stringifies properly - protected against passing path not starting with / - do not allow query, params, fragments and passwords in urls (they were silently ignored previously) --- dvc/path_info.py | 169 ++++++++++++++++++------------------- dvc/remote/s3.py | 3 +- dvc/remote/ssh/__init__.py | 3 +- 3 files changed, 83 insertions(+), 92 deletions(-) diff --git a/dvc/path_info.py b/dvc/path_info.py index 3db44a439..6707c8d81 100644 --- a/dvc/path_info.py +++ b/dvc/path_info.py @@ -8,13 +8,12 @@ from funcy import cached_property from dvc.utils.compat import str, builtin_str, basestring, is_py2 from dvc.utils.compat import pathlib, urlparse +from dvc.utils import relpath # On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs, # which is lossy, thus we can't use that, # see https://github.com/mcmtroffaes/pathlib2/issues/56. -from dvc.utils import relpath - if is_py2: fs_encoding = "utf-8" @@ -112,52 +111,80 @@ class PosixPathInfo(PathInfo, pathlib.PurePosixPath): pass +class _URLPathInfo(PosixPathInfo): + def __str__(self): + return self.__fspath__() + + __unicode__ = __str__ + + class _URLPathParents(object): - def __init__(self, pathcls, scheme, netloc, path): - self._scheme = scheme - self._netloc = netloc - self._parents = path.parents - self._pathcls = pathcls + def __init__(self, src): + self.src = src + self._parents = self.src._path.parents def __len__(self): return len(self._parents) def __getitem__(self, idx): - return self._pathcls.from_parts( - scheme=self._scheme, - netloc=self._netloc, - path=self._parents[idx].fspath, - ) + return self.src.replace(path=self._parents[idx]) def __repr__(self): - return "<{}.parents>".format(self._pathcls.__name__) + return "<{}.parents>".format(self.src) class URLInfo(object): DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22, "hdfs": 0} def __init__(self, url): - self.parsed = urlparse(url) - assert self.parsed.scheme != "remote" + p = urlparse(url) + assert not p.query and not p.params and not p.fragment + assert p.password is None + + self.fill_parts(p.scheme, p.hostname, p.username, p.port, p.path) @classmethod def from_parts( - cls, scheme=None, netloc=None, host=None, user=None, port=None, path="" + cls, scheme=None, host=None, user=None, port=None, path="", netloc=None ): - assert scheme and (bool(host) ^ bool(netloc)) + assert bool(host) ^ bool(netloc) - if netloc is None: - netloc = host - if user: - netloc = user + "@" + host - if port: - netloc += ":" + str(port) - return cls("{}://{}{}".format(scheme, netloc, path)) + if netloc is not None: + return cls("{}://{}{}".format(scheme, netloc, path)) + + obj = cls.__new__(cls) + obj.fill_parts(scheme, host, user, port, path) + return obj + + def fill_parts(self, scheme, host, user, port, path): + assert scheme != "remote" + assert isinstance(path, (basestring, _URLPathInfo)) + + self.scheme, self.host, self.user = scheme, host, user + self.port = int(port) if port else self.DEFAULT_PORTS.get(self.scheme) + + if isinstance(path, _URLPathInfo): + self._spath = builtin_str(path) + self._path = path + else: + if path and path[0] != "/": + path = "/" + path + self._spath = path + + @property + def _base_parts(self): + return (self.scheme, self.host, self.user, self.port) + + @property + def parts(self): + return self._base_parts + self._path.parts + + def replace(self, path=None): + return self.from_parts(*self._base_parts, path=path) @cached_property def url(self): - p = self.parsed - return "{}://{}{}".format(p.scheme, self.netloc, p.path) + return "{}://{}{}".format(self.scheme, self.netloc, self._spath) def __str__(self): return self.url @@ -170,92 +197,60 @@ class URLInfo(object): other = self.__class__(other) return ( self.__class__ == other.__class__ - and self.scheme == other.scheme - and self.netloc == other.netloc + and self._base_parts == other._base_parts and self._path == other._path ) def __hash__(self): - return hash(self.url) + return hash(self.parts) def __div__(self, other): - p = self.parsed - new_path = posixpath.join(p.path, str(other)) - if not new_path.startswith("/"): - new_path = "/" + new_path - new_url = "{}://{}{}".format(p.scheme, p.netloc, new_path) - return self.__class__(new_url) + return self.replace(path=posixpath.join(self._spath, other)) __truediv__ = __div__ - def __getattr__(self, name): - # When deepcopy is called, it creates and object without __init__, - # self.parsed is not initialized and it causes infinite recursion. - # More on this special casing here: - # https://stackoverflow.com/a/47300262/298182 - if name.startswith("__"): - raise AttributeError(name) - return getattr(self.parsed, name) - - @cached_property - def netloc(self): - p = self.parsed - netloc = p.hostname - if p.username: - netloc = p.username + "@" + netloc - if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme): - netloc += ":" + str(p.port) - return netloc - @property - def port(self): - return self.parsed.port or self.DEFAULT_PORTS.get(self.parsed.scheme) - - @property - def host(self): - return self.parsed.hostname - - @property - def user(self): - return self.parsed.username + def path(self): + return self._spath @cached_property def _path(self): - return PosixPathInfo(self.parsed.path) + return _URLPathInfo(self._spath) @property def name(self): return self._path.name - @property - def parts(self): - return (self.scheme, self.netloc) + self._path.parts + @cached_property + def netloc(self): + netloc = self.host + if self.user: + netloc = self.user + "@" + netloc + if self.port and int(self.port) != self.DEFAULT_PORTS.get(self.scheme): + netloc += ":" + str(self.port) + return netloc @property def bucket(self): - return self.parsed.netloc + return self.netloc @property def parent(self): - return self.from_parts( - scheme=self.scheme, - netloc=self.parsed.netloc, - path=self._path.parent.fspath, - ) + return self.replace(path=self._path.parent) @property def parents(self): - return _URLPathParents( - type(self), self.scheme, self.parsed.netloc, self._path - ) + return _URLPathParents(self) def relative_to(self, other): - if isinstance(other, str): - other = URLInfo(other) - if self.scheme != other.scheme or self.netloc != other.netloc: - raise ValueError( - "'{}' does not start with '{}'".format(self, other) - ) + if isinstance(other, basestring): + other = self.__class__(other) + if self.__class__ != other.__class__: + msg = "'{}' has incompatible class with '{}'".format(self, other) + raise ValueError(msg) + if self._base_parts != other._base_parts: + msg = "'{}' does not start with '{}'".format(self, other) + raise ValueError(msg) return self._path.relative_to(other._path) def isin(self, other): @@ -263,14 +258,12 @@ class URLInfo(object): other = self.__class__(other) elif self.__class__ != other.__class__: return False - return ( - self.scheme == other.scheme - and self.netloc == other.netloc - and self._path.isin(other._path) + return self._base_parts == other._base_parts and self._path.isin( + other._path ) class CloudURLInfo(URLInfo): @property def path(self): - return self.parsed.path.lstrip("/") + return self._spath.lstrip("/") diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 7557caecd..169aee3ba 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -4,7 +4,6 @@ from __future__ import unicode_literals import os import logging -import posixpath from funcy import cached_property from dvc.progress import Tqdm @@ -273,4 +272,4 @@ class RemoteS3(RemoteBASE): def walk_files(self, path_info, max_items=None): for fname in self._list_paths(path_info, max_items): - yield path_info / posixpath.relpath(fname, path_info.path) + yield path_info.replace(path=fname) diff --git a/dvc/remote/ssh/__init__.py b/dvc/remote/ssh/__init__.py index c6b39b5b5..7230f016a 100644 --- a/dvc/remote/ssh/__init__.py +++ b/dvc/remote/ssh/__init__.py @@ -4,7 +4,6 @@ import errno import itertools import io import os -import posixpath import getpass import logging import threading @@ -268,7 +267,7 @@ class RemoteSSH(RemoteBASE): def walk_files(self, path_info): with self.ssh(path_info) as ssh: for fname in ssh.walk_files(path_info.path): - yield path_info / posixpath.relpath(fname, path_info.path) + yield path_info.replace(path=fname) def makedirs(self, path_info): with self.ssh(path_info) as ssh: From 3660d9eac470a48e152be8dcbf65058976edb8bb Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Fri, 1 Nov 2019 09:49:56 -0700 Subject: [PATCH 23/64] local output: address PR comment, fix style a bit --- dvc/output/local.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dvc/output/local.py b/dvc/output/local.py index ea0cb9d18..cbd9c9503 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -29,7 +29,6 @@ class OutputLOCAL(OutputBase): # FIXME: if we have Windows path containing / or posix one with \ # then we have #2059 bug and can't really handle that. p = self.REMOTE.path_cls(path) - if not p.is_absolute(): p = self.stage.wdir / p From 89502bee88216f2c9cbb99765d8fbc1aa5e31eb0 Mon Sep 17 00:00:00 2001 From: n3hrox Date: Fri, 1 Nov 2019 18:05:27 +0100 Subject: [PATCH 24/64] fix tests --- tests/func/test_remote.py | 2 +- tests/unit/test_logger.py | 38 ++++++++------------------------------ 2 files changed, 9 insertions(+), 31 deletions(-) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index b938f97f4..af9d041cf 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -245,5 +245,5 @@ def test_partial_push_n_pull(dvc_repo, repo_dir, caplog): def get_last_exc(caplog): - _, exc, _ = caplog.records[-1].exc_info + _, exc, _ = caplog.records[-2].exc_info return exc diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 5560acd4e..bcc061ff3 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -45,11 +45,7 @@ class TestColorFormatter: with caplog.at_level(logging.INFO, logger="dvc"): logger.error("message") - expected = ( - "{red}ERROR{nc}: message\n" - "\n" - "{footer}".format(footer=formatter.footer, **colors) - ) + expected = "{red}ERROR{nc}: message\n".format(**colors) assert expected == formatter.format(caplog.records[0]) @@ -60,11 +56,7 @@ class TestColorFormatter: except Exception: logger.exception("message") - expected = ( - "{red}ERROR{nc}: message\n" - "\n" - "{footer}".format(footer=formatter.footer, **colors) - ) + expected = "{red}ERROR{nc}: message\n".format(**colors) assert expected == formatter.format(caplog.records[0]) @@ -75,11 +67,7 @@ class TestColorFormatter: except Exception: logger.exception("") - expected = ( - "{red}ERROR{nc}: description\n" - "\n" - "{footer}".format(footer=formatter.footer, **colors) - ) + expected = "{red}ERROR{nc}: description\n".format(**colors) assert expected == formatter.format(caplog.records[0]) @@ -90,10 +78,8 @@ class TestColorFormatter: except Exception: logger.exception("message") - expected = ( - "{red}ERROR{nc}: message - description\n" - "\n" - "{footer}".format(footer=formatter.footer, **colors) + expected = "{red}ERROR{nc}: message - description\n".format( + **colors ) assert expected == formatter.format(caplog.records[0]) @@ -110,13 +96,8 @@ class TestColorFormatter: "{red}ERROR{nc}: description\n" "{red}{line}{nc}\n" "{stack_trace}" - "{red}{line}{nc}\n" - "\n" - "{footer}".format( - footer=formatter.footer, - line="-" * 60, - stack_trace=stack_trace, - **colors + "{red}{line}{nc}\n".format( + line="-" * 60, stack_trace=stack_trace, **colors ) ) @@ -138,10 +119,7 @@ class TestColorFormatter: "{red}ERROR{nc}: message - second: first\n" "{red}{line}{nc}\n" "{stack_trace}" - "{red}{line}{nc}\n" - "\n" - "{footer}".format( - footer=formatter.footer, + "{red}{line}{nc}\n".format( line="-" * 60, stack_trace="\n".join([first_traceback, second_traceback]), **colors From ab1ead881ba8dc8d8f940329f786706a7191b5aa Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Fri, 1 Nov 2019 12:40:16 -0600 Subject: [PATCH 25/64] sftp: make move across drives possible Fix #2704 --- dvc/remote/ssh/connection.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dvc/remote/ssh/connection.py b/dvc/remote/ssh/connection.py index dcbb1727c..657a059ec 100644 --- a/dvc/remote/ssh/connection.py +++ b/dvc/remote/ssh/connection.py @@ -185,8 +185,17 @@ class SSHConnection: self.sftp.get(src, dest, callback=pbar.update_to) def move(self, src, dst): + """Use `SFTP RENAME` operation when possible, otherwise, + fallback to copy and remove, since the former one doesn't + work across drives. + """ self.makedirs(posixpath.dirname(dst)) - self.sftp.rename(src, dst) + + try: + self.sftp.rename(src, dst) + except OSError: + self.copy(src, dst) + self.remove(src) def upload(self, src, dest, no_progress_bar=False, progress_title=None): self.makedirs(posixpath.dirname(dest)) From 283f84882209e67d0c989ab02825a7dec5ab2843 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Fri, 1 Nov 2019 13:17:22 -0600 Subject: [PATCH 26/64] sftp: make move an atomic operation --- dvc/remote/ssh/connection.py | 17 +++++++++-------- tests/unit/remote/ssh/test_connection.py | 6 ++++++ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/dvc/remote/ssh/connection.py b/dvc/remote/ssh/connection.py index 657a059ec..f8a38a0eb 100644 --- a/dvc/remote/ssh/connection.py +++ b/dvc/remote/ssh/connection.py @@ -185,17 +185,18 @@ class SSHConnection: self.sftp.get(src, dest, callback=pbar.update_to) def move(self, src, dst): - """Use `SFTP RENAME` operation when possible, otherwise, - fallback to copy and remove, since the former one doesn't - work across drives. + """Atomically move src to dst. + + Moving is performed in two stages to make the whole operation atomic in + case src and dst are on different filesystems and actual physical + copying of data is happening. """ self.makedirs(posixpath.dirname(dst)) - try: - self.sftp.rename(src, dst) - except OSError: - self.copy(src, dst) - self.remove(src) + tmp = tmp_fname(dst) + self.copy(src, tmp) + self.sftp.rename(tmp, dst) + self.remove(src) def upload(self, src, dest, no_progress_bar=False, progress_title=None): self.makedirs(posixpath.dirname(dest)) diff --git a/tests/unit/remote/ssh/test_connection.py b/tests/unit/remote/ssh/test_connection.py index f0c12ec21..6b1300ce3 100644 --- a/tests/unit/remote/ssh/test_connection.py +++ b/tests/unit/remote/ssh/test_connection.py @@ -111,3 +111,9 @@ def test_hardlink(repo_dir, ssh): def test_copy(repo_dir, ssh): ssh.copy("foo", "link") assert filecmp.cmp("foo", "link") + + +def test_move(repo_dir, ssh): + ssh.move("foo", "copy") + assert os.path.exists("copy") + assert not os.path.exists("foo") From 8f7000056b43c8221f96378f2c5c63ef9217d320 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Fri, 1 Nov 2019 13:26:03 -0600 Subject: [PATCH 27/64] sftp: move -> fallback to copy when sftp.rename fails --- dvc/remote/ssh/connection.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dvc/remote/ssh/connection.py b/dvc/remote/ssh/connection.py index f8a38a0eb..fa2e8a5ee 100644 --- a/dvc/remote/ssh/connection.py +++ b/dvc/remote/ssh/connection.py @@ -194,7 +194,12 @@ class SSHConnection: self.makedirs(posixpath.dirname(dst)) tmp = tmp_fname(dst) - self.copy(src, tmp) + + try: + self.sftp.rename(src, tmp) + except OSError: + self.copy(src, tmp) + self.sftp.rename(tmp, dst) self.remove(src) From f4f9f3aac1dd4b4d111fdf8b21f329595bee80d5 Mon Sep 17 00:00:00 2001 From: Aman Sharma Date: Thu, 31 Oct 2019 00:27:09 +0530 Subject: [PATCH 28/64] Replace `path` with `path_info` --- dvc/state.py | 7 +++---- dvc/utils/fs.py | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/dvc/state.py b/dvc/state.py index 61f82549d..1d8e3623d 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -370,13 +370,12 @@ class State(object): # pylint: disable=too-many-instance-attributes assert path_info.scheme == "local" assert checksum is not None - path = fspath_py35(path_info) - assert os.path.exists(path) + assert os.path.exists(fspath_py35(path_info)) actual_mtime, actual_size = get_mtime_and_size( - path, self.repo.dvcignore + path_info, self.repo.dvcignore ) - actual_inode = get_inode(path) + actual_inode = get_inode(path_info) existing_record = self.get_state_record_for_inode(actual_inode) if not existing_record: diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index 811b4195a..edbcec3c7 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -7,7 +7,7 @@ import logging from dvc.exceptions import DvcException from dvc.system import System -from dvc.utils import dict_md5, walk_files +from dvc.utils import dict_md5, walk_files, fspath_py35 from dvc.utils.compat import str @@ -21,12 +21,12 @@ def get_inode(path): def get_mtime_and_size(path, dvcignore): - if os.path.isdir(path): + if os.path.isdir(fspath_py35(path)): size = 0 files_mtimes = {} for file_path in walk_files(path, dvcignore): try: - stat = os.stat(file_path) + stat = os.stat(fspath_py35(file_path)) except OSError as exc: # NOTE: broken symlink case. if exc.errno != errno.ENOENT: @@ -39,7 +39,7 @@ def get_mtime_and_size(path, dvcignore): # max(mtime(f) for f in non_ignored_files) mtime = dict_md5(files_mtimes) else: - base_stat = os.stat(path) + base_stat = os.stat(fspath_py35(path)) size = base_stat.st_size mtime = base_stat.st_mtime mtime = int(nanotime.timestamp(mtime)) From 809a85a340e617d91ee48c9af6cb8924e6b37321 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 1 Nov 2019 23:58:17 +0200 Subject: [PATCH 29/64] dvc: bump to 0.66.3 --- dvc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/version.py b/dvc/version.py index ab922547f..6083e98cb 100644 --- a/dvc/version.py +++ b/dvc/version.py @@ -7,7 +7,7 @@ import os import subprocess -_BASE_VERSION = "0.66.2" +_BASE_VERSION = "0.66.3" def _generate_version(base_version): From 16164e65ad1f036aca2dc3c408e177af98f8c635 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Fri, 1 Nov 2019 16:56:55 -0600 Subject: [PATCH 30/64] sftp: move -> use atomic copying --- dvc/remote/ssh/connection.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/dvc/remote/ssh/connection.py b/dvc/remote/ssh/connection.py index fa2e8a5ee..b2da0b67a 100644 --- a/dvc/remote/ssh/connection.py +++ b/dvc/remote/ssh/connection.py @@ -185,23 +185,27 @@ class SSHConnection: self.sftp.get(src, dest, callback=pbar.update_to) def move(self, src, dst): - """Atomically move src to dst. - - Moving is performed in two stages to make the whole operation atomic in - case src and dst are on different filesystems and actual physical - copying of data is happening. + """Rename src to dst, if it is not possible (in case src and dst are + on different filesystems) and actual physical copying of data is + happening. """ self.makedirs(posixpath.dirname(dst)) + try: + self.sftp.rename(src, dst) + except OSError: + self._atomic_copy(src, dst) + + self.remove(src) + + def _atomic_copy(self, src, dst): tmp = tmp_fname(dst) try: - self.sftp.rename(src, tmp) - except OSError: self.copy(src, tmp) - - self.sftp.rename(tmp, dst) - self.remove(src) + self.sftp.rename(tmp, dst) + finally: + self.remove(tmp) def upload(self, src, dest, no_progress_bar=False, progress_title=None): self.makedirs(posixpath.dirname(dest)) From 21bafccc96b9c442d85d3d8080195ade0022dac5 Mon Sep 17 00:00:00 2001 From: kurianbenoy Date: Thu, 31 Oct 2019 19:11:35 +0530 Subject: [PATCH 31/64] Remove multi-threaded prefix before progress bar(#2686) --- dvc/progress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/progress.py b/dvc/progress.py index 7e85b6e18..bc3b17751 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -20,7 +20,7 @@ class TqdmThreadPoolExecutor(ThreadPoolExecutor): Creates a blank initial dummy progress bar if needed so that workers are forced to create "nested" bars. """ - blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False) + blank_bar = Tqdm(bar_format="", leave=False) if blank_bar.pos > 0: # already nested - don't need a placeholder bar blank_bar.close() From 306c916626a62237c2ef7de73e61d719fd0d8e83 Mon Sep 17 00:00:00 2001 From: kurianbenoy Date: Thu, 31 Oct 2019 19:16:05 +0530 Subject: [PATCH 32/64] Change import order in progress file --- dvc/progress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/progress.py b/dvc/progress.py index bc3b17751..24013c637 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -2,8 +2,8 @@ from __future__ import print_function import logging import sys -from tqdm import tqdm from concurrent.futures import ThreadPoolExecutor +from tqdm import tqdm from funcy import merge from dvc.utils import env2bool From 82dae6c25b86320d75c69761541e3c580a6ab913 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 1 Nov 2019 16:06:00 +0000 Subject: [PATCH 33/64] remove special TqdmThreadPoolExecutor logic - fixes #2686 - closes #2703 --- dvc/progress.py | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index 24013c637..3344fff29 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -2,34 +2,13 @@ from __future__ import print_function import logging import sys -from concurrent.futures import ThreadPoolExecutor from tqdm import tqdm +from concurrent.futures import ThreadPoolExecutor from funcy import merge from dvc.utils import env2bool logger = logging.getLogger(__name__) - - -class TqdmThreadPoolExecutor(ThreadPoolExecutor): - """ - Ensure worker progressbars are cleared away properly. - """ - - def __enter__(self): - """ - Creates a blank initial dummy progress bar if needed so that workers - are forced to create "nested" bars. - """ - blank_bar = Tqdm(bar_format="", leave=False) - if blank_bar.pos > 0: - # already nested - don't need a placeholder bar - blank_bar.close() - self.bar = blank_bar - return super(TqdmThreadPoolExecutor, self).__enter__() - - def __exit__(self, *a, **k): - super(TqdmThreadPoolExecutor, self).__exit__(*a, **k) - self.bar.close() +TqdmThreadPoolExecutor = ThreadPoolExecutor class Tqdm(tqdm): From 440b323e39b82cb646a8efd5f6ef1f867152b264 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 2 Nov 2019 15:22:39 +0000 Subject: [PATCH 34/64] TqdmThreadPoolExecutor => ThreadPoolExecutor - Fixes https://github.com/iterative/dvc/pull/2703#discussion_r341763150 --- dvc/progress.py | 2 -- dvc/remote/local.py | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index 3344fff29..68c6ddd59 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -3,12 +3,10 @@ from __future__ import print_function import logging import sys from tqdm import tqdm -from concurrent.futures import ThreadPoolExecutor from funcy import merge from dvc.utils import env2bool logger = logging.getLogger(__name__) -TqdmThreadPoolExecutor = ThreadPoolExecutor class Tqdm(tqdm): diff --git a/dvc/remote/local.py b/dvc/remote/local.py index dff433756..6c1d37d03 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -4,6 +4,7 @@ import stat import errno import logging from functools import partial +from concurrent.futures import ThreadPoolExecutor from shortuuid import uuid @@ -29,7 +30,7 @@ from dvc.utils import ( ) from dvc.config import Config from dvc.exceptions import DvcException, DownloadError, UploadError -from dvc.progress import Tqdm, TqdmThreadPoolExecutor +from dvc.progress import Tqdm from dvc.path_info import PathInfo @@ -359,7 +360,7 @@ class RemoteLOCAL(RemoteBASE): return 0 if jobs > 1: - with TqdmThreadPoolExecutor(max_workers=jobs) as executor: + with ThreadPoolExecutor(max_workers=jobs) as executor: fails = sum(executor.map(func, *plans)) else: fails = sum(map(func, *plans)) From d454cb8637697e4aa779aa461105217a4efec4c0 Mon Sep 17 00:00:00 2001 From: Aman Sharma Date: Sun, 3 Nov 2019 02:21:11 +0530 Subject: [PATCH 35/64] Modify test to check for directory as well --- tests/unit/utils/test_fs.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/unit/utils/test_fs.py b/tests/unit/utils/test_fs.py index 5c8849903..d479b10d9 100644 --- a/tests/unit/utils/test_fs.py +++ b/tests/unit/utils/test_fs.py @@ -39,15 +39,6 @@ class TestMtimeAndSize(TestDir): self.assertIs(type(dir_size), str) self.assertEqual(dir_size, str(actual_dir_size)) - def test_path_object_and_str_are_valid_types(self): - dvcignore = DvcIgnoreFilter(self.root_dir) - file_time, file_size = get_mtime_and_size(self.DATA, dvcignore) - file_object_time, file_object_size = get_mtime_and_size( - PathInfo(self.DATA), dvcignore - ) - self.assertEqual(file_time, file_object_time) - self.assertEqual(file_size, file_object_size) - class TestContainsLink(TestCase): def test_should_raise_exception_on_base_path_not_in_path(self): @@ -165,3 +156,14 @@ def test_get_inode(repo_dir): path = repo_dir.FOO path_info = PathInfo(path) assert get_inode(path) == get_inode(path_info) + + +@pytest.mark.parametrize("path", [TestDir.DATA, TestDir.DATA_DIR]) +def test_path_object_and_str_are_valid_types_get_mtime_and_size( + path, repo_dir +): + dvcignore = DvcIgnoreFilter(repo_dir.root_dir) + time, size = get_mtime_and_size(path, dvcignore) + object_time, object_size = get_mtime_and_size(PathInfo(path), dvcignore) + assert time == object_time + assert size == object_size From cffc9c8ed9ca9c97fd18b3c09bfe3dc9eaec7abc Mon Sep 17 00:00:00 2001 From: Aman Sharma Date: Sun, 3 Nov 2019 02:38:50 +0530 Subject: [PATCH 36/64] Remove unnecessary wrapping of path --- .gitignore | 1 + dvc/state.py | 1 - dvc/utils/fs.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index d2a206fe3..33be5cb20 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ neatlynx/__pycache__ cache *.pyc .env/ +.env2.7/ cache .dvc.conf.lock diff --git a/dvc/state.py b/dvc/state.py index 1d8e3623d..dacc07693 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -369,7 +369,6 @@ class State(object): # pylint: disable=too-many-instance-attributes """ assert path_info.scheme == "local" assert checksum is not None - assert os.path.exists(fspath_py35(path_info)) actual_mtime, actual_size = get_mtime_and_size( diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index edbcec3c7..6ded4cefd 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -26,7 +26,7 @@ def get_mtime_and_size(path, dvcignore): files_mtimes = {} for file_path in walk_files(path, dvcignore): try: - stat = os.stat(fspath_py35(file_path)) + stat = os.stat(file_path) except OSError as exc: # NOTE: broken symlink case. if exc.errno != errno.ENOENT: From 4007da0298df25873c599837cbbf6cb3e7946c0c Mon Sep 17 00:00:00 2001 From: Pandemic179 Date: Sun, 3 Nov 2019 15:02:30 +0530 Subject: [PATCH 37/64] Rectified simple typos --- dvc/remote/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 6c1d37d03..7f950404a 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -264,7 +264,7 @@ class RemoteLOCAL(RemoteBASE): # This is a performance optimization. We can safely assume that, # if the resources that we want to fetch are already cached, - # there's no need to check the remote storage for the existance of + # there's no need to check the remote storage for the existence of # those files. if download and sorted(local_exists) == sorted(md5s): remote_exists = local_exists From 97b4800d2e55a1f7801132912deb539b77513eda Mon Sep 17 00:00:00 2001 From: eshaan-jeckyll Date: Sun, 3 Nov 2019 15:03:36 +0530 Subject: [PATCH 38/64] typos corrected --- dvc/remote/local.py | 2 +- dvc/remote/s3.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 6c1d37d03..7f950404a 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -264,7 +264,7 @@ class RemoteLOCAL(RemoteBASE): # This is a performance optimization. We can safely assume that, # if the resources that we want to fetch are already cached, - # there's no need to check the remote storage for the existance of + # there's no need to check the remote storage for the existence of # those files. if download and sorted(local_exists) == sorted(md5s): remote_exists = local_exists diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 169aee3ba..30a385e8b 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -150,7 +150,7 @@ class RemoteS3(RemoteBASE): # could use any value >= ContentLength. # # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13', - # then it was transfered as a multipart, which means that the chunk + # then it was transferred as a multipart, which means that the chunk # size used to transfer it was less than ContentLength of that object. # Unfortunately, in general, it doesn't mean that the chunk size was # the same throughout the transfer, so it means that in order to From e26517ec36b43c7682e7b9b17acb048532996bb1 Mon Sep 17 00:00:00 2001 From: ATR-oCiTy Date: Sun, 3 Nov 2019 15:04:42 +0530 Subject: [PATCH 39/64] Simple typos mectified --- dvc/remote/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 169aee3ba..bd875c1c0 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -144,7 +144,7 @@ class RemoteS3(RemoteBASE): # multipart. # # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa', - # then it was transfered as a single part, which means that the chunk + # then it was transferred as a single part, which means that the chunk # size used to transfer it was greater or equal to the ContentLength # of that object. So to preserve that tag over the next transfer, we # could use any value >= ContentLength. From 2c0b209e380494cc15cbd9e22058311ca994e5df Mon Sep 17 00:00:00 2001 From: ATR-oCiTy Date: Sun, 3 Nov 2019 15:25:30 +0530 Subject: [PATCH 40/64] Rectified install: Cask 'dvc' is not installed. error #2680 --- dvc/updater.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/dvc/updater.py b/dvc/updater.py index 21251e4da..18625029f 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -123,7 +123,6 @@ class Updater(object): # pragma: no cover "yum": "Run {yellow}yum{reset} update dvc", "yay": "Run {yellow}yay{reset} {blue}-S{reset} dvc", "formula": "Run {yellow}brew{reset} upgrade dvc", - "cask": "Run {yellow}brew cask{reset} upgrade dvc", "apt": ( "Run {yellow}apt-get{reset} install" " {blue}--only-upgrade{reset} dvc" @@ -169,14 +168,6 @@ class Updater(object): # pragma: no cover return "formula" else: return "pip" - - # NOTE: both pkg and cask put dvc binary into /usr/local/bin, - # so in order to know which method of installation was used, - # we need to actually call `brew cask` - ret = os.system("brew cask ls dvc") - if ret == 0: - return "cask" - return None def _get_windows(self): From 1afc40c1ec83dd9c8055418d4bf7dec0315ea268 Mon Sep 17 00:00:00 2001 From: Naina <54177705+the-thirteenth-star@users.noreply.github.com> Date: Sun, 3 Nov 2019 15:40:27 +0530 Subject: [PATCH 41/64] Update fetch.py --- dvc/repo/fetch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index b69c8c04b..95eba34b1 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -26,7 +26,7 @@ def _fetch( """Download data items from a cloud and imported repositories Returns: - int: number of succesfully downloaded files + int: number of successfully downloaded files Raises: DownloadError: thrown when there are failed downloads, either From 1e22fabece0651278c652b8b9127a832a37341ca Mon Sep 17 00:00:00 2001 From: GeorgeSabu-MEC Date: Sun, 3 Nov 2019 15:42:07 +0530 Subject: [PATCH 42/64] fix typo(correct spelling of existence) --- tests/unit/remote/test_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/remote/test_local.py b/tests/unit/remote/test_local.py index c9681d50f..461e37b79 100644 --- a/tests/unit/remote/test_local.py +++ b/tests/unit/remote/test_local.py @@ -5,7 +5,7 @@ from dvc.remote.local import RemoteLOCAL def test_status_download_optimization(mocker): """When comparing the status to pull a remote cache, And the desired files to fetch are already on the local cache, - Don't check the existance of the desired files on the remote cache + Don't check the existence of the desired files on the remote cache """ remote = RemoteLOCAL(None, {}) From 55d819494a6950f5b8e8b6e7d9df204aca421a08 Mon Sep 17 00:00:00 2001 From: johntharian Date: Sun, 3 Nov 2019 15:51:37 +0530 Subject: [PATCH 43/64] typo changed for existance->existaence --- dvc/remote/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 6c1d37d03..7f950404a 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -264,7 +264,7 @@ class RemoteLOCAL(RemoteBASE): # This is a performance optimization. We can safely assume that, # if the resources that we want to fetch are already cached, - # there's no need to check the remote storage for the existance of + # there's no need to check the remote storage for the existence of # those files. if download and sorted(local_exists) == sorted(md5s): remote_exists = local_exists From 0fece74d00cc6445d0073fc8698559da5cc528b0 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 3 Nov 2019 14:18:25 +0200 Subject: [PATCH 44/64] test: get-url: don't use get_aws_url --- tests/func/test_get_url.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/func/test_get_url.py b/tests/func/test_get_url.py index fab7a4db6..49a8f1861 100644 --- a/tests/func/test_get_url.py +++ b/tests/func/test_get_url.py @@ -12,8 +12,6 @@ from dvc.remote import RemoteS3 from dvc.repo import Repo from dvc.utils import makedirs -from tests.func.test_data_cloud import get_aws_url - def test_get_file(repo_dir): src = repo_dir.FOO @@ -45,7 +43,7 @@ def test_get_url_to_dir(dname, repo_dir): def test_get_url_from_non_local_path_to_dir_and_file(repo_dir, dst): file_name = "from" file_content = "data" - base_info = RemoteS3.path_cls(get_aws_url()) + base_info = RemoteS3.path_cls("s3://bucket/path") from_info = base_info / file_name s3 = boto3.client("s3") From 3514cc5b5a71891460cae7252cbf8f9811008adc Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 3 Nov 2019 14:49:36 +0200 Subject: [PATCH 45/64] test: freeze moto version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1780637a6..ffd35287c 100644 --- a/setup.py +++ b/setup.py @@ -127,7 +127,7 @@ tests_requirements = [ "pydocstyle<4.0", "jaraco.windows==3.9.2", "mock-ssh-server>=0.6.0", - "moto>=1.3.14.dev55", + "moto==1.3.14.dev464", "rangehttpserver==1.2.0", ] From e75c52a170d7099fa4673197265360c859d7deee Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 3 Nov 2019 15:09:29 +0200 Subject: [PATCH 46/64] setup: freeze awscli version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ffd35287c..6817b8dea 100644 --- a/setup.py +++ b/setup.py @@ -116,7 +116,7 @@ tests_requirements = [ "flaky>=3.5.3", "mock>=3.0.0", "xmltodict>=0.11.0", - "awscli>=1.16.125", + "awscli==1.16.266", "google-compute-engine==2.8.13", "pywin32; sys_platform == 'win32'", "Pygments", # required by collective.checkdocs, From bb7a9a567e037c74ce64369c763c9e2616027315 Mon Sep 17 00:00:00 2001 From: kss682 Date: Mon, 4 Nov 2019 20:29:39 +0530 Subject: [PATCH 47/64] Added package manager to dvc version output. Solves issue #2672 --- dvc/command/version.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dvc/command/version.py b/dvc/command/version.py index 078f1fdec..a123bfafe 100644 --- a/dvc/command/version.py +++ b/dvc/command/version.py @@ -18,7 +18,7 @@ from dvc.command.base import CmdBaseNoRepo, append_doc_link from dvc.version import __version__ from dvc.exceptions import DvcException, NotDvcRepoError from dvc.system import System - +from dvc.updater import Updater logger = logging.getLogger(__name__) @@ -31,17 +31,19 @@ class CmdVersion(CmdBaseNoRepo): python_version = platform.python_version() platform_type = platform.platform() binary = is_binary() - + package_manager = Updater("dvc/")._get_package_manager() info = ( "DVC version: {dvc_version}\n" "Python version: {python_version}\n" "Platform: {platform_type}\n" "Binary: {binary}\n" + "Package manager: {package_manager}\n" ).format( dvc_version=dvc_version, python_version=python_version, platform_type=platform_type, binary=binary, + package_manager=package_manager, ) try: From 0786a706975c448989ad3c515523d84fa88373fe Mon Sep 17 00:00:00 2001 From: kss682 Date: Tue, 5 Nov 2019 00:15:56 +0530 Subject: [PATCH 48/64] Package added with get_package_manager function and other utils. import changed accordingly in version Solves issue #2672 --- dvc/command/version.py | 4 ++-- dvc/utils/pkg.py | 51 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 dvc/utils/pkg.py diff --git a/dvc/command/version.py b/dvc/command/version.py index a123bfafe..7cf00c751 100644 --- a/dvc/command/version.py +++ b/dvc/command/version.py @@ -18,7 +18,7 @@ from dvc.command.base import CmdBaseNoRepo, append_doc_link from dvc.version import __version__ from dvc.exceptions import DvcException, NotDvcRepoError from dvc.system import System -from dvc.updater import Updater +from dvc.utils.pkg import get_package_manager logger = logging.getLogger(__name__) @@ -31,7 +31,7 @@ class CmdVersion(CmdBaseNoRepo): python_version = platform.python_version() platform_type = platform.platform() binary = is_binary() - package_manager = Updater("dvc/")._get_package_manager() + package_manager = get_package_manager() info = ( "DVC version: {dvc_version}\n" "Python version: {python_version}\n" diff --git a/dvc/utils/pkg.py b/dvc/utils/pkg.py new file mode 100644 index 000000000..a35bd7306 --- /dev/null +++ b/dvc/utils/pkg.py @@ -0,0 +1,51 @@ +from dvc.utils import is_binary + + +def get_linux(): + import distro + + if not is_binary(): + return "pip" + + package_managers = { + "rhel": "yum", + "centos": "yum", + "fedora": "yum", + "amazon": "yum", + "opensuse": "yum", + "ubuntu": "apt", + "debian": "apt", + } + + return package_managers.get(distro.id()) + + +def get_darwin(): + if not is_binary(): + if __file__.startswith("/usr/local/Cellar"): + return "formula" + else: + return "pip" + return None + + +def get_windows(): + return None if is_binary() else "pip" + + +def get_package_manager(): + import platform + from dvc.exceptions import DvcException + + m = { + "Windows": get_windows(), + "Darwin": get_darwin(), + "Linux": get_linux(), + } + + system = platform.system() + func = m.get(system) + if func is None: + raise DvcException("not supported system '{}'".format(system)) + + return func From 8903dbbc03e81f0aa7daab4fe71d9f36fdc19e62 Mon Sep 17 00:00:00 2001 From: kss682 Date: Tue, 5 Nov 2019 00:31:55 +0530 Subject: [PATCH 49/64] Deleted duplicate from updater Solves issue #2672 --- dvc/updater.py | 48 +----------------------------------------------- 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/dvc/updater.py b/dvc/updater.py index 18625029f..b6571de64 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -9,7 +9,7 @@ from packaging import version from dvc import __version__ from dvc.lock import Lock, LockError -from dvc.utils import is_binary, boxify, env2bool +from dvc.utils import boxify, env2bool logger = logging.getLogger(__name__) @@ -143,49 +143,3 @@ class Updater(object): # pragma: no cover package_manager = self._get_package_manager() return instructions[package_manager] - - def _get_linux(self): - import distro - - if not is_binary(): - return "pip" - - package_managers = { - "rhel": "yum", - "centos": "yum", - "fedora": "yum", - "amazon": "yum", - "opensuse": "yum", - "ubuntu": "apt", - "debian": "apt", - } - - return package_managers.get(distro.id()) - - def _get_darwin(self): - if not is_binary(): - if __file__.startswith("/usr/local/Cellar"): - return "formula" - else: - return "pip" - return None - - def _get_windows(self): - return None if is_binary() else "pip" - - def _get_package_manager(self): - import platform - from dvc.exceptions import DvcException - - m = { - "Windows": self._get_windows, - "Darwin": self._get_darwin, - "Linux": self._get_linux, - } - - system = platform.system() - func = m.get(system) - if func is None: - raise DvcException("not supported system '{}'".format(system)) - - return func() From 3421c6c6e45bbb058fc6c586a308e0074c6cb931 Mon Sep 17 00:00:00 2001 From: kss682 Date: Tue, 5 Nov 2019 00:58:27 +0530 Subject: [PATCH 50/64] Added get_package_manager from dvc.utils.pkg. Solves issue #2672 --- dvc/updater.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/updater.py b/dvc/updater.py index b6571de64..5176c3cb3 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -10,7 +10,7 @@ from packaging import version from dvc import __version__ from dvc.lock import Lock, LockError from dvc.utils import boxify, env2bool - +from dvc.utils.pkg import get_package_manager logger = logging.getLogger(__name__) @@ -140,6 +140,6 @@ class Updater(object): # pragma: no cover ), } - package_manager = self._get_package_manager() + package_manager = get_package_manager() return instructions[package_manager] From 46d221fade8e62c9700150bdeb041e327a55c52a Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 3 Nov 2019 22:38:48 +0200 Subject: [PATCH 51/64] test: s3: try using boto3.client --- tests/func/test_repro.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index 148d2a006..f75edce04 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -964,8 +964,8 @@ class TestReproExternalS3(TestReproExternalBase): return "aws s3 cp {} {}".format(i, o) def write(self, bucket, key, body): - s3 = boto3.resource("s3") - s3.Bucket(bucket).put_object(Key=key, Body=body) + s3 = boto3.client("s3") + s3.put_object(Bucket=bucket, Key=key, Body=body) class TestReproExternalGS(TestReproExternalBase): From 2893c9acad24546f14367e93e35355f59bd6920f Mon Sep 17 00:00:00 2001 From: n3hrox Date: Sun, 3 Nov 2019 22:07:27 +0100 Subject: [PATCH 52/64] move footer to better place and change logging to stderr --- dvc/logger.py | 11 +++++++++++ dvc/main.py | 15 +++------------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/dvc/logger.py b/dvc/logger.py index 075b1b680..f26ef31ce 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -11,6 +11,17 @@ import logging.config import colorama +FOOTER = ( + "\n{yellow}Having any troubles?{nc}" + " Hit us up at {blue}https://dvc.org/support{nc}," + " we are always happy to help!" +).format( + blue=colorama.Fore.BLUE, + nc=colorama.Fore.RESET, + yellow=colorama.Fore.YELLOW, +) + + class LoggingException(Exception): def __init__(self, record): msg = "failed to log {}".format(str(record)) diff --git a/dvc/main.py b/dvc/main.py index d21a51592..287443727 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -3,10 +3,11 @@ from __future__ import unicode_literals import logging -import colorama +import sys from dvc.cli import parse_args from dvc.lock import LockError +from dvc.logger import FOOTER from dvc.config import ConfigError from dvc.analytics import Analytics from dvc.exceptions import NotDvcRepoError, DvcParserError @@ -17,16 +18,6 @@ from dvc.utils.compat import is_py2 logger = logging.getLogger("dvc") -footer = ( - "\n{yellow}Having any troubles?{nc}" - " Hit us up at {blue}https://dvc.org/support{nc}," - " we are always happy to help!" -).format( - blue=colorama.Fore.BLUE, - nc=colorama.Fore.RESET, - yellow=colorama.Fore.YELLOW, -) - def main(argv=None): """Run dvc CLI command. @@ -87,7 +78,7 @@ def main(argv=None): clean_repos() if ret != 0: - logger.info(footer) + print(FOOTER, file=sys.stderr) Analytics().send_cmd(cmd, args, ret) From d25282ba28c45db92e6b2e9f5416dd2a6fe2530d Mon Sep 17 00:00:00 2001 From: n3hrox Date: Sun, 3 Nov 2019 22:14:59 +0100 Subject: [PATCH 53/64] revert print to logger --- dvc/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dvc/main.py b/dvc/main.py index 287443727..5d3f01193 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import logging -import sys from dvc.cli import parse_args from dvc.lock import LockError @@ -78,7 +77,7 @@ def main(argv=None): clean_repos() if ret != 0: - print(FOOTER, file=sys.stderr) + logger.info(FOOTER) Analytics().send_cmd(cmd, args, ret) From 71f8d25ab3fb3fe3983f7fca21a11aa2899b0277 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 3 Nov 2019 23:42:44 +0200 Subject: [PATCH 54/64] test: get-url: try using `with mock_s3()` instead of a decorator --- tests/func/test_get_url.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/func/test_get_url.py b/tests/func/test_get_url.py index 49a8f1861..55eec1a9a 100644 --- a/tests/func/test_get_url.py +++ b/tests/func/test_get_url.py @@ -38,25 +38,25 @@ def test_get_url_to_dir(dname, repo_dir): assert filecmp.cmp(repo_dir.DATA, dst, shallow=False) -@mock_s3 @pytest.mark.parametrize("dst", [".", "./from"]) def test_get_url_from_non_local_path_to_dir_and_file(repo_dir, dst): - file_name = "from" - file_content = "data" - base_info = RemoteS3.path_cls("s3://bucket/path") - from_info = base_info / file_name + with mock_s3(): + file_name = "from" + file_content = "data" + base_info = RemoteS3.path_cls("s3://bucket/path") + from_info = base_info / file_name - s3 = boto3.client("s3") - s3.create_bucket(Bucket=from_info.bucket) - s3.put_object( - Bucket=from_info.bucket, Key=from_info.path, Body=file_content - ) + s3 = boto3.client("s3") + s3.create_bucket(Bucket=from_info.bucket) + s3.put_object( + Bucket=from_info.bucket, Key=from_info.path, Body=file_content + ) - Repo.get_url(from_info.url, dst) + Repo.get_url(from_info.url, dst) - result_path = os.path.join(dst, file_name) if os.path.isdir(dst) else dst + result_path = os.path.join(dst, file_name) if os.path.isdir(dst) else dst - assert os.path.exists(result_path) - assert os.path.isfile(result_path) - with open(result_path, "r") as fd: - assert fd.read() == file_content + assert os.path.exists(result_path) + assert os.path.isfile(result_path) + with open(result_path, "r") as fd: + assert fd.read() == file_content From 46391cb4f9e846bdb5c5d0c7ea5cf06684ded3d6 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 3 Nov 2019 23:47:34 +0200 Subject: [PATCH 55/64] test: fix formatting --- tests/func/test_get_url.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/func/test_get_url.py b/tests/func/test_get_url.py index 55eec1a9a..36c1b2fd7 100644 --- a/tests/func/test_get_url.py +++ b/tests/func/test_get_url.py @@ -54,7 +54,9 @@ def test_get_url_from_non_local_path_to_dir_and_file(repo_dir, dst): Repo.get_url(from_info.url, dst) - result_path = os.path.join(dst, file_name) if os.path.isdir(dst) else dst + result_path = ( + os.path.join(dst, file_name) if os.path.isdir(dst) else dst + ) assert os.path.exists(result_path) assert os.path.isfile(result_path) From 4efd048796564e366b1dfa7a4302aa5d67932481 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 4 Nov 2019 00:05:44 +0200 Subject: [PATCH 56/64] test: try removing get_url test --- tests/func/test_get_url.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/tests/func/test_get_url.py b/tests/func/test_get_url.py index 36c1b2fd7..eae2b556e 100644 --- a/tests/func/test_get_url.py +++ b/tests/func/test_get_url.py @@ -1,14 +1,10 @@ from __future__ import unicode_literals import os -import boto3 import filecmp import pytest -from moto import mock_s3 - -from dvc.remote import RemoteS3 from dvc.repo import Repo from dvc.utils import makedirs @@ -36,29 +32,3 @@ def test_get_url_to_dir(dname, repo_dir): assert os.path.isdir(dname) assert filecmp.cmp(repo_dir.DATA, dst, shallow=False) - - -@pytest.mark.parametrize("dst", [".", "./from"]) -def test_get_url_from_non_local_path_to_dir_and_file(repo_dir, dst): - with mock_s3(): - file_name = "from" - file_content = "data" - base_info = RemoteS3.path_cls("s3://bucket/path") - from_info = base_info / file_name - - s3 = boto3.client("s3") - s3.create_bucket(Bucket=from_info.bucket) - s3.put_object( - Bucket=from_info.bucket, Key=from_info.path, Body=file_content - ) - - Repo.get_url(from_info.url, dst) - - result_path = ( - os.path.join(dst, file_name) if os.path.isdir(dst) else dst - ) - - assert os.path.exists(result_path) - assert os.path.isfile(result_path) - with open(result_path, "r") as fd: - assert fd.read() == file_content From 5227fc58d526b2884a2a87d04a1299576eed0f9b Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 4 Nov 2019 00:38:10 +0200 Subject: [PATCH 57/64] dvc: bump to 0.66.4 --- dvc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/version.py b/dvc/version.py index 6083e98cb..6561d954c 100644 --- a/dvc/version.py +++ b/dvc/version.py @@ -7,7 +7,7 @@ import os import subprocess -_BASE_VERSION = "0.66.3" +_BASE_VERSION = "0.66.4" def _generate_version(base_version): From d0e62a57ab6e2682e7c3baf64738600980718380 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sun, 3 Nov 2019 15:15:23 -0800 Subject: [PATCH 58/64] fix PR template to mention recommendatory nature of CC and DS --- .github/PULL_REQUEST_TEMPLATE.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 74975611d..916f77b04 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,9 +1,17 @@ -* [ ] Have you followed the guidelines in our - [Contributing document](https://dvc.org/doc/user-guide/contributing/core)? +* [ ] ❗ Have you followed the guidelines in the + [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) + list? -* [ ] Does your PR affect documented changes or does it add new functionality - that should be documented? If yes, have you created a PR for - [dvc.org](https://github.com/iterative/dvc.org) documenting it or at - least opened an issue for it? If so, please add a link to it. +* [ ] 📖 This PR does _not_ require [documentation](https://dvc.org/doc) updates + or modifications. If it _does_, have you created a PR for + [dvc.org](https://github.com/iterative/dvc.org) or at least opened an + issue for it? If so, please add a link to it and after that check this + box. + +* [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks + below? Their findings are considered recommendatory. We don't expect all + of them to be fixed. Please, read them carefully and fix those that + actually improve code and/or fix bugs. + +Thank you for the contribution - we'll try to review it as soon as possible 🙏. ------ From 60bdd41cbd3374a761700bce2b92e088f4c7c31b Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sun, 3 Nov 2019 16:17:23 -0800 Subject: [PATCH 59/64] PR template: address review, update DS and CC checks item --- .github/PULL_REQUEST_TEMPLATE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 916f77b04..55bc950f6 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,9 +9,9 @@ box. * [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks - below? Their findings are considered recommendatory. We don't expect all - of them to be fixed. Please, read them carefully and fix those that - actually improve code and/or fix bugs. + below? We consider their findings recommendatory and don't expect + everything to be addresses. Please review them carefully and fix those + that actually improve code or fix bugs. Thank you for the contribution - we'll try to review it as soon as possible 🙏. From 8b05eb5d3633432060095f608fdc8073848fd415 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sun, 3 Nov 2019 16:19:05 -0800 Subject: [PATCH 60/64] PR template: minor sentence with emoji style change --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 55bc950f6..6064756b2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,5 +13,5 @@ everything to be addresses. Please review them carefully and fix those that actually improve code or fix bugs. -Thank you for the contribution - we'll try to review it as soon as possible 🙏. +Thank you for the contribution - we'll try to review it as soon as possible. 🙏 From cbb58e02d6c04950f9eb900e5a14d7cc6baa14ab Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sun, 3 Nov 2019 17:02:11 -0800 Subject: [PATCH 61/64] PR template: change wording for the docs checkbox --- .github/PULL_REQUEST_TEMPLATE.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6064756b2..92da7b400 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,11 +2,13 @@ [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) list? -* [ ] 📖 This PR does _not_ require [documentation](https://dvc.org/doc) updates - or modifications. If it _does_, have you created a PR for - [dvc.org](https://github.com/iterative/dvc.org) or at least opened an - issue for it? If so, please add a link to it and after that check this - box. +* [ ] 📖 Check this box if this PR **does not** require + [documentation](https://dvc.org/doc) updates, or if it does **and** you + have created a separate PR in + [dvc.org](https://github.com/iterative/dvc.org) + with such updates (or at least opened an issue about it in that repo). + Please link below to your PR (or issue) in the + [dvc.org](https://github.com/iterative/dvc.org) repo. * [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect From 3974862f877a42ecd17ec8367903dd48dbc7cc71 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 4 Nov 2019 19:13:39 +0200 Subject: [PATCH 62/64] github: add config for release-drafter bot --- .github/release-drafter.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .github/release-drafter.yml diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 000000000..abf195c69 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,5 @@ +# Config for https://github.com/apps/release-drafter +branches: + - master +template: | + $CHANGES From 8b4cc01110d2d18c8bc0decead4131e29669220f Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 4 Nov 2019 20:20:22 +0200 Subject: [PATCH 63/64] add restyled.io --- .restyled.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .restyled.yaml diff --git a/.restyled.yaml b/.restyled.yaml new file mode 100644 index 000000000..886b50cde --- /dev/null +++ b/.restyled.yaml @@ -0,0 +1,9 @@ +--- +image: restyled/restyler-black:v19.3b0 +command: +- black +arguments: [] +include: +- "**/*.py" +interpreters: +- python From 35507dfc616434513bb95b0ef9b5658903bfe8b2 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 4 Nov 2019 20:31:20 +0200 Subject: [PATCH 64/64] Update .restyled.yaml --- .restyled.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.restyled.yaml b/.restyled.yaml index 886b50cde..d5eeec715 100644 --- a/.restyled.yaml +++ b/.restyled.yaml @@ -1,9 +1,11 @@ --- -image: restyled/restyler-black:v19.3b0 -command: -- black -arguments: [] -include: -- "**/*.py" -interpreters: -- python +restylers: + - name: black + image: restyled/restyler-black:v19.3b0 + command: + - black + arguments: [] + include: + - "**/*.py" + interpreters: + - python