From b2b92879c4a96ff4e340abb545fc15f9dc906b80 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 30 May 2019 02:25:22 +0300 Subject: [PATCH 01/30] appveyor: use our own fork of oss-emulator --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index f498c8f23..e9c8b6f74 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -33,7 +33,7 @@ environment: install: - SET PATH=%RUBY%;%PATH% - - git clone -q https://github.com/nanaya-tachibana/oss-emulator.git + - git clone -q https://github.com/iterative/oss-emulator.git - gem install thor builder - ps: cd oss-emulator - ps: $OSSProcess = Start-Process -FilePath ruby -ArgumentList "bin\emulator -r store -p 50004" -PassThru -NoNewWindow From e3b3f3f371116fae35b4eafee6258eee83d9eaf4 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Wed, 29 May 2019 16:44:53 -0700 Subject: [PATCH 02/30] progress: bug fix with --quiet Related to #1377 --- dvc/progress.py | 2 +- tests/unit/test_progress.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index b25f07d43..31c882f52 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -111,7 +111,7 @@ class Progress(object): logger = logging.getLogger(__name__) - if logger.getEffectiveLevel() == "CRITICAL": + if logger.getEffectiveLevel() == logging.CRITICAL: return print(*args, **kwargs) diff --git a/tests/unit/test_progress.py b/tests/unit/test_progress.py index 8bcee4167..e617c1775 100644 --- a/tests/unit/test_progress.py +++ b/tests/unit/test_progress.py @@ -1,14 +1,19 @@ +import logging import mock -from unittest import TestCase - -from dvc.progress import ProgressCallback +from dvc.progress import progress, ProgressCallback -class TestProgressCallback(TestCase): +def test_quiet(caplog, capsys): + with caplog.at_level(logging.CRITICAL, logger="dvc"): + progress._print("something") + assert capsys.readouterr().out == "" + + +class TestProgressCallback: @mock.patch("dvc.progress.progress") def test_should_init_reset_progress(self, progress_mock): total_files_num = 1 ProgressCallback(total_files_num) - self.assertEqual([mock.call.reset()], progress_mock.method_calls) + assert [mock.call.reset()] == progress_mock.method_calls From 012e045837fb79bcb80c1fb5982d901b4b50a212 Mon Sep 17 00:00:00 2001 From: pawel Date: Wed, 29 May 2019 16:47:13 -0700 Subject: [PATCH 03/30] make get_mtime_and_size use single stat call for files --- dvc/utils/fs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index cecc0a207..9a7bd9e73 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -21,8 +21,9 @@ def get_inode(path): def get_mtime_and_size(path): - size = os.path.getsize(path) - mtime = os.path.getmtime(path) + stat = os.stat(path) + size = stat.st_size + mtime = stat.st_mtime if os.path.isdir(path): for root, dirs, files in dvc_walk(str(path)): From 8df0712b0689d4b1fbce5bd977426c17bc2fdb18 Mon Sep 17 00:00:00 2001 From: Jorge Orpinel Date: Wed, 29 May 2019 21:57:30 -0500 Subject: [PATCH 04/30] update zsh autocomplete script with current command help messages + (#2064) * dvc: fixes dvc.scm.Git.get_diff_trees docstring imprecision. Parentheses were used in the "Returns:" description but its a dict, so changed to curly brackets; and specified the 2 possible forms of the return structure. * dvc: updates zsh autocomplete script with current command help messages. Comparing vs actually running each command's `-h` help output. * dvc: begins updating bask autocomplete script... Only reorders existing commands and adds comments for those missing. * scripts: removes temporary comments in bash autocomplete script. Per https://github.com/iterative/dvc/pull/2064#pullrequestreview-243578115 Related to #2069 --- dvc/command/remove.py | 2 +- dvc/scm/git/__init__.py | 4 +- scripts/completion/dvc.bash | 37 ++++----- scripts/completion/dvc.zsh | 145 +++++++++++++++++++----------------- 4 files changed, 98 insertions(+), 90 deletions(-) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index aa455702b..6f2706674 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -57,7 +57,7 @@ def add_parser(subparsers, parent_parser): "--outs", action="store_true", default=True, - help="Only remove DVC file outputs.(default)", + help="Only remove DVC file outputs. (Default)", ) remove_parser_group.add_argument( "-p", diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 3522f74d0..690b54c3c 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -297,8 +297,8 @@ class Git(Base): b_ref (str): optional second git reference, default None Returns: - dict: dictionary with keys: (a_tree, b_tree, a_ref, b_ref, - equal) + dict: dictionary with keys: {a_ref, b_ref, equal} + or {a_ref, b_ref, a_tree, b_tree} """ diff_dct = {DIFF_EQUAL: False} trees, commits = self._get_diff_trees(a_ref, b_ref) diff --git a/scripts/completion/dvc.bash b/scripts/completion/dvc.bash index d31149459..adc389e0f 100644 --- a/scripts/completion/dvc.bash +++ b/scripts/completion/dvc.bash @@ -12,30 +12,31 @@ _dvc_commands='init destroy add import checkout run pull push fetch \ status repro remove move unprotect gc config remote metrics \ install root lock unlock pipeline commit' -_dvc_global_options="-h --help -q --quiet -V --verbose" _dvc_options="-h --help -v --version" -_dvc_init="--no-scm -f --force" -_dvc_destroy="-f --force" +_dvc_global_options="-h --help -q --quiet -V --verbose" + _dvc_add="-R --recursive -f --file --no-commit" -_dvc_run="--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache -M --metrics-no-cache -y --yes --overwrite-dvc-file --ignore-build-cache --remove-outs --no-commit -w --wdir" +_dvc_checkout="$(compgen -G '*.dvc')" +_dvc_config="-u --unset --local --system --global" +_dvc_destroy="-f --force" +_dvc_fetch="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive" +_dvc_gc="-a --all-branches -T --all-tags -c --cloud -r --remote -f --force -p --project" +_dvc_import="--resume" +_dvc_init="--no-scm -f --force" +_dvc_install="" +_dvc_lock="$(compgen -G '*.dvc')" +_dvc_metrics="" +_dvc_move="$(compgen -G '*')" +_dvc_pipeline="" _dvc_pull="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -f --force -R --recursive" _dvc_push="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive" -_dvc_fetch="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive" -_dvc_status="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud -q --quiet" -_dvc_repro="--dry -f --force -s --single-item -c --cwd -m --metrics -i --interactive -p --pipeline -P --all-pipelines --ignore-build-cache --no-commit -s --single-item" -_dvc_remove="--dry -o --outs -p --purge --force" -_dvc_gc="-a --all-branches -T --all-tags -c --cloud -r --remote -f --force -p --project" -_dvc_config="-u --unset --local --system --global" -_dvc_checkout="$(compgen -G '*.dvc')" -_dvc_move="$(compgen -G '*')" -_dvc_lock="$(compgen -G '*.dvc')" -_dvc_unlock="$(compgen -G '*.dvc')" -_dvc_import="--resume" _dvc_remote="" -_dvc_metrics="" -_dvc_install="" +_dvc_remove="--dry -o --outs -p --purge --force" +_dvc_repro="--dry -f --force -s --single-item -c --cwd -m --metrics -i --interactive -p --pipeline -P --all-pipelines --ignore-build-cache --no-commit -s --single-item" _dvc_root="" -_dvc_pipeline="" +_dvc_run="--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache -M --metrics-no-cache -y --yes --overwrite-dvc-file --ignore-build-cache --remove-outs --no-commit -w --wdir" +_dvc_status="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud -q --quiet" +_dvc_unlock="$(compgen -G '*.dvc')" # Notes: # diff --git a/scripts/completion/dvc.zsh b/scripts/completion/dvc.zsh index 4ceef21b9..a1a0849d5 100644 --- a/scripts/completion/dvc.zsh +++ b/scripts/completion/dvc.zsh @@ -44,21 +44,21 @@ _dvc_commands() { _describe 'dvc commands' _commands } -_dvc_global_options=( - "(-)"{-h,--help}"[Show help message related to the command]" - "(-)"{-q,--quiet}"[Be quiet.]" - "(-)"{-V,--verbose}"[Be verbose.]" +_dvc_options=( + "(-)"{-h,--help}"[Show help message.]" + "(-)"{-V,--version}"[Show program's version]" ) -_dvc_options=( - "(-)"{-h,--help}"[Show this help message and exit]" - "(-)"{-v,--version}"[Show program's version]" +_dvc_global_options=( + "(-)"{-h,--help}"[Show help message related to the command.]" + "(-)"{-q,--quiet}"[Be quiet.]" + "(-)"{-v,--verbose}"[Be verbose.]" ) _dvc_add=( {-R,--recursive}"[Recursively add each file under the directory.]" - {-f,--file}"[Specify name of the stage file.]:File:_files" "--no-commit[Don't put files/directories into cache.]" + {-f,--file}"[Specify name of the DVC file it generates.]:File:_files" "1:File:_files" ) @@ -67,27 +67,28 @@ _dvc_cache=( ) _dvc_checkout=( - {-f,--force}"[Do not prompt when removing working directory files.]" {-d,--with-deps}"[Checkout all dependencies of the specified target.]" + {-f,--force}"[Do not prompt when removing working directory files.]" + {-R,--recursive}"[Checkout all subdirectories of the specified directory.]" "1:Stages:_files -g '(*.dvc|Dvcfile)'" ) _dvc_commit=( "*:Stages:_files -g '(*.dvc|Dvcfile)'" - {-d,--with-deps}"[Commit all dependencies of the specified target.]" {-f,--force}"[Commit even if checksums for dependencies/outputs changed.]" + {-d,--with-deps}"[Commit all dependencies of the specified target.]" {-R,--recursive}"[Commit cache for subdirectories of the specified directory.]" ) _dvc_config=( - {-u,--unset}"[Unset option.]" - "--local[Unset option.]" - "--system[Use system config.]" "--global[Use global config.]" + "--system[Use system config.]" + "--local[Use local config.]" + {-u,--unset}"[Unset option.]" ) _dvc_destroy=( - {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]" + {-f,--force}"[Force destruction.]" ) _dvc_diff=( @@ -95,35 +96,36 @@ _dvc_diff=( ) _dvc_fetch=( + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + "--show-checksums[Show checksums instead of file names.]" + {-r,--remote}"[Remote repository to fetch from.]:Remote repository:" + {-a,--all-branches}"[Fetch cache for all branches.]" + {-T,--all-tags}"[Fetch cache for all tags.]" + {-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]" + {-R,--recursive}"[Fetch cache for subdirectories of specified directory.]" "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--show-checksums[Show checksums instead of file names]" - {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:" - {-r,--remote}"[Remote repository to pull from]:Remote repository:" - {-a,--all-branches}"[Fetch cache for all branches]" - {-T,--all-tags}"[Fetch cache for all tags]" - {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]" - {-R,--recursive}"[Fetch cache for subdirectories of the specified directory.]" ) _dvc_gc=( - {-a,--all-branches}"[Collect garbage for all branches]" - {-T,--all-tags}"[Collect garbage for all tags]" - {-c,--cloud}"[Collect garbage in remote repository]" - {-r,--remote}"[Remote repository to collect garbage in]:Remote repository:" - {-f,--force}"[Force garbage collection - automatically agree to all prompts.]" - - {-p,--projects}"[Keep data files required by these projects in addition to the current one. Useful if you share a single cache across repos.]:Repos:_files" + {-a,--all-branches}"[Keep data files for the tips of all git branches.]" + {-T,--all-tags}"[Keep data files for all git tags.]" + {-c,--cloud}"[Collect garbage in remote repository.]" + {-r,--remote}"[Remote storage to collect garbage in.]:Remote repository:" + {-f,--force}"[Force garbage collection - automatically agree to all prompts.]:Repos:_files" + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + {-p,--projects}"[Keep data files required by these projects in addition to the current one.]:Repos:_files" ) _dvc_import=( "--resume[Resume previously started download.]" + {-f,--file}"[Specify name of the DVC file it generates.]:File:_files" "1:URL:" "2:Output:" ) _dvc_init=( - "--no-scm[Initiate dvc in directory that is not tracket by any scm tool]" - {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]" + "--no-scm[Initiate dvc in directory that is not tracked by any scm tool.]" + {-f,--force}"[Overwrite existing '.dvc' directory. This operation removes local cache.]" ) _dvc_install=() @@ -146,26 +148,26 @@ _dvc_pipeline=( ) _dvc_pull=( - "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--show-checksums[Show checksums instead of file names]" - {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:" - {-r,--remote}"[Remote repository to pull from]:Remote repository:" - {-a,--all-branches}"[Fetch cache for all branches]" - {-T,--all-tags}"[Fetch cache for all tags]" - {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]" + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + "--show-checksums[Show checksums instead of file names.]" + {-r,--remote}"[Remote repository to pull from.]:Remote repository:" + {-a,--all-branches}"[Fetch cache for all branches.]" + {-T,--all-tags}"[Fetch cache for all tags.]" + {-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]" {-f,--force}"[Do not prompt when removing working directory files.]" {-R,--recursive}"[Pull cache for subdirectories of the specified directory.]" + "*:Stages:_files -g '(*.dvc|Dvcfile)'" ) _dvc_push=( + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + "--show-checksums[Show checksums instead of file names.]" + {-r,--remote}"[Remote repository to push to.]:Remote repository:" + {-a,--all-branches}"[Push cache for all branches.]" + {-T,--all-tags}"[Push cache for all tags.]" + {-d,--with-deps}"[Push cache for all dependencies of the specified target.]" + {-R,--recursive}"[Push cache from subdirectories of specified directory.]" "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--show-checksums[Show checksums instead of file names]" - {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:" - {-r,--remote}"[Remote repository to pull from]:Remote repository:" - {-a,--all-branches}"[Fetch cache for all branches]" - {-T,--all-tags}"[Fetch cache for all tags]" - {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]" - {-R,--recursive}"[Push cache for subdirectories of the specified directory.]" ) _dvc_remote=( @@ -175,53 +177,58 @@ _dvc_remote=( _dvc_remove=( "*:Stages:_files -g '(*.dvc|Dvcfile)'" "--dry[Only print the commands that would be executed without actually executing]" - {-o,--outs}"[Only remove DVC file outputs.(default)]" - {-p,--purge}"[Remove DVC file and all its outputs]" + {-o,--outs}"[Only remove DVC file outputs. (Default)]" + {-p,--purge}"[Remove DVC file and all its outputs.]" {-f,--force}"[Force purge.]" ) _dvc_repro=( - "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--dry[Only print the commands that would be executed without actually executing]" - "--ignore-build-cache[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]" - "--no-commit[Don't put files/directories into cache.]" {-f,--force}"[Reproduce even if dependencies were not changed.]" {-s,--single-item}"[Reproduce only single data item without recursive dependencies check.]" + {-c,--cwd}"[Directory within your repo to reproduce from.]:CWD:_files -/" {-m,--metrics}"[Show metrics after reproduction.]" + "--dry[Only print the commands that would be executed without actually executing]" {-i,--interactive}"[Ask for confirmation before reproducing each stage.]" {-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]" - {-P,--all-pipelines}"[Reproduce all pipelines in the project.]" - {-d,--downstream}"[Reproduce the pipeline starting from the specified stage.]" + {-P,--all-pipelines}"[Reproduce all pipelines in the repo.]" + {-R,--recursive}"[Reproduce all stages in the specified directory.]" + "--ignore-build-cache[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]" + "--no-commit[Don't put files/directories into cache.]" + "--downstream[Reproduce the pipeline starting from the specified stage.]" + "*:Stages:_files -g '(*.dvc|Dvcfile)'" ) _dvc_root=() _dvc_run=( + "*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files" + "*"{-o,--outs}"[Declare output file or directory.]:Output data:_files" + "*"{-O,--outs-no-cache}"[Declare output file or directory (do not put into DVC cache).]:Output regular:_files" + "*"{-m,--metrics}"[Declare output metric file or directory.]:Metrics:_files" + "*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache).]:Metrics (no cache):_files" + {-f,--file}"[Specify name of the DVC file it generates.]:File:_files" + {-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/" + {-w,--wdir}"[Directory within your repo to run your command in.]:WDIR:_files -/" "--no-exec[Only create stage file without actually running it.]" + {-y,--yes}"[Deprecated, use --overwrite-dvcfile instead]" "--overwrite-dvcfile[Overwrite existing dvc file without asking for confirmation.]" "--ignore-build-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]" + "--remove-outs[Deprecated, this is now the default behavior]" "--no-commit[Don't put files/directories into cache.]" - {-f,--file}"[Specify name of the stage file.]:File:_files" - {-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/" - {-w,--wdir}"[Directory to run your command and place state file in.]:WDIR:_files -/" - "*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files" - "*"{-o,--outs}"[Declare output data file or data directory.]:Output data:_files" - "*"{-O,--outs-no-cache}"[Declare output regular file or directory.]:Output regular:_files" - "*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache)]:Metrics (no cache):_files" - "*"{-m,--metrics}"[Declare output metric file or directory]:Metrics:_files" - {-y,--yes}"[Automatic 'yes' answer to all prompts.]" + "--outs-persist[Declare output file or directory that will not be removed upon repro.]:Output persistent:_files" + "--outs-persist-no-cache[Declare output file or directory that will not be removed upon repro (do not put into DVC cache).]:Output persistent regular:_files" ) _dvc_status=( + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + "--show-checksums[Show checksums instead of file names.]" + {-q,--quiet}"[Suppresses all output. Exit with 0 if pipeline is up to date, otherwise 1.]" + {-c,--cloud}"[Show status of a local cache compared to a remote repository.]" + {-r,--remote}"[Remote repository to compare local cache to.]:Remote repository:" + {-a,--all-branches}"[Show status of a local cache compared to a remote repository for all branches.]" + {-T,--all-tags}"[Show status of a local cache compared to a remote repository for all tags.]" + {-d,--with-deps}"[Show status for all dependencies of the specified target.]" "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--show-checksums[Show checksums instead of file names]" - {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:" - {-r,--remote}"[Remote repository to pull from]:Remote repository:" - {-a,--all-branches}"[Fetch cache for all branches]" - {-T,--all-tags}"[Fetch cache for all tags]" - {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]" - {-c,--cloud}"[Show status of a local cache compared to a remote repository]" - {-q,--quiet}"[Suppresses all output. Exit with 0 if pipeline is up]" ) _dvc_unlock=( From a1d3013596b413627ac0019fd70daf0c993a98bd Mon Sep 17 00:00:00 2001 From: Andrew Grigorev Date: Thu, 30 May 2019 18:24:13 +0300 Subject: [PATCH 05/30] remote: don't use threadpool for single thread --- dvc/remote/local/__init__.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/dvc/remote/local/__init__.py b/dvc/remote/local/__init__.py index 25bf1d425..3e314da4f 100644 --- a/dvc/remote/local/__init__.py +++ b/dvc/remote/local/__init__.py @@ -464,14 +464,20 @@ class RemoteLOCAL(RemoteBASE): if len(chunks) == 0: return 0 - futures = [] - with ThreadPoolExecutor(max_workers=jobs) as executor: - for from_infos, to_infos, names in chunks: - res = executor.submit(func, from_infos, to_infos, names=names) - futures.append(res) + if jobs > 1: + futures = [] + with ThreadPoolExecutor(max_workers=jobs) as executor: + for from_infos, to_infos, names in chunks: + res = executor.submit( + func, from_infos, to_infos, names=names + ) + futures.append(res) - for f in futures: - f.result() + for f in futures: + f.result() + else: + for from_infos, to_infos, names in chunks: + func(from_infos, to_infos, names=names) return len(chunks) From dba34ec3f1d86506b45842ad5c7529645172b56c Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 30 May 2019 21:10:28 +0300 Subject: [PATCH 06/30] appveyor: don't forget to modify PATH for custom python Related to https://github.com/iterative/dvc/issues/2061 --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index e9c8b6f74..156c87c89 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -32,7 +32,7 @@ environment: RUBY: C:\Ruby25\bin install: - - SET PATH=%RUBY%;%PATH% + - SET PATH=%PYTHON%;%PYTHON\Scripts;%RUBY%;%PATH% - git clone -q https://github.com/iterative/oss-emulator.git - gem install thor builder - ps: cd oss-emulator From c7b7fba3a27180b46700a098b1ecc6cb697828be Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 30 May 2019 21:38:43 +0300 Subject: [PATCH 07/30] appveyor: fix typo --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 156c87c89..dbed2bde1 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -32,7 +32,7 @@ environment: RUBY: C:\Ruby25\bin install: - - SET PATH=%PYTHON%;%PYTHON\Scripts;%RUBY%;%PATH% + - SET PATH=%PYTHON%;%PYTHON\\Scripts;%RUBY%;%PATH% - git clone -q https://github.com/iterative/oss-emulator.git - gem install thor builder - ps: cd oss-emulator From 677a52b8308acdda9af7640a7aecab8a687035af Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 30 May 2019 21:42:07 +0300 Subject: [PATCH 08/30] appveyor: use quotes --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index dbed2bde1..ec2719807 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -32,7 +32,7 @@ environment: RUBY: C:\Ruby25\bin install: - - SET PATH=%PYTHON%;%PYTHON\\Scripts;%RUBY%;%PATH% + - "SET PATH=%PYTHON%;%PYTHON\\Scripts;%RUBY%;%PATH%" - git clone -q https://github.com/iterative/oss-emulator.git - gem install thor builder - ps: cd oss-emulator From 01b8e5525998ab21151a66011b6afb47b9125474 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 30 May 2019 21:45:02 +0300 Subject: [PATCH 09/30] appveyor: install git --- .appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.appveyor.yml b/.appveyor.yml index ec2719807..a78c4c334 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -33,6 +33,7 @@ environment: install: - "SET PATH=%PYTHON%;%PYTHON\\Scripts;%RUBY%;%PATH%" + - cinst git - git clone -q https://github.com/iterative/oss-emulator.git - gem install thor builder - ps: cd oss-emulator From fede39452bfc6b96909c9fbdca37393d9289a30f Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 30 May 2019 21:46:23 +0300 Subject: [PATCH 10/30] appveyor: fix typo --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index a78c4c334..1f5f5c961 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -32,7 +32,7 @@ environment: RUBY: C:\Ruby25\bin install: - - "SET PATH=%PYTHON%;%PYTHON\\Scripts;%RUBY%;%PATH%" + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%RUBY%;%PATH%" - cinst git - git clone -q https://github.com/iterative/oss-emulator.git - gem install thor builder From ea54a2b2b4f036fc08801fb923bc3e319ed0e8aa Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 30 May 2019 21:47:47 +0300 Subject: [PATCH 11/30] appveyor: don't install git with cinst --- .appveyor.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 1f5f5c961..c7c530d3f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -33,7 +33,6 @@ environment: install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%RUBY%;%PATH%" - - cinst git - git clone -q https://github.com/iterative/oss-emulator.git - gem install thor builder - ps: cd oss-emulator From e93534abc1944aef2ed475532e63c3906ffd520e Mon Sep 17 00:00:00 2001 From: Ricardo Serradas Date: Thu, 30 May 2019 14:21:10 -0700 Subject: [PATCH 12/30] Adding a comma to the message to improve understanding --- dvc/scm/git/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 690b54c3c..dd3e1f314 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -242,7 +242,7 @@ class Git(Base): logger.info( "\n" - "To track the changes with git run:\n" + "To track the changes with git, run:\n" "\n" "\tgit add {files}".format(files=" ".join(self.files_to_track)) ) From 0935cfae2ae4268ec2c15569e049ac741784a0ac Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 31 May 2019 11:19:18 +0300 Subject: [PATCH 13/30] tests: show how long tests took --- tests/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__main__.py b/tests/__main__.py index 6a638ae93..2068d7f7d 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -15,6 +15,6 @@ params = " ".join(sys.argv[1:]) cmd = ( "py.test -v -n=4 --timeout=600 --timeout_method=thread --log-level=debug" - " --cov=dvc {params} ".format(params=params) + " --cov=dvc {params} --durations=0".format(params=params) ) check_call(cmd, shell=True) From 688bcaa0e3ca3c67d44c9cf31862f3afc8124e71 Mon Sep 17 00:00:00 2001 From: Aman Sharma Date: Sat, 1 Jun 2019 17:10:59 +0530 Subject: [PATCH 14/30] Add info for link type and filesystem name (#1997) * Add info for link and filesystem type * Remove unnecessary `repo_dir` fixture * Add cache info * Classify `test_version.py` as functional test * Remove redundancy * Optimize some functions involved * Add pathlib2 to install_requires * Display fs type for cache directory * Use `dvc.utils.compat` to import Path * Remove redundancy in tests and make logs more descriptive --- dvc/command/version.py | 80 ++++++++++++++++++++++++++++-- setup.py | 1 + tests/func/test_version.py | 37 ++++++++++++++ tests/unit/command/test_version.py | 25 ---------- 4 files changed, 114 insertions(+), 29 deletions(-) create mode 100644 tests/func/test_version.py delete mode 100644 tests/unit/command/test_version.py diff --git a/dvc/command/version.py b/dvc/command/version.py index 7b17c8f53..34be52980 100644 --- a/dvc/command/version.py +++ b/dvc/command/version.py @@ -1,11 +1,18 @@ from __future__ import unicode_literals +import os +import psutil import platform import argparse import logging +import uuid +from dvc.utils.compat import Path +from dvc.repo import Repo from dvc.command.base import CmdBaseNoRepo, append_doc_link from dvc.version import __version__ +from dvc.exceptions import DvcException, NotDvcRepoError +from dvc.system import System logger = logging.getLogger(__name__) @@ -13,18 +20,83 @@ logger = logging.getLogger(__name__) class CmdVersion(CmdBaseNoRepo): def run(self): + dvc_version = __version__ + python_version = platform.python_version() + platform_type = platform.platform() + info = ( "DVC version: {dvc_version}\n" "Python version: {python_version}\n" - "Platform: {platform_type}" + "Platform: {platform_type}\n" ).format( - dvc_version=__version__, - python_version=platform.python_version(), - platform_type=platform.platform(), + dvc_version=dvc_version, + python_version=python_version, + platform_type=platform_type, + ) + + try: + repo = Repo() + root_directory = repo.root_dir + + info += ( + "Cache: {cache}\n" + "Filesystem type (cache directory): {fs_cache}\n" + ).format( + cache=self.get_linktype_support_info(repo), + fs_cache=self.get_fs_type(repo.cache.local.cache_dir), + ) + + except NotDvcRepoError: + root_directory = os.getcwd() + + info += ("Filesystem type (workspace): {fs_root}").format( + fs_root=self.get_fs_type(os.path.abspath(root_directory)) ) logger.info(info) return 0 + @staticmethod + def get_fs_type(path): + partition = { + Path(part.mountpoint): (part.fstype, part.device) + for part in psutil.disk_partitions() + } + for parent in list(Path(path).parents): + if parent in partition: + return partition[parent] + return ("unkown", "none") + + @staticmethod + def get_linktype_support_info(repo): + links = { + "reflink": System.reflink, + "hardlink": System.hardlink, + "symlink": System.symlink, + } + + fname = "." + str(uuid.uuid4()) + src = os.path.join(repo.cache.local.cache_dir, fname) + open(src, "w").close() + dst = os.path.join(repo.root_dir, fname) + + cache = [] + + for name, link in links.items(): + try: + link(src, dst) + os.unlink(dst) + supported = True + except DvcException: + supported = False + cache.append( + "{name} - {supported}".format( + name=name, supported=True if supported else False + ) + ) + os.remove(src) + + return ", ".join(cache) + def add_parser(subparsers, parent_parser): VERSION_HELP = "Show DVC version and system/environment informaion." diff --git a/setup.py b/setup.py index 6eda216c7..16011e881 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ install_requires = [ "humanize>=0.5.1", "dulwich>=0.19.11", "ruamel.yaml>=0.15.91", + "psutil==5.6.2", ] # Extra dependencies for remote integrations diff --git a/tests/func/test_version.py b/tests/func/test_version.py new file mode 100644 index 000000000..a9089c502 --- /dev/null +++ b/tests/func/test_version.py @@ -0,0 +1,37 @@ +import re + +from dvc.main import main + + +def test_info_in_repo(dvc_repo, caplog): + assert main(["version"]) == 0 + + assert re.search(re.compile(r"DVC version: \d+\.\d+\.\d+"), caplog.text) + assert re.search(re.compile(r"Python version: \d\.\d\.\d"), caplog.text) + assert re.search(re.compile(r"Platform: .*"), caplog.text) + assert re.search( + re.compile(r"Filesystem type \(cache directory\): .*"), caplog.text + ) + assert re.search( + re.compile(r"Filesystem type \(workspace\): .*"), caplog.text + ) + assert re.search( + re.compile(r"(Cache: (.*link - (True|False)(,\s)?){3})"), caplog.text + ) + + +def test_info_outside_of_repo(repo_dir, caplog): + assert main(["version"]) == 0 + + assert re.search(re.compile(r"DVC version: \d+\.\d+\.\d+"), caplog.text) + assert re.search(re.compile(r"Python version: \d\.\d\.\d"), caplog.text) + assert re.search(re.compile(r"Platform: .*"), caplog.text) + assert re.search( + re.compile(r"Filesystem type \(workspace\): .*"), caplog.text + ) + assert not re.search( + re.compile(r"Filesystem type \(cache directory\): .*"), caplog.text + ) + assert not re.search( + re.compile(r"(Cache: (.*link - (True|False)(,\s)?){3})"), caplog.text + ) diff --git a/tests/unit/command/test_version.py b/tests/unit/command/test_version.py deleted file mode 100644 index 57c9184b5..000000000 --- a/tests/unit/command/test_version.py +++ /dev/null @@ -1,25 +0,0 @@ -import re - -from dvc.command.version import CmdVersion -from dvc.cli import parse_args - - -def test_run_in_repo(repo_dir, dvc_repo): - cmd = CmdVersion(parse_args(["version"])) - ret = cmd.run_cmd() - assert ret == 0 - - -def test_run_outside_of_repo(repo_dir): - cmd = CmdVersion(parse_args(["version"])) - ret = cmd.run_cmd() - assert ret == 0 - - -def test_info(caplog): - cmd = CmdVersion(parse_args(["version"])) - cmd.run() - - assert re.search(re.compile(r"DVC version: \d+\.\d+\.\d+"), caplog.text) - assert re.search(re.compile(r"Python version: \d\.\d\.\d"), caplog.text) - assert re.search(re.compile(r"Platform: .*"), caplog.text) From c8b2c10d7f9a6f21a54362ae24cba193db7d2a84 Mon Sep 17 00:00:00 2001 From: Kurian Benoy Date: Sat, 1 Jun 2019 23:05:47 +0530 Subject: [PATCH 15/30] scripts:add description for debian package (#2025) * scripts:add description for debian package * add DESC variable to build_posix * Fix typos --- scripts/build_posix.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_posix.sh b/scripts/build_posix.sh index ec5564316..74b86933c 100755 --- a/scripts/build_posix.sh +++ b/scripts/build_posix.sh @@ -15,6 +15,7 @@ fi BUILD_DIR=build BIN_DIR=$BUILD_DIR/$INSTALL_DIR/bin +DESC='Data Version Control - datasets, models, and experiments versioning for ML or data science projects' LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib print_error() @@ -46,6 +47,7 @@ fpm_build() fpm -s dir \ -f \ -t $1 \ + --description $DESC \ $FPM_FLAGS \ -n dvc \ -v $VERSION \ From 9a5a01b2b1573b2f594ed7437c20c439c99ea185 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 1 Jun 2019 17:52:45 -0700 Subject: [PATCH 16/30] enable deepsource analyzer --- .deepsource.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .deepsource.toml diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 000000000..3bcc0cc37 --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,15 @@ +version = 1 + +test_patterns = [ + '*/tests/*' +] + +exclude_patterns = [ + +] + +[[analyzers]] +name = 'python' +enabled = true +runtime_version = '3.x.x' + From a76b0acaed1bc02a4dbafcbc2d69f8f5f7d00507 Mon Sep 17 00:00:00 2001 From: Andrew Grigorev Date: Sun, 2 Jun 2019 13:34:06 +0300 Subject: [PATCH 17/30] ci: don't limit appveyor to master and devel --- .appveyor.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index c7c530d3f..cd270f5c7 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,8 +1,3 @@ -branches: - only: - - master - - devel - cache: - C:\ProgramData\chocolatey\bin -> appveyor.yml - C:\ProgramData\chocolatey\lib -> appveyor.yml From 616d79f2d8c3e25d2d45d9f64cbd1d6314a58c02 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 2 Jun 2019 13:59:20 +0300 Subject: [PATCH 18/30] tests: freeze pytest version --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 16011e881..ff4465b51 100644 --- a/setup.py +++ b/setup.py @@ -77,11 +77,11 @@ tests_requirements = [ "wheel>=0.31.1", "pydot>=1.2.4", # Test requirements: - "pytest>=4.4.0", - "pytest-timeout>=1.3.3", - "pytest-cov>=2.6.1", - "pytest-xdist>=1.26.1", - "pytest-mock>=1.10.4", + "pytest==4.4.0", + "pytest-timeout==1.3.3", + "pytest-cov==2.6.1", + "pytest-xdist==1.26.1", + "pytest-mock==1.10.4", "flaky>=3.5.3", "mock>=3.0.0", "xmltodict>=0.11.0", From 76c47558655ba8e6bafa8efa06f6368a12b030f6 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 2 Jun 2019 14:44:56 +0300 Subject: [PATCH 19/30] tests: test hdfs on cron and releases Related to https://github.com/iterative/dvc/issues/1629#issuecomment-498023424 Signed-off-by: Ruslan Kuprieiev --- scripts/ci/install_hadoop.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/ci/install_hadoop.sh b/scripts/ci/install_hadoop.sh index 4ef1f2ee4..7306c4a9e 100755 --- a/scripts/ci/install_hadoop.sh +++ b/scripts/ci/install_hadoop.sh @@ -5,6 +5,11 @@ set -x set -e +if [[ -z "$TRAVIS_TAG" && "$TRAVIS_EVENT_TYPE" != "cron" ]]; then + echo "Skipping installing hadoop." + exit 0 +fi + sudo apt-get update -y sudo apt-get install openjdk-8-jdk java -version From 122b94129f1d949a1cafab24301ca8d491165202 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 2 Jun 2019 15:07:36 +0300 Subject: [PATCH 20/30] Revert "tests: test hdfs on cron and releases" This reverts commit 76c47558655ba8e6bafa8efa06f6368a12b030f6. --- scripts/ci/install_hadoop.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/ci/install_hadoop.sh b/scripts/ci/install_hadoop.sh index 7306c4a9e..4ef1f2ee4 100755 --- a/scripts/ci/install_hadoop.sh +++ b/scripts/ci/install_hadoop.sh @@ -5,11 +5,6 @@ set -x set -e -if [[ -z "$TRAVIS_TAG" && "$TRAVIS_EVENT_TYPE" != "cron" ]]; then - echo "Skipping installing hadoop." - exit 0 -fi - sudo apt-get update -y sudo apt-get install openjdk-8-jdk java -version From 5fce9a01e446cb8f6aa4162c51467e09fe45cfa1 Mon Sep 17 00:00:00 2001 From: Andrew Grigorev Date: Sun, 2 Jun 2019 15:41:21 +0300 Subject: [PATCH 21/30] tests: fix for pytest>=4.6.0 caplog --- setup.py | 10 +++++----- tests/func/test_add.py | 2 +- tests/func/test_data_cloud.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index ff4465b51..d8a0c4972 100644 --- a/setup.py +++ b/setup.py @@ -77,11 +77,11 @@ tests_requirements = [ "wheel>=0.31.1", "pydot>=1.2.4", # Test requirements: - "pytest==4.4.0", - "pytest-timeout==1.3.3", - "pytest-cov==2.6.1", - "pytest-xdist==1.26.1", - "pytest-mock==1.10.4", + "pytest>=4.6.0", + "pytest-timeout>=1.3.3", + "pytest-cov>=2.6.1", + "pytest-xdist>=1.26.1", + "pytest-mock>=1.10.4", "flaky>=3.5.3", "mock>=3.0.0", "xmltodict>=0.11.0", diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 0105c93da..9cef92c8c 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -114,7 +114,7 @@ class TestAddCmdDirectoryRecursive(TestDvc): with self._caplog.at_level(logging.WARNING, logger="dvc"): assert main(["add", "--recursive", "large-dir"]) == 0 - assert warning in self._caplog.text + assert warning in self._caplog.messages class TestAddDirectoryWithForwardSlash(TestDvc): diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index d0e20bbed..987b7029e 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -870,12 +870,12 @@ class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc): checksum_bar = file_md5(self.BAR)[0] self.message_header = ( "Some of the cache files do not exist neither locally " - "nor on remote. Missing cache files: \n" + "nor on remote. Missing cache files: " ) - self.message_bar_part = "\nname: {}, md5: {}\n".format( + self.message_bar_part = "name: {}, md5: {}".format( self.BAR, checksum_bar ) - self.message_foo_part = "\nname: {}, md5: {}\n".format( + self.message_foo_part = "name: {}, md5: {}".format( self.FOO, checksum_foo ) From 67e6dc253a72cd884e908d7a031f90845445453a Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Mon, 3 Jun 2019 12:17:48 -0700 Subject: [PATCH 22/30] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index fabda7040..e93869c9f 100644 --- a/README.rst +++ b/README.rst @@ -31,8 +31,8 @@ | -**Data Science Version Control** or **DVC** is an **open-source** tool for data science and -machine learning projects. Key features: +**Data Version Control** or **DVC** is an **open-source** tool for data science and machine +learning projects. Key features: #. simple **command line** Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services; From f41dcea248277b8ede99d6d30df33ad5b4b2cf28 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 6 Jun 2019 13:44:13 +0300 Subject: [PATCH 23/30] tests: don't use var before assignment ``` repo_dir = git = @pytest.fixture def dvc_repo(repo_dir, git): try: dvc = DvcRepo.init(repo_dir._root_dir) dvc.scm.commit("init dvc") yield dvc finally: > dvc.scm.git.close() E UnboundLocalError: local variable 'dvc' referenced before assignment /home/efiop/git/dvc/tests/conftest.py:76: UnboundLocalError ``` --- tests/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9de032b64..7e7d1208e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -68,12 +68,14 @@ def git(repo_dir): @pytest.fixture def dvc_repo(repo_dir, git): + dvc = None try: dvc = DvcRepo.init(repo_dir._root_dir) dvc.scm.commit("init dvc") yield dvc finally: - dvc.scm.git.close() + if dvc: + dvc.scm.git.close() here = os.path.abspath(os.path.dirname(__file__)) From 9e9e078de3f3dec52bb5d60163c0073ffd7117fd Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 31 May 2019 10:31:44 +0300 Subject: [PATCH 24/30] config: move logic from CLI to API Signed-off-by: Ruslan Kuprieiev --- dvc/cache.py | 5 +- dvc/command/cache.py | 10 +- dvc/command/config.py | 85 ++------ dvc/command/remote.py | 127 +++-------- dvc/config.py | 387 ++++++++++++++++------------------ dvc/data_cloud.py | 4 +- dvc/dependency/__init__.py | 3 +- dvc/main.py | 4 + dvc/output/__init__.py | 3 +- dvc/remote/__init__.py | 11 +- dvc/remote/config.py | 155 ++++++++++++++ dvc/repo/__init__.py | 16 +- tests/basic_env.py | 9 +- tests/func/test_config.py | 27 +-- tests/func/test_data_cloud.py | 1 + tests/func/test_remote.py | 19 +- 16 files changed, 426 insertions(+), 440 deletions(-) create mode 100644 dvc/remote/config.py diff --git a/dvc/cache.py b/dvc/cache.py index e1b9d7b5d..48432ea3b 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -42,7 +42,7 @@ class Cache(object): Config.SECTION_CACHE_PROTECTED: protected, } - self.local = Remote(repo, settings) + self.local = Remote(repo, **settings) self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3) self.gs = self._get_remote(config, Config.SECTION_CACHE_GS) self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH) @@ -87,5 +87,4 @@ class Cache(object): if not remote: return None - settings = self.repo.config.get_remote_settings(remote) - return Remote(self.repo, settings) + return Remote(self.repo, name=remote) diff --git a/dvc/command/cache.py b/dvc/command/cache.py index 80b189cae..c2ac03d20 100644 --- a/dvc/command/cache.py +++ b/dvc/command/cache.py @@ -3,18 +3,14 @@ from __future__ import unicode_literals import argparse from dvc.command.base import fix_subparsers, append_doc_link -from dvc.command.remote import CmdRemoteAdd from dvc.command.config import CmdConfig +from dvc.remote.config import set_cache_dir class CmdCacheDir(CmdConfig): def run(self): - self.args.name = "cache.dir" - self.args.value = CmdRemoteAdd.resolve_path( - self.args.value, self.configobj.filename - ) - - return super(CmdCacheDir, self).run() + set_cache_dir(self.config, self.args.value, level=self.args.level) + return 0 def add_parser(subparsers, parent_parser): diff --git a/dvc/command/config.py b/dvc/command/config.py index f62ac5d7d..8e3bdba40 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -1,12 +1,10 @@ from __future__ import unicode_literals -import os import argparse import logging from dvc.command.base import CmdBaseNoRepo, append_doc_link from dvc.config import Config -from dvc.exceptions import DvcException logger = logging.getLogger(__name__) @@ -14,87 +12,48 @@ logger = logging.getLogger(__name__) class CmdConfig(CmdBaseNoRepo): def __init__(self, args): - from dvc.repo import Repo, NotDvcRepoError - super(CmdConfig, self).__init__(args) - try: - dvc_dir = os.path.join(Repo.find_root(), Repo.DVC_DIR) - saved_exc = None - except NotDvcRepoError as exc: - dvc_dir = None - saved_exc = exc - - self.config = Config(dvc_dir, validate=False) - if self.args.system: - self.configobj = self.config._system_config - elif self.args.glob: - self.configobj = self.config._global_config - elif self.args.local: - if dvc_dir is None: - raise saved_exc - self.configobj = self.config._local_config - else: - if dvc_dir is None: - raise saved_exc - self.configobj = self.config._repo_config - - def _unset(self, section, opt=None, configobj=None): - if configobj is None: - configobj = self.configobj - - try: - self.config.unset(configobj, section, opt) - self.config.save(configobj) - except DvcException: - logger.exception("failed to unset '{}'".format(self.args.name)) - return 1 - return 0 - - def _show(self, section, opt): - try: - self.config.show(self.configobj, section, opt) - except DvcException: - logger.exception("failed to show '{}'".format(self.args.name)) - return 1 - return 0 - - def _set(self, section, opt, value): - try: - self.config.set(self.configobj, section, opt, value) - self.config.save(self.configobj) - except DvcException: - logger.exception( - "failed to set '{}.{}' to '{}'".format(section, opt, value) - ) - return 1 - return 0 + self.config = Config(validate=False) def run(self): section, opt = self.args.name.lower().strip().split(".", 1) if self.args.unset: - return self._unset(section, opt) + self.config.unset(section, opt, level=self.args.level) elif self.args.value is None: - return self._show(section, opt) + logger.info(self.config.get(section, opt, level=self.args.level)) else: - return self._set(section, opt, self.args.value) + self.config.set( + section, opt, self.args.value, level=self.args.level + ) + + return 0 parent_config_parser = argparse.ArgumentParser(add_help=False) parent_config_parser.add_argument( "--global", - dest="glob", - action="store_true", - default=False, + dest="level", + action="store_const", + const=Config.LEVEL_GLOBAL, help="Use global config.", ) parent_config_parser.add_argument( - "--system", action="store_true", default=False, help="Use system config." + "--system", + dest="level", + action="store_const", + const=Config.LEVEL_SYSTEM, + help="Use system config.", ) parent_config_parser.add_argument( - "--local", action="store_true", default=False, help="Use local config." + "--local", + dest="level", + action="store_const", + const=Config.LEVEL_LOCAL, + help="Use local config.", ) +parent_config_parser.set_defaults(level=Config.LEVEL_REPO) def add_parser(subparsers, parent_parser): diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 803592ef4..2fe5b1f92 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -1,130 +1,63 @@ from __future__ import unicode_literals import argparse -import os -import re import logging from dvc.command.base import append_doc_link, fix_subparsers from dvc.command.config import CmdConfig -from dvc.config import Config - +from dvc.remote.config import remote_add, remote_remove, remote_modify +from dvc.remote.config import remote_list, remote_default logger = logging.getLogger(__name__) class CmdRemoteAdd(CmdConfig): - @staticmethod - def resolve_path(path, config_file): - """Resolve path relative to config file location. - - Args: - path: Path to be resolved. - config_file: Path to config file, which `path` is specified - relative to. - - Returns: - Path relative to the `config_file` location. If `path` is an - absolute path then it will be returned without change. - - """ - if os.path.isabs(path): - return path - return os.path.relpath(path, os.path.dirname(config_file)) - def run(self): - from dvc.remote import _get, RemoteLOCAL - - remote = _get({Config.SECTION_REMOTE_URL: self.args.url}) - if remote == RemoteLOCAL and not self.args.url.startswith("remote://"): - self.args.url = self.resolve_path( - self.args.url, self.configobj.filename - ) - - section = Config.SECTION_REMOTE_FMT.format(self.args.name) - if (section in self.configobj.keys()) and not self.args.force: - logger.error( - "Remote with name {} already exists. " - "Use -f (--force) to overwrite remote " - "with new value".format(self.args.name) - ) - return 1 - - ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url) - if ret != 0: - return ret - - if self.args.default: - msg = "Setting '{}' as a default remote.".format(self.args.name) - logger.info(msg) - ret = self._set( - Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, self.args.name - ) - - return ret + remote_add( + self.config, + self.args.name, + self.args.url, + force=self.args.force, + default=self.args.default, + level=self.args.level, + ) + return 0 class CmdRemoteRemove(CmdConfig): - def _remove_default(self, config): - core = config.get(Config.SECTION_CORE, None) - if core is None: - return 0 - - default = core.get(Config.SECTION_CORE_REMOTE, None) - if default is None: - return 0 - - if default == self.args.name: - return self._unset( - Config.SECTION_CORE, - opt=Config.SECTION_CORE_REMOTE, - configobj=config, - ) - def run(self): - section = Config.SECTION_REMOTE_FMT.format(self.args.name) - ret = self._unset(section) - if ret != 0: - return ret - - for configobj in [ - self.config._local_config, - self.config._repo_config, - self.config._global_config, - self.config._system_config, - ]: - self._remove_default(configobj) - self.config.save(configobj) - if configobj == self.configobj: - break - + remote_remove(self.config, self.args.name, level=self.args.level) return 0 class CmdRemoteModify(CmdConfig): def run(self): - section = Config.SECTION_REMOTE_FMT.format(self.args.name) - self.args.name = "{}.{}".format(section, self.args.option) - return super(CmdRemoteModify, self).run() + remote_modify( + self.config, + self.args.name, + self.args.option, + self.args.value, + level=self.args.level, + ) + return 0 class CmdRemoteDefault(CmdConfig): def run(self): - self.args.value = self.args.name - self.args.name = "core.remote" - return super(CmdRemoteDefault, self).run() + remote_default( + self.config, + self.args.name, + unset=self.args.unset, + level=self.args.level, + ) + return 0 class CmdRemoteList(CmdConfig): def run(self): - for section in self.configobj.keys(): - r = re.match(Config.SECTION_REMOTE_REGEX, section) - if r: - name = r.group("name") - url = self.configobj[section].get( - Config.SECTION_REMOTE_URL, "" - ) - logger.info("{}\t{}".format(name, url)) + remotes = remote_list(self.config, level=self.args.level) + for name, url in remotes.items(): + logger.info("{}\t{}".format(name, url)) return 0 diff --git a/dvc/config.py b/dvc/config.py index 40e705e27..555a8ca15 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -2,15 +2,17 @@ from __future__ import unicode_literals -from dvc.utils.compat import str, open, urlparse +from dvc.utils.compat import str, open import os +import re +import copy import errno import configobj import logging -from schema import Schema, Optional, And, Use, Regex -from dvc.exceptions import DvcException +from schema import Schema, Optional, And, Use, Regex, SchemaError +from dvc.exceptions import DvcException, NotDvcRepoError logger = logging.getLogger(__name__) @@ -20,12 +22,12 @@ class ConfigError(DvcException): Args: msg (str): error message. - ex (Exception): optional exception that has caused this error. + cause (Exception): optional exception that has caused this error. """ - def __init__(self, msg, ex=None): + def __init__(self, msg, cause=None): super(ConfigError, self).__init__( - "config file error: {}".format(msg), ex + "config file error: {}".format(msg), cause=cause ) @@ -134,6 +136,11 @@ class Config(object): # pylint: disable=too-many-instance-attributes CONFIG = "config" CONFIG_LOCAL = "config.local" + LEVEL_LOCAL = 0 + LEVEL_REPO = 1 + LEVEL_GLOBAL = 2 + LEVEL_SYSTEM = 3 + BOOL_SCHEMA = And(str, is_bool, Use(to_bool)) SECTION_CORE = "core" @@ -285,30 +292,20 @@ class Config(object): # pylint: disable=too-many-instance-attributes } def __init__(self, dvc_dir=None, validate=True): - self.system_config_file = os.path.join( - self.get_system_config_dir(), self.CONFIG - ) - self.global_config_file = os.path.join( - self.get_global_config_dir(), self.CONFIG - ) + self.dvc_dir = dvc_dir + self.validate = validate - if dvc_dir is not None: - self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) - self.config_file = os.path.join(dvc_dir, self.CONFIG) - self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL) + if not dvc_dir: + try: + from dvc.repo import Repo + + self.dvc_dir = os.path.join(Repo.find_dvc_dir()) + except NotDvcRepoError: + self.dvc_dir = None else: - self.dvc_dir = None - self.config_file = None - self.config_local_file = None + self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) - self._system_config = None - self._global_config = None - self._repo_config = None - self._local_config = None - - self.config = None - - self.load(validate=validate) + self.load() @staticmethod def get_global_config_dir(): @@ -350,35 +347,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes open(config_file, "w+").close() return Config(dvc_dir) - def _load(self): - self._system_config = configobj.ConfigObj(self.system_config_file) - self._global_config = configobj.ConfigObj(self.global_config_file) - - if self.config_file is not None: - self._repo_config = configobj.ConfigObj(self.config_file) - else: - self._repo_config = configobj.ConfigObj() - - if self.config_local_file is not None: - self._local_config = configobj.ConfigObj(self.config_local_file) - else: - self._local_config = configobj.ConfigObj() - - self.config = None - - def _load_config(self, path): - config = configobj.ConfigObj(path) - config = self._lower(config) - self._resolve_paths(config, path) - return config - - @staticmethod - def _resolve_path(path, config_file): - assert os.path.isabs(config_file) - config_dir = os.path.dirname(config_file) - return os.path.abspath(os.path.join(config_dir, path)) - - def _resolve_cache_path(self, config, fname): + def _resolve_cache_path(self, config): cache = config.get(self.SECTION_CACHE) if cache is None: return @@ -387,72 +356,86 @@ class Config(object): # pylint: disable=too-many-instance-attributes if cache_dir is None: return - cache[self.PRIVATE_CWD] = os.path.dirname(fname) + cache[self.PRIVATE_CWD] = os.path.dirname(config.filename) - def _resolve_paths(self, config, fname): - if fname is None: - return + def _resolve_paths(self, config): + if config.filename is None: + return config - self._resolve_cache_path(config, fname) - for section in config.values(): + ret = copy.deepcopy(config) + self._resolve_cache_path(ret) + + for section in ret.values(): if self.SECTION_REMOTE_URL not in section.keys(): continue - section[self.PRIVATE_CWD] = os.path.dirname(fname) + section[self.PRIVATE_CWD] = os.path.dirname(ret.filename) - def load(self, validate=True): + return ret + + def _load_configs(self): + system_config_file = os.path.join( + self.get_system_config_dir(), self.CONFIG + ) + + global_config_file = os.path.join( + self.get_global_config_dir(), self.CONFIG + ) + + self._system_config = configobj.ConfigObj(system_config_file) + self._global_config = configobj.ConfigObj(global_config_file) + self._repo_config = configobj.ConfigObj() + self._local_config = configobj.ConfigObj() + + if not self.dvc_dir: + return + + config_file = os.path.join(self.dvc_dir, self.CONFIG) + config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL) + + self._repo_config = configobj.ConfigObj(config_file) + self._local_config = configobj.ConfigObj(config_local_file) + + @property + def config_local_file(self): + return self._local_config.filename + + @property + def config_file(self): + return self._repo_config.filename + + def load(self): """Loads config from all the config files. - Args: - validate (bool): optional flag to tell dvc if it should validate - the config or just load it as is. 'True' by default. - - Raises: dvc.config.ConfigError: thrown if config has invalid format. """ - self._load() + self._load_configs() + + self.config = configobj.ConfigObj() + for c in [ + self._system_config, + self._global_config, + self._repo_config, + self._local_config, + ]: + c = self._resolve_paths(c) + c = self._lower(c) + self.config.merge(c) + + if not self.validate: + return + + d = self.config.dict() try: - self.config = self._load_config(self.system_config_file) - user = self._load_config(self.global_config_file) - config = self._load_config(self.config_file) - local = self._load_config(self.config_local_file) - - # NOTE: schema doesn't support ConfigObj.Section validation, so we - # need to convert our config to dict before passing it to - for conf in [user, config, local]: - self.config = self._merge(self.config, conf) - - if validate: - self.config = Schema(self.SCHEMA).validate(self.config) - - # NOTE: now converting back to ConfigObj - self.config = configobj.ConfigObj( - self.config, write_empty_values=True - ) - self.config.filename = self.config_file - self._resolve_paths(self.config, self.config_file) - except Exception as ex: - raise ConfigError(ex) - - @staticmethod - def _get_key(conf, name, add=False): - for k in conf.keys(): - if k.lower() == name.lower(): - return k - - if add: - conf[name] = {} - return name - - return None + d = Schema(self.SCHEMA).validate(d) + except SchemaError as exc: + raise ConfigError("config format error", cause=exc) + self.config = configobj.ConfigObj(d, write_empty_values=True) def save(self, config=None): """Saves config to config files. - Args: - config (configobj.ConfigObj): optional config object to save. - Raises: dvc.config.ConfigError: thrown if failed to write config file. """ @@ -467,129 +450,103 @@ class Config(object): # pylint: disable=too-many-instance-attributes ] for conf in clist: - if conf.filename is None: - continue + self._save(conf) - try: - logger.debug("Writing '{}'.".format(conf.filename)) - dname = os.path.dirname(os.path.abspath(conf.filename)) - try: - os.makedirs(dname) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - conf.write() - except Exception as exc: - msg = "failed to write config '{}'".format(conf.filename) - raise ConfigError(msg, exc) - - def get_remote_settings(self, name): - import posixpath - - """ - Args: - name (str): The name of the remote that we want to retrieve - - Returns: - dict: The content beneath the given remote name. - - Example: - >>> config = {'remote "server"': {'url': 'ssh://localhost/'}} - >>> get_remote_settings("server") - {'url': 'ssh://localhost/'} - """ - settings = self.config.get( - self.SECTION_REMOTE_FMT.format(name.lower()) - ) - - if settings is None: - raise ConfigError( - "unable to find remote section '{}'".format(name) - ) - - parsed = urlparse(settings["url"]) - - # Support for cross referenced remotes. - # This will merge the settings, giving priority to the outer reference. - # For example, having: - # - # dvc remote add server ssh://localhost - # dvc remote modify server user root - # dvc remote modify server ask_password true - # - # dvc remote add images remote://server/tmp/pictures - # dvc remote modify images user alice - # dvc remote modify images ask_password false - # dvc remote modify images password asdf1234 - # - # Results on a config dictionary like: - # - # { - # "url": "ssh://localhost/tmp/pictures", - # "user": "alice", - # "password": "asdf1234", - # "ask_password": False, - # } - # - if parsed.scheme == "remote": - reference = self.get_remote_settings(parsed.netloc) - url = posixpath.join(reference["url"], parsed.path.lstrip("/")) - merged = reference.copy() - merged.update(settings) - merged["url"] = url - return merged - - return settings + self.load() @staticmethod - def unset(config, section, opt=None): + def _save(config): + if config.filename is None: + return + + logger.debug("Writing '{}'.".format(config.filename)) + dname = os.path.dirname(os.path.abspath(config.filename)) + try: + os.makedirs(dname) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + config.write() + + def unset(self, section, opt=None, level=None, force=False): """Unsets specified option and/or section in the config. Args: - config (configobj.ConfigObj): config to work on. section (str): section name. opt (str): optional option name. + level (int): config level to use. + force (bool): don't error-out even if section doesn't exist. False + by default. + + Raises: + dvc.config.ConfigError: thrown if section doesn't exist and + `force != True`. """ + config = self.get_configobj(level) + if section not in config.keys(): + if force: + return raise ConfigError("section '{}' doesn't exist".format(section)) - if opt is None: - del config[section] - return + if opt: + if opt not in config[section].keys(): + if force: + return + raise ConfigError( + "option '{}.{}' doesn't exist".format(section, opt) + ) + del config[section][opt] - if opt not in config[section].keys(): - raise ConfigError( - "option '{}.{}' doesn't exist".format(section, opt) - ) - del config[section][opt] - - if not config[section]: + if not config[section]: + del config[section] + else: del config[section] - @staticmethod - def set(config, section, opt, value): + self.save(config) + + def set(self, section, opt, value, level=None, force=True): """Sets specified option in the config. Args: - config (configobj.ConfigObj): config to work on. section (str): section name. opt (str): option name. value: value to set option to. + level (int): config level to use. + force (bool): set option even if section already exists. True by + default. + + Raises: + dvc.config.ConfigError: thrown if section already exists and + `force != True`. + """ + config = self.get_configobj(level) + if section not in config.keys(): config[section] = {} + elif not force: + raise ConfigError( + "Section '{}' already exists. Use `-f|--force` to overwrite " + "section with new value.".format(section) + ) config[section][opt] = value + self.save(config) - @staticmethod - def show(config, section, opt): - """Prints option value from the config. + def get(self, section, opt, level=None): + """Return option value from the config. Args: - config (configobj.ConfigObj): config to work on. section (str): section name. opt (str): option name. + level (int): config level to use. + + Returns: + value (str, int): option value. """ + config = self.get_configobj(level) + if section not in config.keys(): raise ConfigError("section '{}' doesn't exist".format(section)) @@ -598,25 +555,35 @@ class Config(object): # pylint: disable=too-many-instance-attributes "option '{}.{}' doesn't exist".format(section, opt) ) - logger.info(config[section][opt]) - - @staticmethod - def _merge(first, second): - res = {} - sections = list(first.keys()) + list(second.keys()) - for section in sections: - first_copy = first.get(section, {}).copy() - second_copy = second.get(section, {}).copy() - first_copy.update(second_copy) - res[section] = first_copy - return res + return config[section][opt] @staticmethod def _lower(config): - new_config = {} + new_config = configobj.ConfigObj() for s_key, s_value in config.items(): new_s = {} for key, value in s_value.items(): new_s[key.lower()] = str(value) new_config[s_key.lower()] = new_s return new_config + + def get_configobj(self, level): + configs = { + self.LEVEL_LOCAL: self._local_config, + self.LEVEL_REPO: self._repo_config, + self.LEVEL_GLOBAL: self._global_config, + self.LEVEL_SYSTEM: self._system_config, + } + + return configs.get(level, self._repo_config) + + def list_options(self, section_regex, option, level=None): + ret = {} + config = self.get_configobj(level) + for section in config.keys(): + r = re.match(section_regex, section) + if r: + name = r.group("name") + value = config[section].get(option, "") + ret[name] = value + return ret diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 5802826af..d55eb86fd 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -25,7 +25,6 @@ class DataCloud(object): Args: repo (dvc.repo.Repo): repo instance that belongs to the repo that we are working on. - config (dict): config of the repo. Raises: config.ConfigError: thrown when config has invalid format. @@ -75,8 +74,7 @@ class DataCloud(object): ) def _init_remote(self, remote): - config = self.repo.config.get_remote_settings(remote) - return Remote(self.repo, config) + return Remote(self.repo, name=remote) def _init_compat(self): name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower() diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 5c5ba67d4..78ac2da31 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -52,8 +52,7 @@ def _get(stage, p, info): parsed = urlparse(p) if parsed.scheme == "remote": - settings = stage.repo.config.get_remote_settings(parsed.netloc) - remote = Remote(stage.repo, settings) + remote = Remote(stage.repo, name=parsed.netloc) return DEP_MAP[remote.scheme](stage, p, info, remote=remote) for d in DEPS: diff --git a/dvc/main.py b/dvc/main.py index 12db9e112..f00dc0ced 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals import logging from dvc.cli import parse_args +from dvc.config import ConfigError from dvc.analytics import Analytics from dvc.exceptions import NotDvcRepoError, DvcParserError @@ -36,6 +37,9 @@ def main(argv=None): cmd = args.func(args) ret = cmd.run_cmd() + except ConfigError: + logger.exception("configuration error") + ret = 251 except KeyboardInterrupt: logger.exception("interrupted by the user") ret = 252 diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index 60db6a50c..ca8ad7777 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -61,8 +61,7 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None): parsed = urlparse(p) if parsed.scheme == "remote": - settings = stage.repo.config.get_remote_settings(parsed.netloc) - remote = Remote(stage.repo, settings) + remote = Remote(stage.repo, name=parsed.netloc) return OUTS_MAP[remote.scheme]( stage, p, diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index 4eb124199..f5ed528e0 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -10,6 +10,8 @@ from dvc.remote.http import RemoteHTTP from dvc.remote.https import RemoteHTTPS from dvc.remote.oss import RemoteOSS +from .config import get_remote_settings + REMOTES = [ RemoteAZURE, @@ -31,5 +33,10 @@ def _get(config): return RemoteLOCAL -def Remote(repo, config): - return _get(config)(repo, config) +def Remote(repo, **kwargs): + name = kwargs.get("name") + if name: + settings = get_remote_settings(repo.config, name) + else: + settings = kwargs + return _get(settings)(repo, settings) diff --git a/dvc/remote/config.py b/dvc/remote/config.py new file mode 100644 index 000000000..5b8672330 --- /dev/null +++ b/dvc/remote/config.py @@ -0,0 +1,155 @@ +import os +import logging +import posixpath + +from dvc.utils.compat import urlparse +from dvc.config import Config, ConfigError + + +logger = logging.getLogger(__name__) + + +def get_remote_settings(config, name): + """ + Args: + name (str): The name of the remote that we want to retrieve + + Returns: + dict: The content beneath the given remote name. + + Example: + >>> config = {'remote "server"': {'url': 'ssh://localhost/'}} + >>> get_remote_settings("server") + {'url': 'ssh://localhost/'} + """ + settings = config.config.get( + config.SECTION_REMOTE_FMT.format(name.lower()) + ) + + if settings is None: + raise ConfigError("unable to find remote section '{}'".format(name)) + + parsed = urlparse(settings["url"]) + + # Support for cross referenced remotes. + # This will merge the settings, giving priority to the outer reference. + # For example, having: + # + # dvc remote add server ssh://localhost + # dvc remote modify server user root + # dvc remote modify server ask_password true + # + # dvc remote add images remote://server/tmp/pictures + # dvc remote modify images user alice + # dvc remote modify images ask_password false + # dvc remote modify images password asdf1234 + # + # Results on a config dictionary like: + # + # { + # "url": "ssh://localhost/tmp/pictures", + # "user": "alice", + # "password": "asdf1234", + # "ask_password": False, + # } + # + if parsed.scheme == "remote": + reference = get_remote_settings(config, parsed.netloc) + url = posixpath.join(reference["url"], parsed.path.lstrip("/")) + merged = reference.copy() + merged.update(settings) + merged["url"] = url + return merged + + return settings + + +def _resolve_remote_path(path, config_file): + """Resolve path relative to config file location. + + Args: + path: Path to be resolved. + config_file: Path to config file, which `path` is specified + relative to. + + Returns: + Path relative to the `config_file` location. If `path` is an + absolute path then it will be returned without change. + + """ + if os.path.isabs(path): + return path + return os.path.relpath(path, os.path.dirname(config_file)) + + +def remote_add(config, name, url, default=False, force=False, level=None): + from dvc.remote import _get, RemoteLOCAL + + configobj = config.get_configobj(level) + remote = _get({Config.SECTION_REMOTE_URL: url}) + if remote == RemoteLOCAL and not url.startswith("remote://"): + url = _resolve_remote_path(url, configobj.filename) + + config.set( + config.SECTION_REMOTE_FMT.format(name), + config.SECTION_REMOTE_URL, + url, + force=force, + level=level, + ) + if default: + logger.info("Setting '{}' as a default remote.".format(name)) + config.set( + config.SECTION_CORE, config.SECTION_CORE_REMOTE, name, level=level + ) + + +def remote_remove(config, name, level=None): + config.unset(config.SECTION_REMOTE_FMT.format(name), level=level) + + if level is None: + level = config.LEVEL_REPO + + for lev in [ + config.LEVEL_LOCAL, + config.LEVEL_REPO, + config.LEVEL_GLOBAL, + config.LEVEL_SYSTEM, + ]: + config.unset( + config.SECTION_CORE, + config.SECTION_CORE_REMOTE, + level=lev, + force=True, + ) + if lev == level: + break + + +def remote_modify(config, name, option, value, level=None): + config.set( + config.SECTION_REMOTE_FMT.format(name), option, value, level=level + ) + + +def remote_list(config, level=None): + return config.list_options( + config.SECTION_REMOTE_REGEX, config.SECTION_REMOTE_URL, level=level + ) + + +def remote_default(config, name, unset=False, level=None): + if unset: + config.unset(config.SECTION_CORE, config.SECTION_CORE_REMOTE) + return + config.set( + config.SECTION_CORE, config.SECTION_CORE_REMOTE, name, level=level + ) + + +def set_cache_dir(config, dname, level=None): + configobj = config.get_configobj(level) + path = _resolve_remote_path(dname, configobj.filename) + config.set( + config.SECTION_CACHE, config.SECTION_CACHE_DIR, path, level=level + ) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 0f43957c1..101e9c300 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -5,6 +5,7 @@ import logging import dvc.prompt as prompt +from dvc.config import Config from dvc.exceptions import ( DvcException, NotDvcRepoError, @@ -41,7 +42,6 @@ class Repo(object): from dvc.repo.brancher import brancher def __init__(self, root_dir=None): - from dvc.config import Config from dvc.state import State from dvc.lock import Lock from dvc.scm import SCM @@ -89,15 +89,15 @@ class Repo(object): def __repr__(self): return "Repo: '{root_dir}'".format(root_dir=self.root_dir) - @staticmethod - def find_root(root=None): + @classmethod + def find_root(cls, root=None): if root is None: root = os.getcwd() else: root = os.path.abspath(os.path.realpath(root)) while True: - dvc_dir = os.path.join(root, Repo.DVC_DIR) + dvc_dir = os.path.join(root, cls.DVC_DIR) if os.path.isdir(dvc_dir): return root if os.path.ismount(root): @@ -105,10 +105,10 @@ class Repo(object): root = os.path.dirname(root) raise NotDvcRepoError(root) - @staticmethod - def find_dvc_dir(root=None): - root_dir = Repo.find_root(root) - return os.path.join(root_dir, Repo.DVC_DIR) + @classmethod + def find_dvc_dir(cls, root=None): + root_dir = cls.find_root(root) + return os.path.join(root_dir, cls.DVC_DIR) @staticmethod def init(root_dir=os.curdir, no_scm=False, force=False): diff --git a/tests/basic_env.py b/tests/basic_env.py index 0c0b7ab70..8fc20e08c 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -57,6 +57,10 @@ class TestDirFixture(object): root_dir = self.mkdtemp() self._root_dir = os.path.realpath(root_dir) + @property + def root_dir(self): + return self._root_dir + def _pushd(self, d): if not hasattr(self, "_saved_dir"): self._saved_dir = os.path.realpath(os.curdir) @@ -193,9 +197,8 @@ class TestDvcDataFileFixture(TestDvcGitInitializedFixture): shutil.copytree(self.dvc.cache.local.cache_dir, self.cache_dir) class MockConfig: - system = None - glob = None - local = None + level = 1 + force = False default = True name = self.REMOTE url = self.cache_dir diff --git a/tests/func/test_config.py b/tests/func/test_config.py index 0de3bafd9..25d11cfb5 100644 --- a/tests/func/test_config.py +++ b/tests/func/test_config.py @@ -1,7 +1,6 @@ import configobj from dvc.main import main -from dvc.command.config import CmdConfig from tests.basic_env import TestDvc @@ -70,35 +69,19 @@ class TestConfigCLI(TestDvc): # FIXME check stdout/err ret = main(["config", "non_existing_section.field"]) - self.assertEqual(ret, 1) + self.assertEqual(ret, 251) ret = main(["config", "global.non_existing_field"]) - self.assertEqual(ret, 1) + self.assertEqual(ret, 251) ret = main(["config", "non_existing_section.field", "-u"]) - self.assertEqual(ret, 1) + self.assertEqual(ret, 251) ret = main(["config", "global.non_existing_field", "-u"]) - self.assertEqual(ret, 1) + self.assertEqual(ret, 251) ret = main(["config", "core.remote", "myremote"]) self.assertEqual(ret, 0) ret = main(["config", "core.non_existing_field", "-u"]) - self.assertEqual(ret, 1) - - def test_failed_write(self): - class A(object): - system = False - glob = False - local = False - name = "core.remote" - value = "myremote" - unset = False - - args = A() - cmd = CmdConfig(args) - - cmd.configobj.write = None - ret = cmd.run() - self.assertNotEqual(ret, 0) + self.assertEqual(ret, 251) diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 987b7029e..97daa1a91 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -39,6 +39,7 @@ from tests.utils import spy TEST_REMOTE = "upstream" TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE) TEST_CONFIG = { + Config.SECTION_CACHE: {}, Config.SECTION_CORE: {Config.SECTION_CORE_REMOTE: TEST_REMOTE}, TEST_SECTION: {Config.SECTION_REMOTE_URL: ""}, } diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index fd403cc1b..9f1b07d59 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -5,7 +5,6 @@ from dvc.remote.base import RemoteBASE from mock import patch from dvc.main import main -from dvc.command.remote import CmdRemoteAdd from dvc.config import Config from tests.basic_env import TestDvc @@ -33,22 +32,6 @@ class TestRemote(TestDvc): self.assertEqual(main(["remote", "list"]), 0) - def test_failed_write(self): - class A(object): - system = False - glob = False - local = False - name = "myremote" - url = "s3://remote" - unset = False - - args = A() - cmd = CmdRemoteAdd(args) - - cmd.configobj.write = None - ret = cmd.run() - self.assertNotEqual(ret, 0) - def test_relative_path(self): dname = os.path.join("..", "path", "to", "dir") ret = main(["remote", "add", "mylocal", dname]) @@ -64,7 +47,7 @@ class TestRemote(TestDvc): remote_name = "a" remote_url = "s3://bucket/name" self.assertEqual(main(["remote", "add", remote_name, remote_url]), 0) - self.assertEqual(main(["remote", "add", remote_name, remote_url]), 1) + self.assertEqual(main(["remote", "add", remote_name, remote_url]), 251) self.assertEqual( main(["remote", "add", "-f", remote_name, remote_url]), 0 ) From f69836c2b35c481fdd9741af77a5d594b85ff6c2 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 3 Jun 2019 15:51:58 +0300 Subject: [PATCH 25/30] tests: remove fixme msg https://github.com/iterative/dvc/issues/2086 Signed-off-by: Ruslan Kuprieiev --- tests/func/test_config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/func/test_config.py b/tests/func/test_config.py index 25d11cfb5..f6cae78d3 100644 --- a/tests/func/test_config.py +++ b/tests/func/test_config.py @@ -66,8 +66,6 @@ class TestConfigCLI(TestDvc): self._do_test(True) def test_non_existing(self): - # FIXME check stdout/err - ret = main(["config", "non_existing_section.field"]) self.assertEqual(ret, 251) From 8ef9133194c1932444a5f41a0cfd0b0d2bf8b34b Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 6 Jun 2019 16:00:44 +0300 Subject: [PATCH 26/30] remote: use RemoteConfig class Signed-off-by: Ruslan Kuprieiev --- dvc/cache.py | 14 +++ dvc/command/cache.py | 8 +- dvc/command/remote.py | 37 +++--- dvc/config.py | 2 +- dvc/remote/__init__.py | 5 +- dvc/remote/config.py | 254 ++++++++++++++++++++--------------------- 6 files changed, 167 insertions(+), 153 deletions(-) diff --git a/dvc/cache.py b/dvc/cache.py index 48432ea3b..c834ed0ae 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -7,6 +7,20 @@ import os from dvc.config import Config +class CacheConfig(object): + def __init__(self, config): + self.config = config + + def set_dir(self, dname, level=None): + from dvc.remote.config import RemoteConfig + + configobj = self.config.get_configobj(level) + path = RemoteConfig.resolve_path(dname, configobj.filename) + self.config.set( + Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level + ) + + class Cache(object): """Class that manages cache locations of a dvc repo. diff --git a/dvc/command/cache.py b/dvc/command/cache.py index c2ac03d20..b774aca8b 100644 --- a/dvc/command/cache.py +++ b/dvc/command/cache.py @@ -4,12 +4,16 @@ import argparse from dvc.command.base import fix_subparsers, append_doc_link from dvc.command.config import CmdConfig -from dvc.remote.config import set_cache_dir +from dvc.cache import CacheConfig class CmdCacheDir(CmdConfig): + def __init__(self, args): + super(CmdCacheDir, self).__init__(args) + self.config = CacheConfig(self.config) + def run(self): - set_cache_dir(self.config, self.args.value, level=self.args.level) + self.config.set_dir(self.args.value, level=self.args.level) return 0 diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 2fe5b1f92..df1251d0e 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -5,16 +5,20 @@ import logging from dvc.command.base import append_doc_link, fix_subparsers from dvc.command.config import CmdConfig -from dvc.remote.config import remote_add, remote_remove, remote_modify -from dvc.remote.config import remote_list, remote_default +from dvc.remote.config import RemoteConfig logger = logging.getLogger(__name__) -class CmdRemoteAdd(CmdConfig): +class CmdRemoteConfig(CmdConfig): + def __init__(self, args): + super(CmdRemoteConfig, self).__init__(args) + self.config = RemoteConfig(self.config) + + +class CmdRemoteAdd(CmdRemoteConfig): def run(self): - remote_add( - self.config, + self.config.add( self.args.name, self.args.url, force=self.args.force, @@ -24,16 +28,15 @@ class CmdRemoteAdd(CmdConfig): return 0 -class CmdRemoteRemove(CmdConfig): +class CmdRemoteRemove(CmdRemoteConfig): def run(self): - remote_remove(self.config, self.args.name, level=self.args.level) + self.config.remove(self.args.name, level=self.args.level) return 0 -class CmdRemoteModify(CmdConfig): +class CmdRemoteModify(CmdRemoteConfig): def run(self): - remote_modify( - self.config, + self.config.modify( self.args.name, self.args.option, self.args.value, @@ -42,21 +45,17 @@ class CmdRemoteModify(CmdConfig): return 0 -class CmdRemoteDefault(CmdConfig): +class CmdRemoteDefault(CmdRemoteConfig): def run(self): - remote_default( - self.config, - self.args.name, - unset=self.args.unset, - level=self.args.level, + self.config.set_default( + self.args.name, unset=self.args.unset, level=self.args.level ) return 0 -class CmdRemoteList(CmdConfig): +class CmdRemoteList(CmdRemoteConfig): def run(self): - remotes = remote_list(self.config, level=self.args.level) - for name, url in remotes.items(): + for name, url in self.config.list(level=self.args.level).items(): logger.info("{}\t{}".format(name, url)) return 0 diff --git a/dvc/config.py b/dvc/config.py index 555a8ca15..8222bf093 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -534,7 +534,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes config[section][opt] = value self.save(config) - def get(self, section, opt, level=None): + def get(self, section, opt=None, level=None): """Return option value from the config. Args: diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index f5ed528e0..e8ffe81f4 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -10,7 +10,7 @@ from dvc.remote.http import RemoteHTTP from dvc.remote.https import RemoteHTTPS from dvc.remote.oss import RemoteOSS -from .config import get_remote_settings +from .config import RemoteConfig REMOTES = [ @@ -36,7 +36,8 @@ def _get(config): def Remote(repo, **kwargs): name = kwargs.get("name") if name: - settings = get_remote_settings(repo.config, name) + remote_config = RemoteConfig(repo.config) + settings = remote_config.get_settings(name) else: settings = kwargs return _get(settings)(repo, settings) diff --git a/dvc/remote/config.py b/dvc/remote/config.py index 5b8672330..54b33ed2b 100644 --- a/dvc/remote/config.py +++ b/dvc/remote/config.py @@ -9,147 +9,143 @@ from dvc.config import Config, ConfigError logger = logging.getLogger(__name__) -def get_remote_settings(config, name): - """ - Args: - name (str): The name of the remote that we want to retrieve +class RemoteConfig(object): + def __init__(self, config): + self.config = config - Returns: - dict: The content beneath the given remote name. + def get_settings(self, name): + """ + Args: + name (str): The name of the remote that we want to retrieve - Example: - >>> config = {'remote "server"': {'url': 'ssh://localhost/'}} - >>> get_remote_settings("server") - {'url': 'ssh://localhost/'} - """ - settings = config.config.get( - config.SECTION_REMOTE_FMT.format(name.lower()) - ) + Returns: + dict: The content beneath the given remote name. - if settings is None: - raise ConfigError("unable to find remote section '{}'".format(name)) - - parsed = urlparse(settings["url"]) - - # Support for cross referenced remotes. - # This will merge the settings, giving priority to the outer reference. - # For example, having: - # - # dvc remote add server ssh://localhost - # dvc remote modify server user root - # dvc remote modify server ask_password true - # - # dvc remote add images remote://server/tmp/pictures - # dvc remote modify images user alice - # dvc remote modify images ask_password false - # dvc remote modify images password asdf1234 - # - # Results on a config dictionary like: - # - # { - # "url": "ssh://localhost/tmp/pictures", - # "user": "alice", - # "password": "asdf1234", - # "ask_password": False, - # } - # - if parsed.scheme == "remote": - reference = get_remote_settings(config, parsed.netloc) - url = posixpath.join(reference["url"], parsed.path.lstrip("/")) - merged = reference.copy() - merged.update(settings) - merged["url"] = url - return merged - - return settings - - -def _resolve_remote_path(path, config_file): - """Resolve path relative to config file location. - - Args: - path: Path to be resolved. - config_file: Path to config file, which `path` is specified - relative to. - - Returns: - Path relative to the `config_file` location. If `path` is an - absolute path then it will be returned without change. - - """ - if os.path.isabs(path): - return path - return os.path.relpath(path, os.path.dirname(config_file)) - - -def remote_add(config, name, url, default=False, force=False, level=None): - from dvc.remote import _get, RemoteLOCAL - - configobj = config.get_configobj(level) - remote = _get({Config.SECTION_REMOTE_URL: url}) - if remote == RemoteLOCAL and not url.startswith("remote://"): - url = _resolve_remote_path(url, configobj.filename) - - config.set( - config.SECTION_REMOTE_FMT.format(name), - config.SECTION_REMOTE_URL, - url, - force=force, - level=level, - ) - if default: - logger.info("Setting '{}' as a default remote.".format(name)) - config.set( - config.SECTION_CORE, config.SECTION_CORE_REMOTE, name, level=level + Example: + >>> config = {'remote "server"': {'url': 'ssh://localhost/'}} + >>> get_settings("server") + {'url': 'ssh://localhost/'} + """ + settings = self.config.config.get( + Config.SECTION_REMOTE_FMT.format(name.lower()) ) + if settings is None: + raise ConfigError( + "unable to find remote section '{}'".format(name) + ) -def remote_remove(config, name, level=None): - config.unset(config.SECTION_REMOTE_FMT.format(name), level=level) + parsed = urlparse(settings["url"]) - if level is None: - level = config.LEVEL_REPO + # Support for cross referenced remotes. + # This will merge the settings, giving priority to the outer reference. + # For example, having: + # + # dvc remote add server ssh://localhost + # dvc remote modify server user root + # dvc remote modify server ask_password true + # + # dvc remote add images remote://server/tmp/pictures + # dvc remote modify images user alice + # dvc remote modify images ask_password false + # dvc remote modify images password asdf1234 + # + # Results on a config dictionary like: + # + # { + # "url": "ssh://localhost/tmp/pictures", + # "user": "alice", + # "password": "asdf1234", + # "ask_password": False, + # } + # + if parsed.scheme == "remote": + reference = self.get_settings(parsed.netloc) + url = posixpath.join(reference["url"], parsed.path.lstrip("/")) + merged = reference.copy() + merged.update(settings) + merged["url"] = url + return merged - for lev in [ - config.LEVEL_LOCAL, - config.LEVEL_REPO, - config.LEVEL_GLOBAL, - config.LEVEL_SYSTEM, - ]: - config.unset( - config.SECTION_CORE, - config.SECTION_CORE_REMOTE, - level=lev, - force=True, + return settings + + @staticmethod + def resolve_path(path, config_file): + """Resolve path relative to config file location. + + Args: + path: Path to be resolved. + config_file: Path to config file, which `path` is specified + relative to. + + Returns: + Path relative to the `config_file` location. If `path` is an + absolute path then it will be returned without change. + + """ + if os.path.isabs(path): + return path + return os.path.relpath(path, os.path.dirname(config_file)) + + def add(self, name, url, default=False, force=False, level=None): + from dvc.remote import _get, RemoteLOCAL + + configobj = self.config.get_configobj(level) + remote = _get({Config.SECTION_REMOTE_URL: url}) + if remote == RemoteLOCAL and not url.startswith("remote://"): + url = self.resolve_path(url, configobj.filename) + + self.config.set( + Config.SECTION_REMOTE_FMT.format(name), + Config.SECTION_REMOTE_URL, + url, + force=force, + level=level, ) - if lev == level: - break + if default: + logger.info("Setting '{}' as a default remote.".format(name)) + self.config.set( + Config.SECTION_CORE, + Config.SECTION_CORE_REMOTE, + name, + level=level, + ) + def remove(self, name, level=None): + self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level) -def remote_modify(config, name, option, value, level=None): - config.set( - config.SECTION_REMOTE_FMT.format(name), option, value, level=level - ) + if level is None: + level = Config.LEVEL_REPO + for lev in [ + Config.LEVEL_LOCAL, + Config.LEVEL_REPO, + Config.LEVEL_GLOBAL, + Config.LEVEL_SYSTEM, + ]: + self.config.unset( + Config.SECTION_CORE, + Config.SECTION_CORE_REMOTE, + level=lev, + force=True, + ) + if lev == level: + break -def remote_list(config, level=None): - return config.list_options( - config.SECTION_REMOTE_REGEX, config.SECTION_REMOTE_URL, level=level - ) + def modify(self, name, option, value, level=None): + self.config.set( + Config.SECTION_REMOTE_FMT.format(name), option, value, level=level + ) + def list(self, level=None): + return self.config.list_options( + Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level + ) -def remote_default(config, name, unset=False, level=None): - if unset: - config.unset(config.SECTION_CORE, config.SECTION_CORE_REMOTE) - return - config.set( - config.SECTION_CORE, config.SECTION_CORE_REMOTE, name, level=level - ) - - -def set_cache_dir(config, dname, level=None): - configobj = config.get_configobj(level) - path = _resolve_remote_path(dname, configobj.filename) - config.set( - config.SECTION_CACHE, config.SECTION_CACHE_DIR, path, level=level - ) + def set_default(self, name, unset=False, level=None): + if unset: + self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE) + return + self.config.set( + Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level + ) From 3dffe8e5c0fc68e903932b2dee8f5508a0ede0b6 Mon Sep 17 00:00:00 2001 From: Jorge Orpinel Date: Fri, 7 Jun 2019 03:33:20 -0500 Subject: [PATCH 27/30] dvc: i/DVC file[s]?/ -> /DVC-file($1)/ Standardizes way to refer to DVC-file(s) (Related to #2102) --- dvc/command/add.py | 2 +- dvc/command/checkout.py | 2 +- dvc/command/commit.py | 4 ++-- dvc/command/data_sync.py | 2 +- dvc/command/destroy.py | 2 +- dvc/command/imp.py | 2 +- dvc/command/lock.py | 8 ++++---- dvc/command/move.py | 2 +- dvc/command/pipeline.py | 6 +++--- dvc/command/remove.py | 8 ++++---- dvc/command/repro.py | 4 ++-- dvc/command/run.py | 4 ++-- dvc/command/tag.py | 12 ++++++------ dvc/repo/__init__.py | 2 +- dvc/repo/checkout.py | 2 +- dvc/repo/reproduce.py | 2 +- dvc/repo/status.py | 2 +- dvc/stage.py | 8 ++++---- scripts/completion/dvc.zsh | 24 ++++++++++++------------ 19 files changed, 49 insertions(+), 49 deletions(-) diff --git a/dvc/command/add.py b/dvc/command/add.py index 43a1d2b51..45c74c06f 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -54,7 +54,7 @@ def add_parser(subparsers, parent_parser): help="Don't put files/directories into cache.", ) add_parser.add_argument( - "-f", "--file", help="Specify name of the DVC file it generates." + "-f", "--file", help="Specify name of the DVC-file it generates." ) add_parser.add_argument( "targets", nargs="+", help="Input files/directories." diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index 5a9aa05eb..99d376542 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -51,5 +51,5 @@ def add_parser(subparsers, parent_parser): default=False, help="Checkout all subdirectories of the specified directory.", ) - checkout_parser.add_argument("targets", nargs="*", help="DVC files.") + checkout_parser.add_argument("targets", nargs="*", help="DVC-files.") checkout_parser.set_defaults(func=CmdCheckout) diff --git a/dvc/command/commit.py b/dvc/command/commit.py index fd8f017e9..8ba3b99f3 100644 --- a/dvc/command/commit.py +++ b/dvc/command/commit.py @@ -34,7 +34,7 @@ class CmdCommit(CmdBase): def add_parser(subparsers, parent_parser): - COMMIT_HELP = "Save changed data to cache and update DVC files." + COMMIT_HELP = "Save changed data to cache and update DVC-files." commit_parser = subparsers.add_parser( "commit", @@ -65,6 +65,6 @@ def add_parser(subparsers, parent_parser): help="Commit cache for subdirectories of the specified directory.", ) commit_parser.add_argument( - "targets", nargs="*", default=None, help="DVC files." + "targets", nargs="*", default=None, help="DVC-files." ) commit_parser.set_defaults(func=CmdCommit) diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index f31c2aa87..e18ddacf4 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -113,7 +113,7 @@ def shared_parent_parser(): help="Show checksums instead of file names.", ) shared_parent_parser.add_argument( - "targets", nargs="*", default=None, help="DVC files." + "targets", nargs="*", default=None, help="DVC-files." ) return shared_parent_parser diff --git a/dvc/command/destroy.py b/dvc/command/destroy.py index 0ffacc127..1fad83cbf 100644 --- a/dvc/command/destroy.py +++ b/dvc/command/destroy.py @@ -35,7 +35,7 @@ class CmdDestroy(CmdBase): def add_parser(subparsers, parent_parser): - DESTROY_HELP = "Remove DVC files, local DVC config and data cache." + DESTROY_HELP = "Remove DVC-files, local DVC config and data cache." destroy_parser = subparsers.add_parser( "destroy", diff --git a/dvc/command/imp.py b/dvc/command/imp.py index b549bac4e..b284ebb28 100644 --- a/dvc/command/imp.py +++ b/dvc/command/imp.py @@ -65,6 +65,6 @@ def add_parser(subparsers, parent_parser): "out", nargs="?", help="Destination path to put files to." ) import_parser.add_argument( - "-f", "--file", help="Specify name of the DVC file it generates." + "-f", "--file", help="Specify name of the DVC-file it generates." ) import_parser.set_defaults(func=CmdImport) diff --git a/dvc/command/lock.py b/dvc/command/lock.py index e435e8dcc..e934380ed 100644 --- a/dvc/command/lock.py +++ b/dvc/command/lock.py @@ -37,7 +37,7 @@ class CmdUnlock(CmdLockBase): def add_parser(subparsers, parent_parser): - LOCK_HELP = "Lock DVC file." + LOCK_HELP = "Lock DVC-file." lock_parser = subparsers.add_parser( "lock", parents=[parent_parser], @@ -45,10 +45,10 @@ def add_parser(subparsers, parent_parser): help=LOCK_HELP, formatter_class=argparse.RawDescriptionHelpFormatter, ) - lock_parser.add_argument("targets", nargs="+", help="DVC files.") + lock_parser.add_argument("targets", nargs="+", help="DVC-files.") lock_parser.set_defaults(func=CmdLock) - UNLOCK_HELP = "Unlock DVC file." + UNLOCK_HELP = "Unlock DVC-file." unlock_parser = subparsers.add_parser( "unlock", parents=[parent_parser], @@ -56,5 +56,5 @@ def add_parser(subparsers, parent_parser): help=UNLOCK_HELP, formatter_class=argparse.RawDescriptionHelpFormatter, ) - unlock_parser.add_argument("targets", nargs="+", help="DVC files.") + unlock_parser.add_argument("targets", nargs="+", help="DVC-files.") unlock_parser.set_defaults(func=CmdUnlock) diff --git a/dvc/command/move.py b/dvc/command/move.py index 4fce351f0..1991e2dca 100644 --- a/dvc/command/move.py +++ b/dvc/command/move.py @@ -27,7 +27,7 @@ def add_parser(subparsers, parent_parser): MOVE_HELP = "Rename or move a DVC controlled data file or a directory." MOVE_DESCRIPTION = ( "Rename or move a DVC controlled data file or a directory.\n" - "It renames and modifies the corresponding DVC file to reflect the" + "It renames and modifies the corresponding DVC-file to reflect the" " changes." ) diff --git a/dvc/command/pipeline.py b/dvc/command/pipeline.py index 949015dfb..fe71ecb8a 100644 --- a/dvc/command/pipeline.py +++ b/dvc/command/pipeline.py @@ -209,14 +209,14 @@ def add_parser(subparsers, parent_parser): "--commands", action="store_true", default=False, - help="Print commands instead of paths to DVC files.", + help="Print commands instead of paths to DVC-files.", ) pipeline_show_group.add_argument( "-o", "--outs", action="store_true", default=False, - help="Print output files instead of paths to DVC files.", + help="Print output files instead of paths to DVC-files.", ) pipeline_show_parser.add_argument( "-l", @@ -244,7 +244,7 @@ def add_parser(subparsers, parent_parser): help="Output DAG as Dependencies Tree.", ) pipeline_show_parser.add_argument( - "targets", nargs="*", help="DVC files. 'Dvcfile' by default." + "targets", nargs="*", help="DVC-files. 'Dvcfile' by default." ) pipeline_show_parser.set_defaults(func=CmdPipelineShow) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index 6f2706674..4a006a575 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -43,7 +43,7 @@ class CmdRemove(CmdBase): def add_parser(subparsers, parent_parser): - REMOVE_HELP = "Remove outputs of DVC file." + REMOVE_HELP = "Remove outputs of DVC-file." remove_parser = subparsers.add_parser( "remove", parents=[parent_parser], @@ -57,14 +57,14 @@ def add_parser(subparsers, parent_parser): "--outs", action="store_true", default=True, - help="Only remove DVC file outputs. (Default)", + help="Only remove DVC-file outputs. (Default)", ) remove_parser_group.add_argument( "-p", "--purge", action="store_true", default=False, - help="Remove DVC file and all its outputs.", + help="Remove DVC-file and all its outputs.", ) remove_parser.add_argument( "-f", @@ -73,5 +73,5 @@ def add_parser(subparsers, parent_parser): default=False, help="Force purge.", ) - remove_parser.add_argument("targets", nargs="+", help="DVC files.") + remove_parser.add_argument("targets", nargs="+", help="DVC-files.") remove_parser.set_defaults(func=CmdRemove) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 271d6908d..3c7ee2f21 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -58,7 +58,7 @@ class CmdRepro(CmdBase): def add_parser(subparsers, parent_parser): - REPRO_HELP = "Check for changes and reproduce DVC file and dependencies." + REPRO_HELP = "Check for changes and reproduce DVC-file and dependencies." repro_parser = subparsers.add_parser( "repro", parents=[parent_parser], @@ -69,7 +69,7 @@ def add_parser(subparsers, parent_parser): repro_parser.add_argument( "targets", nargs="*", - help="DVC file to reproduce (default - 'Dvcfile').", + help="DVC-file to reproduce (default - 'Dvcfile').", ) repro_parser.add_argument( "-f", diff --git a/dvc/command/run.py b/dvc/command/run.py index e711744ee..6afbc5073 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -127,7 +127,7 @@ def add_parser(subparsers, parent_parser): "(do not put into DVC cache).", ) run_parser.add_argument( - "-f", "--file", help="Specify name of the DVC file it generates." + "-f", "--file", help="Specify name of the DVC-file it generates." ) run_parser.add_argument( "-c", "--cwd", default=None, help="Deprecated, use -w and -f instead." @@ -155,7 +155,7 @@ def add_parser(subparsers, parent_parser): "--overwrite-dvcfile", action="store_true", default=False, - help="Overwrite existing dvc file without asking for confirmation.", + help="Overwrite existing DVC-file without asking for confirmation.", ) run_parser.add_argument( "--ignore-build-cache", diff --git a/dvc/command/tag.py b/dvc/command/tag.py index 5701db2e5..679ccbb92 100644 --- a/dvc/command/tag.py +++ b/dvc/command/tag.py @@ -82,14 +82,14 @@ def add_parser(subparsers, parent_parser): ) tag_add_parser.add_argument("tag", help="Dvc tag.") tag_add_parser.add_argument( - "targets", nargs="*", default=[None], help="Dvc files." + "targets", nargs="*", default=[None], help="DVC-files." ) tag_add_parser.add_argument( "-d", "--with-deps", action="store_true", default=False, - help="Add tag for all dependencies of the specified DVC file.", + help="Add tag for all dependencies of the specified DVC-file.", ) tag_add_parser.add_argument( "-R", @@ -109,14 +109,14 @@ def add_parser(subparsers, parent_parser): ) tag_remove_parser.add_argument("tag", help="Dvc tag.") tag_remove_parser.add_argument( - "targets", nargs="*", default=[None], help="Dvc files." + "targets", nargs="*", default=[None], help="DVC-files." ) tag_remove_parser.add_argument( "-d", "--with-deps", action="store_true", default=False, - help="Remove tag for all dependencies of the specified DVC file.", + help="Remove tag for all dependencies of the specified DVC-file.", ) tag_remove_parser.add_argument( "-R", @@ -135,14 +135,14 @@ def add_parser(subparsers, parent_parser): help=TAG_LIST_HELP, ) tag_list_parser.add_argument( - "targets", nargs="*", default=[None], help="Dvc files." + "targets", nargs="*", default=[None], help="DVC-files." ) tag_list_parser.add_argument( "-d", "--with-deps", action="store_true", default=False, - help="List tags for all dependencies of the specified DVC file.", + help="List tags for all dependencies of the specified DVC-file.", ) tag_list_parser.add_argument( "-R", diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 101e9c300..884852947 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -318,7 +318,7 @@ class Repo(object): for stage in stages: if active and not target and stage.locked: logger.warning( - "DVC file '{path}' is locked. Its dependencies are" + "DVC-file '{path}' is locked. Its dependencies are" " not going to be pushed/pulled/fetched.".format( path=stage.relpath ) diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 8083feabc..2e1f3dd9f 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -47,7 +47,7 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False): for stage in stages: if stage.locked: logger.warning( - "DVC file '{path}' is locked. Its dependencies are" + "DVC-file '{path}' is locked. Its dependencies are" " not going to be checked out.".format(path=stage.relpath) ) diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py index 4a7b51ce7..daa2e40da 100644 --- a/dvc/repo/reproduce.py +++ b/dvc/repo/reproduce.py @@ -15,7 +15,7 @@ def _reproduce_stage(stages, node, force, dry, interactive, no_commit): if stage.locked: logger.warning( - "DVC file '{path}' is locked. Its dependencies are" + "DVC-file '{path}' is locked. Its dependencies are" " not going to be reproduced.".format(path=stage.relpath) ) diff --git a/dvc/repo/status.py b/dvc/repo/status.py index e9425d4d8..a3a2c7e0f 100644 --- a/dvc/repo/status.py +++ b/dvc/repo/status.py @@ -14,7 +14,7 @@ def _local_status(self, target=None, with_deps=False): for stage in stages: if stage.locked: logger.warning( - "DVC file '{path}' is locked. Its dependencies are" + "DVC-file '{path}' is locked. Its dependencies are" " not going to be shown in the status output.".format( path=stage.relpath ) diff --git a/dvc/stage.py b/dvc/stage.py index a5b018230..3fece0483 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -54,7 +54,7 @@ class StageFileAlreadyExistsError(DvcException): class StageFileIsNotDvcFileError(DvcException): def __init__(self, fname): - msg = "'{}' is not a dvc file".format(fname) + msg = "'{}' is not a DVC-file".format(fname) sname = fname + Stage.STAGE_FILE_SUFFIX if Stage.is_stage_file(sname): @@ -218,7 +218,7 @@ class Stage(object): if self.is_callback: logger.warning( - "Dvc file '{fname}' is a 'callback' stage " + "DVC-file '{fname}' is a 'callback' stage " "(has a command and no dependencies) and thus always " "considered as changed.".format(fname=self.relpath) ) @@ -253,7 +253,7 @@ class Stage(object): def _changed_md5(self): if self.changed_md5(): - logger.warning("Dvc file '{}' changed.".format(self.relpath)) + logger.warning("DVC-file '{}' changed.".format(self.relpath)) return True return False @@ -583,7 +583,7 @@ class Stage(object): # it raises the proper exceptions by priority: # 1. when the file doesn't exists - # 2. filename is not a dvc filename + # 2. filename is not a DVC-file # 3. path doesn't represent a regular file Stage._check_file_exists(repo, fname) Stage._check_dvc_filename(fname) diff --git a/scripts/completion/dvc.zsh b/scripts/completion/dvc.zsh index a1a0849d5..0faddee63 100644 --- a/scripts/completion/dvc.zsh +++ b/scripts/completion/dvc.zsh @@ -15,28 +15,28 @@ _dvc_commands() { "add:Take data files or directories under DVC control." "cache:Manage cache settings." "checkout:Checkout data files from cache." - "commit:Save changed data to cache and update DVC files." + "commit:Save changed data to cache and update DVC-files." "config:Get or set config settings." - "destroy:Remove DVC files, local DVC config and data cache." + "destroy:Remove DVC-files, local DVC config and data cache." "diff:Show a diff of a DVC controlled data file or a directory." "fetch:Fetch data files from a DVC remote storage." "gc:Collect unused data from DVC cache or a remote storage." "import:Download or copy files from URL and take under DVC control." "init:Initialize DVC in the current directory." "install:Install DVC git hooks into the repository." - "lock:Lock DVC file." + "lock:Lock DVC-file." "metrics:Commands to add, manage, collect and display metrics." "move:Rename or move a DVC controlled data file or a directory." "pipeline:Manage pipelines." "pull:Pull data files from a DVC remote storage." "push:Push data files to a DVC remote storage." "remote:Manage remote storage configuration." - "remove:Remove outputs of DVC file." - "repro:Check for changes and reproduce DVC file and dependencies." + "remove:Remove outputs of DVC-file." + "repro:Check for changes and reproduce DVC-file and dependencies." "root:Relative path to project's directory." "run:Generate a stage file from a command and execute the command." "status:Show changed stages, compare local cache and a remote storage." - "unlock:Unlock DVC file." + "unlock:Unlock DVC-file." "unprotect:Unprotect data file/directory." "version:Show DVC version and system/environment informaion." ) @@ -58,7 +58,7 @@ _dvc_global_options=( _dvc_add=( {-R,--recursive}"[Recursively add each file under the directory.]" "--no-commit[Don't put files/directories into cache.]" - {-f,--file}"[Specify name of the DVC file it generates.]:File:_files" + {-f,--file}"[Specify name of the DVC-file it generates.]:File:_files" "1:File:_files" ) @@ -118,7 +118,7 @@ _dvc_gc=( _dvc_import=( "--resume[Resume previously started download.]" - {-f,--file}"[Specify name of the DVC file it generates.]:File:_files" + {-f,--file}"[Specify name of the DVC-file it generates.]:File:_files" "1:URL:" "2:Output:" ) @@ -177,8 +177,8 @@ _dvc_remote=( _dvc_remove=( "*:Stages:_files -g '(*.dvc|Dvcfile)'" "--dry[Only print the commands that would be executed without actually executing]" - {-o,--outs}"[Only remove DVC file outputs. (Default)]" - {-p,--purge}"[Remove DVC file and all its outputs.]" + {-o,--outs}"[Only remove DVC-file outputs. (Default)]" + {-p,--purge}"[Remove DVC-file and all its outputs.]" {-f,--force}"[Force purge.]" ) @@ -206,12 +206,12 @@ _dvc_run=( "*"{-O,--outs-no-cache}"[Declare output file or directory (do not put into DVC cache).]:Output regular:_files" "*"{-m,--metrics}"[Declare output metric file or directory.]:Metrics:_files" "*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache).]:Metrics (no cache):_files" - {-f,--file}"[Specify name of the DVC file it generates.]:File:_files" + {-f,--file}"[Specify name of the DVC-file it generates.]:File:_files" {-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/" {-w,--wdir}"[Directory within your repo to run your command in.]:WDIR:_files -/" "--no-exec[Only create stage file without actually running it.]" {-y,--yes}"[Deprecated, use --overwrite-dvcfile instead]" - "--overwrite-dvcfile[Overwrite existing dvc file without asking for confirmation.]" + "--overwrite-dvcfile[Overwrite existing DVC-file without asking for confirmation.]" "--ignore-build-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]" "--remove-outs[Deprecated, this is now the default behavior]" "--no-commit[Don't put files/directories into cache.]" From fd3c84d682aba203f068e5a91062090a84923a6b Mon Sep 17 00:00:00 2001 From: dlg227 Date: Fri, 17 May 2019 16:40:53 -0400 Subject: [PATCH 28/30] remote: add basic support for s3 server side encryption. Fixes #1967 --- dvc/config.py | 3 +++ dvc/remote/s3.py | 28 +++++++++++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/dvc/config.py b/dvc/config.py index 8222bf093..1fe903fd0 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -208,6 +208,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes SECTION_AWS_REGION = "region" SECTION_AWS_PROFILE = "profile" SECTION_AWS_USE_SSL = "use_ssl" + SECTION_AWS_SSE = "sse" SECTION_AWS_SCHEMA = { SECTION_AWS_STORAGEPATH: str, Optional(SECTION_AWS_REGION): str, @@ -216,6 +217,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional(SECTION_AWS_ENDPOINT_URL): str, Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA, Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA, + Optional(SECTION_AWS_SSE): str, } # backward compatibility @@ -256,6 +258,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional(SECTION_AWS_ENDPOINT_URL): str, Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA, Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA, + Optional(SECTION_AWS_SSE): str, Optional(SECTION_GCP_PROJECTNAME): str, Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA, Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA, diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 99ff523d2..d1fcd1b4a 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -64,6 +64,12 @@ class RemoteS3(RemoteBASE): self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True) + self.extra_args = {} + + self.sse = config.get(Config.SECTION_AWS_SSE, "") + if self.sse: + self.extra_args["ServerSideEncryption"] = self.sse + shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH) if shared_creds: os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds) @@ -112,9 +118,11 @@ class RemoteS3(RemoteBASE): return obj @classmethod - def _copy_multipart(cls, s3, from_info, to_info, size, n_parts): + def _copy_multipart( + cls, s3, from_info, to_info, size, n_parts, extra_args={} + ): mpu = s3.create_multipart_upload( - Bucket=to_info.bucket, Key=to_info.path + Bucket=to_info.bucket, Key=to_info.path, **extra_args ) mpu_id = mpu["UploadId"] @@ -154,7 +162,7 @@ class RemoteS3(RemoteBASE): ) @classmethod - def _copy(cls, s3, from_info, to_info): + def _copy(cls, s3, from_info, to_info, extra_args={}): # NOTE: object's etag depends on the way it was uploaded to s3 or the # way it was copied within the s3. More specifically, it depends on # the chunk size that was used to transfer it, which would affect @@ -182,10 +190,12 @@ class RemoteS3(RemoteBASE): _, _, parts_suffix = etag.partition("-") if parts_suffix: n_parts = int(parts_suffix) - cls._copy_multipart(s3, from_info, to_info, size, n_parts) + cls._copy_multipart( + s3, from_info, to_info, size, n_parts, extra_args + ) else: source = {"Bucket": from_info.bucket, "Key": from_info.path} - s3.copy(source, to_info.bucket, to_info.path) + s3.copy(source, to_info.bucket, to_info.path, ExtraArgs=extra_args) cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path) if etag != cached_etag: @@ -193,7 +203,7 @@ class RemoteS3(RemoteBASE): def copy(self, from_info, to_info, s3=None): s3 = s3 if s3 else self.s3 - self._copy(s3, from_info, to_info) + self._copy(s3, from_info, to_info, extra_args=self.extra_args) def remove(self, path_info): if path_info.scheme != "s3": @@ -257,7 +267,11 @@ class RemoteS3(RemoteBASE): try: s3.upload_file( - from_info.path, to_info.bucket, to_info.path, Callback=cb + from_info.path, + to_info.bucket, + to_info.path, + Callback=cb, + ExtraArgs=self.extra_args, ) except Exception: msg = "failed to upload '{}'".format(from_info.path) From fa6b764f97bd7f14fd10eefa8105128bc371b0ed Mon Sep 17 00:00:00 2001 From: dlg227 Date: Sat, 8 Jun 2019 17:10:58 -0400 Subject: [PATCH 29/30] refactor of remote s3 extra_args --- dvc/remote/s3.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index d1fcd1b4a..900f8b7bb 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -117,19 +117,16 @@ class RemoteS3(RemoteBASE): ) return obj - @classmethod - def _copy_multipart( - cls, s3, from_info, to_info, size, n_parts, extra_args={} - ): + def _copy_multipart(self, s3, from_info, to_info, size, n_parts): mpu = s3.create_multipart_upload( - Bucket=to_info.bucket, Key=to_info.path, **extra_args + Bucket=to_info.bucket, Key=to_info.path, **self.extra_args ) mpu_id = mpu["UploadId"] parts = [] byte_position = 0 for i in range(1, n_parts + 1): - obj = cls.get_head_object( + obj = self.get_head_object( s3, from_info.bucket, from_info.path, PartNumber=i ) part_size = obj["ContentLength"] @@ -161,8 +158,7 @@ class RemoteS3(RemoteBASE): MultipartUpload={"Parts": parts}, ) - @classmethod - def _copy(cls, s3, from_info, to_info, extra_args={}): + def _copy(self, s3, from_info, to_info): # NOTE: object's etag depends on the way it was uploaded to s3 or the # way it was copied within the s3. More specifically, it depends on # the chunk size that was used to transfer it, which would affect @@ -183,27 +179,27 @@ class RemoteS3(RemoteBASE): # preserve etag, we need to transfer each part separately, so the # object is transfered in the same chunks as it was originally. - obj = cls.get_head_object(s3, from_info.bucket, from_info.path) + obj = self.get_head_object(s3, from_info.bucket, from_info.path) etag = obj["ETag"].strip('"') size = obj["ContentLength"] _, _, parts_suffix = etag.partition("-") if parts_suffix: n_parts = int(parts_suffix) - cls._copy_multipart( - s3, from_info, to_info, size, n_parts, extra_args - ) + self._copy_multipart(s3, from_info, to_info, size, n_parts) else: source = {"Bucket": from_info.bucket, "Key": from_info.path} - s3.copy(source, to_info.bucket, to_info.path, ExtraArgs=extra_args) + s3.copy( + source, to_info.bucket, to_info.path, ExtraArgs=self.extra_args + ) - cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path) + cached_etag = self.get_etag(s3, to_info.bucket, to_info.path) if etag != cached_etag: raise ETagMismatchError(etag, cached_etag) def copy(self, from_info, to_info, s3=None): s3 = s3 if s3 else self.s3 - self._copy(s3, from_info, to_info, extra_args=self.extra_args) + self._copy(s3, from_info, to_info) def remove(self, path_info): if path_info.scheme != "s3": From 00a2e9bb7fd0b384bb1bd4c38193082400f7b66d Mon Sep 17 00:00:00 2001 From: Andrew Grigorev Date: Mon, 10 Jun 2019 13:11:32 +0300 Subject: [PATCH 30/30] config: choices refactoring --- dvc/config.py | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/dvc/config.py b/dvc/config.py index 1fe903fd0..b6f68759f 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -45,24 +45,6 @@ def supported_cache_type(types): return True -def supported_loglevel(level): - """Checks if log level config option has a valid value. - - Args: - level (str): log level name. - """ - return level in ["info", "debug", "warning", "error"] - - -def supported_cloud(cloud): - """Checks if obsoleted cloud option has a valid value. - - Args: - cloud (str): cloud type name. - """ - return cloud in ["aws", "gcp", "local", ""] - - def is_bool(val): """Checks that value is a boolean. @@ -111,6 +93,23 @@ def is_percent(val): return int(val) >= 0 and int(val) <= 100 +class Choices(object): + """Checks that value belongs to the specified set of values + + Args: + *choices: pass allowed values as arguments, or pass a list or + tuple as a single argument + """ + + def __init__(self, *choices): + if len(choices) == 1 and isinstance(choices[0], (list, tuple)): + choices = choices[0] + self.choices = choices + + def __call__(self, value): + return value in self.choices + + class Config(object): # pylint: disable=too-many-instance-attributes """Class that manages configuration files for a dvc repo. @@ -145,7 +144,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes SECTION_CORE = "core" SECTION_CORE_LOGLEVEL = "loglevel" - SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel) + SECTION_CORE_LOGLEVEL_SCHEMA = And( + Use(str.lower), Choices("info", "debug", "warning", "error") + ) SECTION_CORE_REMOTE = "remote" SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA SECTION_CORE_INTERACTIVE = "interactive" @@ -180,7 +181,11 @@ class Config(object): # pylint: disable=too-many-instance-attributes # backward compatibility SECTION_CORE_CLOUD = "cloud" - SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud) + SECTION_CORE_CLOUD_SCHEMA = And( + Use(str.lower), + # only the clouds which were supported by obsoleted cloud config + Choices("aws", "gcp", "local", ""), + ) SECTION_CORE_STORAGEPATH = "storagepath" SECTION_CORE_SCHEMA = {