Compare commits

..
3 Commits
Author SHA1 Message Date
Ruslan Kuprieiev 31efaac4f3 dvc: bump to 1.2.3 2020-07-29 01:25:45 +03:00
Ruslan Kuprieiev 5d1c1c418c gdrive: add workaround for downloading empty files (#4287)
* [WIP] gdrive: add workaround for downloading empty files

Better to solve it on PyDrive's side:
https://github.com/iterative/dvc/issues/4286

Fixes #4286

* deps: limit flatten_json

```
pylint...................................................................Failed
- hook id: pylint
- exit code: 2

Registered custom plugin. Some checks will be disabled for tests.
************* Module dvc.command.experiments
dvc/command/experiments.py:14:4: E0401: Unable to import 'flatten_json'
(import-error)
dvc/command/experiments.py:39:4: E0401: Unable to import 'flatten_json'
(import-error)

-----------------------------------
Your code has been rated at 9.99/10

Registered custom plugin. Some checks will be disabled for tests.
************* Module dvc.utils.diff
dvc/utils/diff.py:38:8: E0401: Unable to import 'flatten_json'
(import-error)
************* Module dvc.command.metrics
dvc/command/metrics.py:17:4: E0401: Unable to import 'flatten_json'
(import-error)

-----------------------------------
```
2020-07-29 01:25:06 +03:00
f4595a02b2 version: redesign output format for the dvc version command #3499 (#4114)
* Version: Redesign output format #3499

Redesigned the output format for the command: dvc version

* version: reformat output format #3499

Done some suggested fixes in code for the dvc command output format issue.

* version: redesign output format #3499

* version: redesign output format for dvc command #3499

* version: redesign output format #3499

* version: redesign output format #3499

* Redesign output format for the dvc version command #3499

* Redesign output format for the dvc version command #3499

* Update dvc/command/version.py

Co-authored-by: Saugat Pachhai <[email protected]>

Co-authored-by: Ruslan Kuprieiev <[email protected]>
Co-authored-by: Saugat Pachhai <[email protected]>
2020-07-28 18:22:26 +03:00
4 changed files with 51 additions and 40 deletions
+30 -17
View File
@@ -10,7 +10,7 @@ from dvc.command.base import CmdBaseNoRepo, append_doc_link
from dvc.exceptions import DvcException, NotDvcRepoError
from dvc.scm.base import SCMError
from dvc.system import System
from dvc.utils import is_binary, relpath
from dvc.utils import relpath
from dvc.utils.pkg import PKG
from dvc.version import __version__
@@ -27,13 +27,18 @@ class CmdVersion(CmdBaseNoRepo):
def run(self):
from dvc.repo import Repo
package = PKG
if PKG is None:
package = ""
else:
package = f"({PKG})"
info = [
f"DVC version: {__version__}",
f"Python version: {platform.python_version()}",
f"Platform: {platform.platform()}",
f"Binary: {is_binary()}",
f"Package: {PKG}",
f"Supported remotes: {self.get_supported_remotes()}",
f"DVC version: {__version__} {package}",
"---------------------------------",
f"Platform: Python {platform.python_version()} on "
f"{platform.platform()}",
f"Supports: {self.get_supported_remotes()}",
]
try:
@@ -46,13 +51,13 @@ class CmdVersion(CmdBaseNoRepo):
# `dvc config cache.shared group`.
if os.path.exists(repo.cache.local.cache_dir):
info.append(
"Cache: {}".format(self.get_linktype_support_info(repo))
"Cache types: {}".format(
self.get_linktype_support_info(repo)
)
)
if psutil:
fs_type = self.get_fs_type(repo.cache.local.cache_dir)
info.append(
f"Filesystem type (cache directory): {fs_type}"
)
info.append(f"Cache directory: {fs_type}")
else:
logger.warning(
"Unable to detect supported link types, as cache "
@@ -68,11 +73,11 @@ class CmdVersion(CmdBaseNoRepo):
root_directory = os.getcwd()
info.append("Repo: dvc, git (broken)")
else:
info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
if psutil:
fs_root = self.get_fs_type(os.path.abspath(root_directory))
info.append(f"Workspace directory: {fs_root}")
if psutil:
fs_root = self.get_fs_type(os.path.abspath(root_directory))
info.append(f"Filesystem type (workspace): {fs_root}")
info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
logger.info("\n".join(info))
return 0
@@ -80,7 +85,7 @@ class CmdVersion(CmdBaseNoRepo):
@staticmethod
def get_fs_type(path):
partition = {
pathlib.Path(part.mountpoint): (part.fstype, part.device)
pathlib.Path(part.mountpoint): (part.fstype + " on " + part.device)
for part in psutil.disk_partitions(all=True)
}
@@ -115,7 +120,9 @@ class CmdVersion(CmdBaseNoRepo):
os.unlink(dst)
except DvcException:
status = "not supported"
cache.append(f"{name} - {status}")
if status == "supported":
cache.append(name)
os.remove(src)
return ", ".join(cache)
@@ -129,6 +136,12 @@ class CmdVersion(CmdBaseNoRepo):
if not tree_cls.get_missing_deps():
supported_remotes.append(tree_cls.scheme)
if len(supported_remotes) == len(TREES):
return "All remotes"
if len(supported_remotes) == 1:
return supported_remotes
return ", ".join(supported_remotes)
+10 -1
View File
@@ -381,6 +381,9 @@ class GDriveTree(BaseTree):
# it does not create a file on the remote
gdrive_file = self._drive.CreateFile(param)
gdrive_file.FetchMetadata(fields="fileSize")
size = gdrive_file["fileSize"]
with Tqdm(
desc=progress_desc,
disable=no_progress_bar,
@@ -388,7 +391,13 @@ class GDriveTree(BaseTree):
# explicit `bar_format` as `total` will be set by `update_to`
bar_format=Tqdm.BAR_FMT_DEFAULT,
) as pbar:
gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
if size:
gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
else:
# PyDrive doesn't like downloading empty files
# https://github.com/iterative/dvc/issues/4286
with open(to_file, "w"):
pass
@contextmanager
@_gdrive_retry
+1 -1
View File
@@ -6,7 +6,7 @@
import os
import subprocess
_BASE_VERSION = "1.2.2"
_BASE_VERSION = "1.2.3"
def _generate_version(base_version):
+10 -21
View File
@@ -16,15 +16,10 @@ def test_info_in_repo(scm_init, tmp_dir, caplog):
assert main(["version"]) == 0
assert re.search(r"DVC version: \d+\.\d+\.\d+", caplog.text)
assert re.search(r"Python version: \d\.\d\.\d", caplog.text)
assert re.search(r"Platform: .*", caplog.text)
assert re.search(r"Binary: (True|False)", caplog.text)
assert re.search(r"Package: .*", caplog.text)
assert re.search(r"Supported remotes: .*", caplog.text)
assert re.search(
r"(Cache: (.*link - (not )?supported(,\s)?){3})", caplog.text
)
assert re.search(r"DVC version: \d+\.\d+\.\d+\+.*", caplog.text)
assert re.search(r"Platform: Python \d\.\d\.\d on .*", caplog.text)
assert re.search(r"Supports: .*", caplog.text)
assert re.search(r"Cache types: .*", caplog.text)
if scm_init:
assert "Repo: dvc, git" in caplog.text
@@ -58,26 +53,20 @@ def test_fs_info_in_repo(tmp_dir, dvc, caplog):
os.mkdir(dvc.cache.local.cache_dir)
assert main(["version"]) == 0
assert "Filesystem type (cache directory): " in caplog.text
assert "Filesystem type (workspace): " in caplog.text
assert re.search(r"Cache directory: .* on .*", caplog.text)
assert re.search(r"Workspace directory: .* on .*", caplog.text)
def test_info_outside_of_repo(tmp_dir, caplog):
assert main(["version"]) == 0
assert re.search(r"DVC version: \d+\.\d+\.\d+", caplog.text)
assert re.search(r"Python version: \d\.\d\.\d", caplog.text)
assert re.search(r"Platform: .*", caplog.text)
assert re.search(r"Binary: (True|False)", caplog.text)
assert re.search(r"Package: .*", caplog.text)
assert re.search(r"Supported remotes: .*", caplog.text)
assert not re.search(r"(Cache: (.*link - (not )?(,\s)?){3})", caplog.text)
assert re.search(r"DVC version: \d+\.\d+\.\d+\+.*", caplog.text)
assert re.search(r"Platform: Python \d\.\d\.\d on .*", caplog.text)
assert re.search(r"Supports: .*", caplog.text)
assert not re.search(r"Cache types: .*", caplog.text)
assert "Repo:" not in caplog.text
@pytest.mark.skipif(psutil is None, reason="No psutil.")
def test_fs_info_outside_of_repo(tmp_dir, caplog):
assert main(["version"]) == 0
assert "Filesystem type (cache directory): " not in caplog.text
assert "Filesystem type (workspace): " in caplog.text