From b4d0d37ac8b99695e9ed25be6f65ea106024e087 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Thu, 4 Apr 2019 21:48:33 -0600 Subject: [PATCH] dvc: refactor package versioning logic - Use `version.py` to compute a dynamic version stored in `__version__`. Version information was scattered across the source code (`VERSION_BASE`, `VERSION`, `__version__`, and `version.py`). - Read `__version__` from `setup.py` and **pin** it using custom `build_py` command class. This way, we can maintain consistancy between the build version and the runtime one. Also, this gives us the ability to run `setup.py` without depending on on `dvc` itself. Close #1824 --- .gitignore | 2 - dvc/__init__.py | 68 +-------------------------------- dvc/analytics.py | 4 +- dvc/cli.py | 5 ++- dvc/state.py | 6 ++- dvc/updater.py | 16 ++------ dvc/version.py | 52 +++++++++++++++++++++++++ scripts/build_posix.sh | 2 +- scripts/innosetup/config_gen.py | 4 +- setup.py | 38 +++++++++++++++++- tests/unit/test_updater.py | 6 +-- 11 files changed, 108 insertions(+), 95 deletions(-) create mode 100644 dvc/version.py diff --git a/.gitignore b/.gitignore index fe7916dbb..8f4a190e2 100644 --- a/.gitignore +++ b/.gitignore @@ -24,8 +24,6 @@ innosetup/config.ini .coverage -dvc/version.py - *.swp pip-wheel-metadata/ diff --git a/dvc/__init__.py b/dvc/__init__.py index 661c793b9..8c1eba236 100644 --- a/dvc/__init__.py +++ b/dvc/__init__.py @@ -6,75 +6,9 @@ Make your data science projects reproducible and shareable. from __future__ import unicode_literals -import os +from dvc.version import __version__ # noqa: F401 import warnings - -VERSION_BASE = "0.34.2" -__version__ = VERSION_BASE - -PACKAGEPATH = os.path.abspath(os.path.dirname(__file__)) -HOMEPATH = os.path.dirname(PACKAGEPATH) -VERSIONPATH = os.path.join(PACKAGEPATH, "version.py") - - -def _update_version_file(): - """Dynamically update version file.""" - try: - from git import Repo - from git.exc import InvalidGitRepositoryError - except ImportError: - return __version__ - - try: - repo = Repo(HOMEPATH) - except InvalidGitRepositoryError: - return __version__ - - sha = repo.head.object.hexsha - short_sha = repo.git.rev_parse(sha, short=6) - dirty = ".mod" if repo.is_dirty() else "" - ver = "{}+{}{}".format(__version__, short_sha, dirty) - - # Write a helper file, that will be installed with the package - # and will provide a true version of the installed dvc - with open(VERSIONPATH, "w+") as fobj: - fobj.write("# AUTOGENERATED by dvc/__init__.py\n") - fobj.write('version = "{}"\n'.format(ver)) - - return ver - - -def _remove_version_file(): - """Remove version.py so that it doesn't get into the release.""" - if os.path.exists(VERSIONPATH): - os.unlink(VERSIONPATH) - - -if os.path.exists(os.path.join(HOMEPATH, "setup.py")): - # dvc is run directly from source without installation or - # __version__ is called from setup.py - if ( - os.getenv("APPVEYOR_REPO_TAG", "").lower() != "true" - and os.getenv("TRAVIS_TAG", "") == "" - and os.getenv("DVC_TEST", "").lower() != "true" - ): - __version__ = _update_version_file() - else: # pragma: no cover - _remove_version_file() -else: # pragma: no cover - # dvc was installed with pip or something. Hopefully we have our - # auto-generated version.py to help us provide a true version - try: - from dvc.version import version - - __version__ = version - except Exception: - pass - -VERSION = __version__ - - # Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432. # We don't directly import numpy, but our dependency networkx does, causing # these warnings in some environments. Luckily these warnings are benign and diff --git a/dvc/analytics.py b/dvc/analytics.py index 7565280e7..0b7a4e94f 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -9,7 +9,7 @@ import json import errno import dvc.logger as logger -from dvc import VERSION +from dvc import __version__ class Analytics(object): @@ -164,7 +164,7 @@ class Analytics(object): from dvc.repo import Repo from dvc.exceptions import NotDvcRepoError - self.info[self.PARAM_DVC_VERSION] = VERSION + self.info[self.PARAM_DVC_VERSION] = __version__ self.info[self.PARAM_IS_BINARY] = is_binary() self.info[self.PARAM_USER_ID] = self._get_user_id() diff --git a/dvc/cli.py b/dvc/cli.py index d8af5717a..9393e957f 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -33,7 +33,6 @@ import dvc.command.daemon as daemon import dvc.command.commit as commit import dvc.command.tag as tag from dvc.exceptions import DvcParserError -from dvc import VERSION COMMANDS = [ @@ -86,7 +85,9 @@ class VersionAction(argparse.Action): # pragma: no cover """Shows dvc version and exits.""" def __call__(self, parser, namespace, values, option_string=None): - print(VERSION) + from dvc import __version__ + + print(__version__) sys.exit(0) diff --git a/dvc/state.py b/dvc/state.py index 9b20f2fa4..d7aee402d 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -164,7 +164,7 @@ class State(object): # pylint: disable=too-many-instance-attributes return num def _prepare_db(self, empty=False): - from dvc import VERSION + from dvc import __version__ if not empty: cmd = "PRAGMA user_version;" @@ -176,7 +176,9 @@ class State(object): # pylint: disable=too-many-instance-attributes version = ret[0][0] if version > self.VERSION: - raise StateVersionTooNewError(VERSION, self.VERSION, version) + raise StateVersionTooNewError( + __version__, self.VERSION, version + ) elif version < self.VERSION: msg = ( "State file version '{}' is too old. " diff --git a/dvc/updater.py b/dvc/updater.py index 09b2032bf..b7bd00760 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -5,9 +5,10 @@ import os import time import requests import colorama +from pkg_resources import parse_version +from dvc import __version__ import dvc.logger as logger -from dvc import VERSION_BASE from dvc.lock import Lock, LockError from dvc.utils import is_binary @@ -22,7 +23,7 @@ class Updater(object): # pragma: no cover self.dvc_dir = dvc_dir self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE) self.lock = Lock(dvc_dir, self.updater_file + ".lock") - self.current = VERSION_BASE + self.current = parse_version(__version__).base_version def _is_outdated_file(self): ctime = os.path.getmtime(self.updater_file) @@ -89,16 +90,7 @@ class Updater(object): # pragma: no cover json.dump(info, fobj) def _is_outdated(self): - l_major, l_minor, l_patch = [int(x) for x in self.latest.split(".")] - c_major, c_minor, c_patch = [int(x) for x in self.current.split(".")] - - if l_major != c_major: - return l_major > c_major - - if l_minor != c_minor: - return l_minor > c_minor - - return l_patch > c_patch + return parse_version(self.current) < parse_version(self.latest) def _notify(self): if not sys.stdout.isatty(): diff --git a/dvc/version.py b/dvc/version.py new file mode 100644 index 000000000..f075e4f8e --- /dev/null +++ b/dvc/version.py @@ -0,0 +1,52 @@ +# Used from setup.py, so don't pull any additional dependencies +import os +import subprocess + + +def generate_version(base_version): + """Generate a version with information about the git repository""" + pkg_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + + if not is_git_repo(pkg_dir) or not have_git(pkg_dir): + return base_version + + return "{base_version}+{short_sha}{dirty}".format( + base_version=base_version, + short_sha=git_revision(pkg_dir).decode("utf-8")[0:6], + dirty=".mod" if is_dirty(pkg_dir) else "", + ) + + +def is_git_repo(dir_path): + """Is the given directory version-controlled with git?""" + return os.path.exists(os.path.join(dir_path, ".git")) + + +def have_git(dir_path): + """Can we run the git executable?""" + try: + subprocess.check_output(["git", "--help"]) + return True + except subprocess.CalledProcessError: + return False + except OSError: + return False + + +def git_revision(dir_path): + """Get the SHA-1 of the HEAD of a git repository.""" + return subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=dir_path + ).strip() + + +def is_dirty(dir_path): + """Check whether a git repository has uncommitted changes.""" + try: + subprocess.check_call(["git", "diff", "--quiet"], cwd=dir_path) + return False + except subprocess.CalledProcessError: + return True + + +__version__ = generate_version(base_version="0.34.2") diff --git a/scripts/build_posix.sh b/scripts/build_posix.sh index a35b9f1e8..233a1fede 100755 --- a/scripts/build_posix.sh +++ b/scripts/build_posix.sh @@ -42,7 +42,7 @@ command_exists() fpm_build() { print_info "Building $1..." - VERSION=$(python -c "import dvc; from dvc import VERSION; print(str(VERSION))") + VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))") fpm -s dir \ -f \ -t $1 \ diff --git a/scripts/innosetup/config_gen.py b/scripts/innosetup/config_gen.py index 6f626df74..35e643684 100644 --- a/scripts/innosetup/config_gen.py +++ b/scripts/innosetup/config_gen.py @@ -1,10 +1,10 @@ # This script generates config.ini for setup.iss script -from dvc import VERSION +from dvc import __version__ from dvc.utils.compat import ConfigParser config = ConfigParser.ConfigParser() config.add_section("Version") -config.set("Version", "Version", VERSION) +config.set("Version", "Version", __version__) with open("scripts/innosetup/config.ini", "w") as f: config.write(f) diff --git a/setup.py b/setup.py index 07fe236ba..f2b823ad0 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,38 @@ from setuptools import setup, find_packages -from dvc import VERSION +from setuptools.command.build_py import build_py as _build_py +import os + + +# https://packaging.python.org/guides/single-sourcing-package-version/ +pkg_dir = os.path.dirname(__file__) + +# This will define __version__ implicitly +with open(os.path.join(pkg_dir, "dvc", "version.py")) as fobj: + exec(fobj.read()) + +version = __version__ # noqa: F821 + + +# To achieve consistency between the build version and the one provided +# by your package during runtime, you need to **pin** the build version. +# +# This custom class will replace the version.py module with a **static** +# `__version__` that your package can read at runtime, assuring consistancy. +# +# References: +# - https://docs.python.org/3.7/distutils/extending.html +# - https://github.com/python/mypy +class build_py(_build_py): + def pin_version(self): + path = os.path.join(self.build_lib, "dvc") + self.mkpath(path) + with open(os.path.join(path, "version.py"), "w") as fobj: + fobj.write("# AUTOGENERATED at build time by setup.py\n") + fobj.write('__version__ = "{}"\n'.format(version)) + + def run(self): + self.execute(self.pin_version, ()) + _build_py.run(self) install_requires = [ @@ -34,7 +67,7 @@ all_remotes = gs + s3 + azure + ssh setup( name="dvc", - version=VERSION, + version=version, description="Git for data scientists - manage your code and data together", long_description=open("README.rst", "r").read(), author="Dmitry Petrov", @@ -61,5 +94,6 @@ setup( include_package_data=True, url="http://dataversioncontrol.com", entry_points={"console_scripts": ["dvc = dvc.main:main"]}, + cmdclass={"build_py": build_py}, zip_safe=False, ) diff --git a/tests/unit/test_updater.py b/tests/unit/test_updater.py index 490409d9e..119b53380 100644 --- a/tests/unit/test_updater.py +++ b/tests/unit/test_updater.py @@ -2,7 +2,7 @@ import os import json import mock -from dvc import VERSION +from dvc import __version__ from tests.basic_env import TestDvc @@ -24,7 +24,7 @@ def mocked_requests_get(*args, **kwargs): def json(self): return self.json_data - return MockResponse({"version": VERSION}, 200) + return MockResponse({"version": __version__}, 200) class TestUpdater(TestDvc): @@ -36,4 +36,4 @@ class TestUpdater(TestDvc): self.assertTrue(os.path.isfile(self.dvc.updater.updater_file)) with open(self.dvc.updater.updater_file, "r") as fobj: info = json.load(fobj) - self.assertEqual(info["version"], VERSION) + self.assertEqual(info["version"], __version__)