dvc: refactor package versioning logic

- Use `version.py` to compute a dynamic version stored in `__version__`.
  Version information was scattered across the source code
  (`VERSION_BASE`, `VERSION`, `__version__`, and `version.py`).

- Read `__version__` from `setup.py` and **pin** it using custom
  `build_py` command class.
  This way, we can maintain consistancy between the build version
  and the runtime one.
  Also, this gives us the ability to run `setup.py` without depending on
  on `dvc` itself.

Close #1824

.gitignore

@@ -24,8 +24,6 @@ innosetup/config.ini
 
 .coverage
 
-dvc/version.py
-
 *.swp
 
 pip-wheel-metadata/

dvc/__init__.py

@@ -6,75 +6,9 @@ Make your data science projects reproducible and shareable.
 
 from __future__ import unicode_literals
 
-import os
+from dvc.version import __version__  # noqa: F401
 import warnings
 
-
-VERSION_BASE = "0.34.2"
-__version__ = VERSION_BASE
-
-PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
-HOMEPATH = os.path.dirname(PACKAGEPATH)
-VERSIONPATH = os.path.join(PACKAGEPATH, "version.py")
-
-
-def _update_version_file():
-    """Dynamically update version file."""
-    try:
-        from git import Repo
-        from git.exc import InvalidGitRepositoryError
-    except ImportError:
-        return __version__
-
-    try:
-        repo = Repo(HOMEPATH)
-    except InvalidGitRepositoryError:
-        return __version__
-
-    sha = repo.head.object.hexsha
-    short_sha = repo.git.rev_parse(sha, short=6)
-    dirty = ".mod" if repo.is_dirty() else ""
-    ver = "{}+{}{}".format(__version__, short_sha, dirty)
-
-    # Write a helper file, that will be installed with the package
-    # and will provide a true version of the installed dvc
-    with open(VERSIONPATH, "w+") as fobj:
-        fobj.write("# AUTOGENERATED by dvc/__init__.py\n")
-        fobj.write('version = "{}"\n'.format(ver))
-
-    return ver
-
-
-def _remove_version_file():
-    """Remove version.py so that it doesn't get into the release."""
-    if os.path.exists(VERSIONPATH):
-        os.unlink(VERSIONPATH)
-
-
-if os.path.exists(os.path.join(HOMEPATH, "setup.py")):
-    # dvc is run directly from source without installation or
-    # __version__ is called from setup.py
-    if (
-        os.getenv("APPVEYOR_REPO_TAG", "").lower() != "true"
-        and os.getenv("TRAVIS_TAG", "") == ""
-        and os.getenv("DVC_TEST", "").lower() != "true"
-    ):
-        __version__ = _update_version_file()
-    else:  # pragma: no cover
-        _remove_version_file()
-else:  # pragma: no cover
-    # dvc was installed with pip or something. Hopefully we have our
-    # auto-generated version.py to help us provide a true version
-    try:
-        from dvc.version import version
-
-        __version__ = version
-    except Exception:
-        pass
-
-VERSION = __version__
-
-
 # Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
 # We don't directly import numpy, but our dependency networkx does, causing
 # these warnings in some environments. Luckily these warnings are benign and

dvc/analytics.py

@@ -9,7 +9,7 @@ import json
 import errno
 
 import dvc.logger as logger
-from dvc import VERSION
+from dvc import __version__
 
 
 class Analytics(object):
@@ -164,7 +164,7 @@ class Analytics(object):
         from dvc.repo import Repo
         from dvc.exceptions import NotDvcRepoError
 
-        self.info[self.PARAM_DVC_VERSION] = VERSION
+        self.info[self.PARAM_DVC_VERSION] = __version__
         self.info[self.PARAM_IS_BINARY] = is_binary()
         self.info[self.PARAM_USER_ID] = self._get_user_id()
 

dvc/cli.py

@@ -33,7 +33,6 @@ import dvc.command.daemon as daemon
 import dvc.command.commit as commit
 import dvc.command.tag as tag
 from dvc.exceptions import DvcParserError
-from dvc import VERSION
 
 
 COMMANDS = [
@@ -86,7 +85,9 @@ class VersionAction(argparse.Action):  # pragma: no cover
     """Shows dvc version and exits."""
 
     def __call__(self, parser, namespace, values, option_string=None):
-        print(VERSION)
+        from dvc import __version__
+
+        print(__version__)
         sys.exit(0)
 
 

dvc/state.py

@@ -164,7 +164,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         return num
 
     def _prepare_db(self, empty=False):
-        from dvc import VERSION
+        from dvc import __version__
 
         if not empty:
             cmd = "PRAGMA user_version;"
@@ -176,7 +176,9 @@ class State(object):  # pylint: disable=too-many-instance-attributes
             version = ret[0][0]
 
             if version > self.VERSION:
-                raise StateVersionTooNewError(VERSION, self.VERSION, version)
+                raise StateVersionTooNewError(
+                    __version__, self.VERSION, version
+                )
             elif version < self.VERSION:
                 msg = (
                     "State file version '{}' is too old. "

dvc/updater.py

@@ -5,9 +5,10 @@ import os
 import time
 import requests
 import colorama
+from pkg_resources import parse_version
 
+from dvc import __version__
 import dvc.logger as logger
-from dvc import VERSION_BASE
 from dvc.lock import Lock, LockError
 from dvc.utils import is_binary
 
@@ -22,7 +23,7 @@ class Updater(object):  # pragma: no cover
         self.dvc_dir = dvc_dir
         self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
         self.lock = Lock(dvc_dir, self.updater_file + ".lock")
-        self.current = VERSION_BASE
+        self.current = parse_version(__version__).base_version
 
     def _is_outdated_file(self):
         ctime = os.path.getmtime(self.updater_file)
@@ -89,16 +90,7 @@ class Updater(object):  # pragma: no cover
             json.dump(info, fobj)
 
     def _is_outdated(self):
-        l_major, l_minor, l_patch = [int(x) for x in self.latest.split(".")]
-        c_major, c_minor, c_patch = [int(x) for x in self.current.split(".")]
-
-        if l_major != c_major:
-            return l_major > c_major
-
-        if l_minor != c_minor:
-            return l_minor > c_minor
-
-        return l_patch > c_patch
+        return parse_version(self.current) < parse_version(self.latest)
 
     def _notify(self):
         if not sys.stdout.isatty():

dvc/version.py

@@ -0,0 +1,52 @@
+# Used from setup.py, so don't pull any additional dependencies
+import os
+import subprocess
+
+
+def generate_version(base_version):
+    """Generate a version with information about the git repository"""
+    pkg_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+
+    if not is_git_repo(pkg_dir) or not have_git(pkg_dir):
+        return base_version
+
+    return "{base_version}+{short_sha}{dirty}".format(
+        base_version=base_version,
+        short_sha=git_revision(pkg_dir).decode("utf-8")[0:6],
+        dirty=".mod" if is_dirty(pkg_dir) else "",
+    )
+
+
+def is_git_repo(dir_path):
+    """Is the given directory version-controlled with git?"""
+    return os.path.exists(os.path.join(dir_path, ".git"))
+
+
+def have_git(dir_path):
+    """Can we run the git executable?"""
+    try:
+        subprocess.check_output(["git", "--help"])
+        return True
+    except subprocess.CalledProcessError:
+        return False
+    except OSError:
+        return False
+
+
+def git_revision(dir_path):
+    """Get the SHA-1 of the HEAD of a git repository."""
+    return subprocess.check_output(
+        ["git", "rev-parse", "HEAD"], cwd=dir_path
+    ).strip()
+
+
+def is_dirty(dir_path):
+    """Check whether a git repository has uncommitted changes."""
+    try:
+        subprocess.check_call(["git", "diff", "--quiet"], cwd=dir_path)
+        return False
+    except subprocess.CalledProcessError:
+        return True
+
+
+__version__ = generate_version(base_version="0.34.2")

scripts/build_posix.sh

@@ -42,7 +42,7 @@ command_exists()
 fpm_build()
 {
 	print_info "Building $1..."
-	VERSION=$(python -c "import dvc; from dvc import VERSION; print(str(VERSION))")
+	VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))")
 	fpm -s dir \
 	    -f \
 	    -t $1 \

scripts/innosetup/config_gen.py

@@ -1,10 +1,10 @@
 # This script generates config.ini for setup.iss script
-from dvc import VERSION
+from dvc import __version__
 from dvc.utils.compat import ConfigParser
 
 config = ConfigParser.ConfigParser()
 config.add_section("Version")
-config.set("Version", "Version", VERSION)
+config.set("Version", "Version", __version__)
 
 with open("scripts/innosetup/config.ini", "w") as f:
     config.write(f)

setup.py

@@ -1,5 +1,38 @@
 from setuptools import setup, find_packages
-from dvc import VERSION
+from setuptools.command.build_py import build_py as _build_py
+import os
+
+
+# https://packaging.python.org/guides/single-sourcing-package-version/
+pkg_dir = os.path.dirname(__file__)
+
+# This will define __version__ implicitly
+with open(os.path.join(pkg_dir, "dvc", "version.py")) as fobj:
+    exec(fobj.read())
+
+version = __version__  # noqa: F821
+
+
+# To achieve consistency between the build version and the one provided
+# by your package during runtime, you need to **pin** the build version.
+#
+# This custom class will replace the version.py module with a **static**
+# `__version__` that your package can read at runtime, assuring consistancy.
+#
+# References:
+#   - https://docs.python.org/3.7/distutils/extending.html
+#   - https://github.com/python/mypy
+class build_py(_build_py):
+    def pin_version(self):
+        path = os.path.join(self.build_lib, "dvc")
+        self.mkpath(path)
+        with open(os.path.join(path, "version.py"), "w") as fobj:
+            fobj.write("# AUTOGENERATED at build time by setup.py\n")
+            fobj.write('__version__ = "{}"\n'.format(version))
+
+    def run(self):
+        self.execute(self.pin_version, ())
+        _build_py.run(self)
 
 
 install_requires = [
@@ -34,7 +67,7 @@ all_remotes = gs + s3 + azure + ssh
 
 setup(
     name="dvc",
-    version=VERSION,
+    version=version,
     description="Git for data scientists - manage your code and data together",
     long_description=open("README.rst", "r").read(),
     author="Dmitry Petrov",
@@ -61,5 +94,6 @@ setup(
     include_package_data=True,
     url="http://dataversioncontrol.com",
     entry_points={"console_scripts": ["dvc = dvc.main:main"]},
+    cmdclass={"build_py": build_py},
     zip_safe=False,
 )

tests/unit/test_updater.py

@@ -2,7 +2,7 @@ import os
 import json
 import mock
 
-from dvc import VERSION
+from dvc import __version__
 from tests.basic_env import TestDvc
 
 
@@ -24,7 +24,7 @@ def mocked_requests_get(*args, **kwargs):
         def json(self):
             return self.json_data
 
-    return MockResponse({"version": VERSION}, 200)
+    return MockResponse({"version": __version__}, 200)
 
 
 class TestUpdater(TestDvc):
@@ -36,4 +36,4 @@ class TestUpdater(TestDvc):
         self.assertTrue(os.path.isfile(self.dvc.updater.updater_file))
         with open(self.dvc.updater.updater_file, "r") as fobj:
             info = json.load(fobj)
-        self.assertEqual(info["version"], VERSION)
+        self.assertEqual(info["version"], __version__)