Coalesce requirements.txt into setup.py (#2017)

* Coalesce `requirements.txt` and `tests/requirements.txt` into `setup.py`

* Remove requirements for tests

* Parse tests requirements in setup.py

* Append extra dependenices in test_requirements in setup.py

* Modify CI build to pass the tests

* Correctly install black requirements

* Install all requirements to pass the tests

.appveyor.yml

@@ -47,7 +47,7 @@ install:
   - cinst gsutil
   - cinst openssl.light --version 1.1.1
   - python -m pip install -U pip setuptools wheel
-  - pip install -r requirements.txt
+  - pip install -e .[all]
   - python setup.py install
 
 build: false
@@ -57,7 +57,7 @@ before_test:
   - aws configure set aws_secret_access_key "%aws_secret_access_key%"
   - aws configure set region us-east-2
   - openssl enc -d -aes-256-cbc -md md5 -k "%GCP_CREDS%" -in scripts\ci\gcp-creds.json.enc -out scripts\ci\gcp-creds.json & exit 0
-  - pip install -r tests/requirements.txt
+  - pip install -e .[tests]
 
 test_script:
   - python -mtests

requirements.txt

@@ -1,33 +0,0 @@
-azure-storage-blob==1.3.0
-boto3==1.9.115
-ply>=3.9 # See https://github.com/pyinstaller/pyinstaller/issues/1945
-configparser>=3.5.0
-zc.lockfile>=1.2.1
-future>=0.16.0
-google-cloud-storage==1.13.0
-PyInstaller==3.4
-colorama>=0.3.9
-configobj>=5.0.6
-networkx>=2.1
-gitpython>=2.1.8
-setuptools>=34.0.0
-nanotime>=0.5.2
-pyasn1>=0.4.1
-schema>=0.6.7
-paramiko>=2.4.1
-jsonpath-ng>=1.4.3
-requests>=2.22.0
-wheel>=0.31.1
-futures>=3.2.0; python_version == "2.7"
-grandalf==0.6
-pydot>=1.2.4
-asciimatics>=1.10.0
-distro>=1.3.0
-appdirs>=1.4.3
-treelib>=1.5.5
-inflect>=2.1.0
-humanize>=0.5.1
-dulwich>=0.19.11
-ruamel.yaml>=0.15.91
-pathlib2==2.3.3; python_version == "2.7"
-oss2==2.6.1

scripts/build_posix.sh

@@ -78,7 +78,7 @@ install_dependencies()
 	gem install --no-document fpm
 
 	print_info "Installing requirements..."
-	pip install -r requirements.txt
+	pip install -e .[all]
 
 	print_info "Installing pyinstaller..."
 	pip install pyinstaller

scripts/build_windows.cmd

@@ -15,7 +15,7 @@ if %errorlevel% neq 0 (echo Error: pip not found && goto :error)
 if not exist "C:\Program Files (x86)\Inno Setup 5\ISCC.exe" (echo Error: Couldn't find Inno Setup compiler. Please go to jrsoftware.org/isinfo.php and install Inno Setup 5 && goto :error)
 
 echo ====== Installing requirements... ======
-call pip install -r requirements.txt || goto :error
+call pip install -e .[all] || goto :error
 call pip install pyinstaller || goto :error
 
 echo ====== Building dvc binary... ======

scripts/ci/before_install.sh

@@ -19,7 +19,7 @@ if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
     python --version
 fi
 
-pip install -r tests/requirements.txt
+pip install -e .[tests]
 
 # stop the build if there are any readme formatting errors
 python setup.py checkdocs

scripts/ci/install.sh

@@ -15,8 +15,8 @@ function retry {
 # networking issues. Thus, let's retry a few times to see if it will eventially
 # work or not.
 retry pip install --upgrade pip setuptools wheel
-retry pip install -r requirements.txt
-retry pip install -r tests/requirements.txt
+retry pip install -e .[all]
+retry pip install -e .[tests]
 
 git config --global user.email "[email protected]"
 git config --global user.name "DVC Tester"

setup.py

@@ -1,6 +1,7 @@
 from setuptools import setup, find_packages
 from setuptools.command.build_py import build_py as _build_py
 import os
+import sys
 
 
 # https://packaging.python.org/guides/single-sourcing-package-version/
@@ -69,6 +70,34 @@ oss = ["oss2==2.6.1"]
 ssh = ["paramiko>=2.4.1"]
 all_remotes = gs + s3 + azure + ssh + oss
 
+# Extra dependecies to run tests
+tests_requirements = [
+    "PyInstaller==3.4",
+    "wheel>=0.31.1",
+    "pydot>=1.2.4",
+    # Test requirements:
+    "pytest>=4.4.0",
+    "pytest-timeout>=1.3.3",
+    "pytest-cov>=2.6.1",
+    "pytest-xdist>=1.26.1",
+    "pytest-mock>=1.10.4",
+    "flaky>=3.5.3",
+    "mock>=3.0.0",
+    "xmltodict>=0.11.0",
+    "awscli>=1.16.125",
+    "google-compute-engine",
+    "pywin32; sys_platform == 'win32'",
+    "Pygments",  # required by collective.checkdocs,
+    "collective.checkdocs",
+    "flake8",
+    "flake8-docstrings",
+    "jaraco.windows==3.9.2",
+    "mock-ssh-server>=0.5.0",
+]
+
+if (sys.version_info) >= (3, 6):
+    tests_requirements.append("black==19.3b0")
+
 setup(
     name="dvc",
     version=version,
@@ -87,7 +116,8 @@ setup(
         "oss": oss,
         "ssh": ssh,
         # NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1
-        ':python_version=="2.7"': ["futures", "pathlib2"],
+        ":python_version=='2.7'": ["futures", "pathlib2"],
+        "tests": tests_requirements,
     },
     keywords="data science, data version control, machine learning",
     python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",

tests/requirements.txt

@@ -1,18 +0,0 @@
-pytest>=4.4.0
-pytest-timeout>=1.3.3
-pytest-cov>=2.6.1
-pytest-xdist>=1.26.1
-pytest-mock>=1.10.4
-flaky>=3.5.3
-mock>=3.0.0
-xmltodict>=0.11.0
-awscli>=1.16.125
-google-compute-engine
-pywin32; sys_platform == 'win32'
-Pygments  # required by collective.checkdocs
-collective.checkdocs
-black==19.3b0 ; python_version >= '3.6'
-flake8
-flake8-docstrings
-jaraco.windows==3.9.2
-mock-ssh-server >= 0.5.0