diff --git a/MANIFEST.in b/MANIFEST.in index 226626ef5..5099b94cc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include fastentrypoints.py include LICENSE -include test_requirements.txt +recursive-include requirements *.txt diff --git a/requirements/azure.txt b/requirements/azure.txt new file mode 100644 index 000000000..1bd6d816d --- /dev/null +++ b/requirements/azure.txt @@ -0,0 +1,3 @@ +adlfs==2021.8.2 +azure-identity>=1.4.0 +knack diff --git a/requirements/gdrive.txt b/requirements/gdrive.txt new file mode 100644 index 000000000..e5e8ee184 --- /dev/null +++ b/requirements/gdrive.txt @@ -0,0 +1 @@ +pydrive2[fsspec]>=1.9.2 diff --git a/requirements/gs.txt b/requirements/gs.txt new file mode 100644 index 000000000..4616d9a0f --- /dev/null +++ b/requirements/gs.txt @@ -0,0 +1 @@ +gcsfs==2021.8.1 diff --git a/requirements/hdfs.txt b/requirements/hdfs.txt new file mode 100644 index 000000000..06b8ec737 --- /dev/null +++ b/requirements/hdfs.txt @@ -0,0 +1 @@ +pyarrow>=2.0.0; python_version < '3.10' diff --git a/requirements/oss.txt b/requirements/oss.txt new file mode 100644 index 000000000..92e507ee2 --- /dev/null +++ b/requirements/oss.txt @@ -0,0 +1 @@ +ossfs==2021.8.0 diff --git a/requirements/s3.txt b/requirements/s3.txt new file mode 100644 index 000000000..495ee8285 --- /dev/null +++ b/requirements/s3.txt @@ -0,0 +1,2 @@ +s3fs==2021.8.1 +aiobotocore[boto3]>1.0.1 diff --git a/requirements/ssh.txt b/requirements/ssh.txt new file mode 100644 index 000000000..c6238d7b3 --- /dev/null +++ b/requirements/ssh.txt @@ -0,0 +1 @@ +sshfs>=2021.8.1 diff --git a/requirements/ssh_gssapi.txt b/requirements/ssh_gssapi.txt new file mode 100644 index 000000000..6345c0896 --- /dev/null +++ b/requirements/ssh_gssapi.txt @@ -0,0 +1 @@ +sshfs[gssapi]>=2021.8.1 diff --git a/requirements/terraform.txt b/requirements/terraform.txt new file mode 100644 index 000000000..a7bbfa0e7 --- /dev/null +++ b/requirements/terraform.txt @@ -0,0 +1 @@ +tpi[ssh]>=0.0.0 diff --git a/test_requirements.txt b/requirements/tests.txt similarity index 100% rename from test_requirements.txt rename to requirements/tests.txt diff --git a/requirements/webdav.txt b/requirements/webdav.txt new file mode 100644 index 000000000..e5c592334 --- /dev/null +++ b/requirements/webdav.txt @@ -0,0 +1 @@ +webdav4>=0.9.1 diff --git a/requirements/webhdfs.txt b/requirements/webhdfs.txt new file mode 100644 index 000000000..bf50d1a6a --- /dev/null +++ b/requirements/webhdfs.txt @@ -0,0 +1 @@ +hdfs==2.5.8 diff --git a/setup.py b/setup.py index bede25d4a..294ce3765 100644 --- a/setup.py +++ b/setup.py @@ -97,29 +97,22 @@ install_requires = [ # Extra dependencies for remote integrations +extra_requirements = { + path.stem: path.read_text().strip().splitlines() + for path in Path("requirements").glob("*.txt") +} -gs = ["gcsfs==2021.8.1"] -gdrive = ["pydrive2[fsspec]>=1.9.2"] -s3 = ["s3fs==2021.8.1", "aiobotocore[boto3]>1.0.1"] -azure = ["adlfs==2021.8.2", "azure-identity>=1.4.0", "knack"] -oss = ["ossfs==2021.8.0"] -ssh = ["sshfs>=2021.8.1"] - -hdfs = ["pyarrow>=2.0.0; python_version < '3.10'"] -webhdfs = ["hdfs==2.5.8"] -webdav = ["webdav4>=0.9.1"] # gssapi should not be included in all_remotes, because it doesn't have wheels # for linux and mac, so it will fail to compile if user doesn't have all the # requirements, including kerberos itself. Once all the wheels are available, # we can start shipping it by default. -ssh_gssapi = ["sshfs[gssapi]>=2021.8.1"] -all_remotes = gs + s3 + azure + ssh + oss + gdrive + hdfs + webhdfs + webdav -terraform = ["tpi[ssh]>=0.0.0"] - -tests_requirements = ( - Path("test_requirements.txt").read_text().strip().splitlines() -) + terraform +extra_requirements["all"] = [ + requirements + for key, requirements in extra_requirements.items() + if key not in ("tests", "ssh_gssapi", "terraform") +] +extra_requirements["tests"] += extra_requirements["terraform"] setup( name="dvc", @@ -133,21 +126,7 @@ setup( download_url="https://github.com/iterative/dvc", license="Apache License 2.0", install_requires=install_requires, - extras_require={ - "all": all_remotes, - "gs": gs, - "gdrive": gdrive, - "s3": s3, - "azure": azure, - "oss": oss, - "ssh": ssh, - "ssh_gssapi": ssh_gssapi, - "hdfs": hdfs, - "webhdfs": webhdfs, - "webdav": webdav, - "terraform": terraform, - "tests": tests_requirements, - }, + extras_require=extra_requirements, keywords="data-science data-version-control machine-learning git" " developer-tools reproducibility collaboration ai", python_requires=">=3.6",