mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-10 07:18:27 +01:00
Ruslan Kuprieiev
5822d53687
```
2021-09-21T00:15:31.6789059Z ____________________ ERROR at setup of test_get_url[azure] _____________________
2021-09-21T00:15:31.6790443Z [gw0] linux -- Python 3.7.12 /opt/hostedtoolcache/Python/3.7.12/x64/bin/python
2021-09-21T00:15:31.6791202Z
2021-09-21T00:15:31.6792291Z request = <FixtureRequest for <Function test_get_url[azure]>>
2021-09-21T00:15:31.6793561Z
2021-09-21T00:15:31.6794120Z def fill(request):
2021-09-21T00:15:31.6794792Z item = request._pyfuncitem
2021-09-21T00:15:31.6795627Z fixturenames = getattr(item, "fixturenames", None)
2021-09-21T00:15:31.6797090Z if fixturenames is None:
2021-09-21T00:15:31.6797922Z fixturenames = request.fixturenames
2021-09-21T00:15:31.6798497Z
2021-09-21T00:15:31.6799393Z if hasattr(item, 'callspec'):
2021-09-21T00:15:31.6800256Z for param, val in sorted_by_dependency(item.callspec.params, fixturenames):
2021-09-21T00:15:31.6801309Z if val is not None and is_lazy_fixture(val):
2021-09-21T00:15:31.6802233Z > item.callspec.params[param] = request.getfixturevalue(val.name)
2021-09-21T00:15:31.6802912Z
2021-09-21T00:15:31.6803937Z /opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/site-packages/pytest_lazyfixture.py:35:
2021-09-21T00:15:31.6804952Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-09-21T00:15:31.6806085Z /home/runner/work/dvc/dvc/tests/remotes/azure.py:102: in azure
2021-09-21T00:15:31.6807090Z container.get_container_properties()
2021-09-21T00:15:31.6808748Z /opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/site-packages/azure/core/tracing/decorator.py:83: in wrapper_use_tracer
2021-09-21T00:15:31.6822551Z return func(*args, **kwargs)
2021-09-21T00:15:31.6824143Z /opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/site-packages/azure/storage/blob/_container_client.py:497: in get_container_properties
2021-09-21T00:15:31.6825038Z process_storage_error(error)
2021-09-21T00:15:31.6825652Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-09-21T00:15:31.6825908Z
2021-09-21T00:15:31.6826916Z storage_error = HttpResponseError("Operation returned an invalid status 'The value for one of the HTTP headers is not in the correct format.'")
2021-09-21T00:15:31.6827586Z
2021-09-21T00:15:31.6828351Z def process_storage_error(storage_error): # pylint:disable=too-many-statements
2021-09-21T00:15:31.6829086Z raise_error = HttpResponseError
2021-09-21T00:15:31.6829597Z serialized = False
2021-09-21T00:15:31.6830066Z if not storage_error.response:
2021-09-21T00:15:31.6830554Z raise storage_error
2021-09-21T00:15:31.6831558Z # If it is one of those three then it has been serialized prior by the generated layer.
2021-09-21T00:15:31.6832480Z if isinstance(storage_error, (PartialBatchErrorException,
2021-09-21T00:15:31.6833636Z ClientAuthenticationError, ResourceNotFoundError, ResourceExistsError)):
2021-09-21T00:15:31.6834687Z serialized = True
2021-09-21T00:15:31.6835753Z error_code = storage_error.response.headers.get('x-ms-error-code')
2021-09-21T00:15:31.6836524Z error_message = storage_error.message
2021-09-21T00:15:31.6837001Z additional_data = {}
2021-09-21T00:15:31.6837398Z error_dict = {}
2021-09-21T00:15:31.6837739Z try:
2021-09-21T00:15:31.6838506Z error_body = ContentDecodePolicy.deserialize_from_http_generics(storage_error.response)
2021-09-21T00:15:31.6839345Z # If it is an XML response
2021-09-21T00:15:31.6839837Z if isinstance(error_body, Element):
2021-09-21T00:15:31.6840285Z error_dict = {
2021-09-21T00:15:31.6840750Z child.tag.lower(): child.text
2021-09-21T00:15:31.6841435Z for child in error_body
2021-09-21T00:15:31.6841822Z }
2021-09-21T00:15:31.6842195Z # If it is a JSON response
2021-09-21T00:15:31.6842692Z elif isinstance(error_body, dict):
2021-09-21T00:15:31.6843413Z error_dict = error_body.get('error', {})
2021-09-21T00:15:31.6843893Z elif not error_code:
2021-09-21T00:15:31.6844347Z _LOGGER.warning(
2021-09-21T00:15:31.6845611Z 'Unexpected return type % from ContentDecodePolicy.deserialize_from_http_generics.', type(error_body))
2021-09-21T00:15:31.6846887Z error_dict = {'message': str(error_body)}
2021-09-21T00:15:31.6847294Z
2021-09-21T00:15:31.6847730Z # If we extracted from a Json or XML response
2021-09-21T00:15:31.6848205Z if error_dict:
2021-09-21T00:15:31.6848792Z error_code = error_dict.get('code')
2021-09-21T00:15:31.6849704Z error_message = error_dict.get('message')
2021-09-21T00:15:31.6850578Z additional_data = {k: v for k, v in error_dict.items() if k not in {'code', 'message'}}
2021-09-21T00:15:31.6851392Z except DecodeError:
2021-09-21T00:15:31.6851801Z pass
2021-09-21T00:15:31.6852290Z
2021-09-21T00:15:31.6852601Z try:
2021-09-21T00:15:31.6853156Z # This check would be unnecessary if we have already serialized the error
2021-09-21T00:15:31.6853821Z if error_code and not serialized:
2021-09-21T00:15:31.6854423Z error_code = StorageErrorCode(error_code)
2021-09-21T00:15:31.6855164Z if error_code in [StorageErrorCode.condition_not_met,
2021-09-21T00:15:31.6856229Z StorageErrorCode.blob_overwritten]:
2021-09-21T00:15:31.6856938Z raise_error = ResourceModifiedError
2021-09-21T00:15:31.6857739Z if error_code in [StorageErrorCode.invalid_authentication_info,
2021-09-21T00:15:31.6858641Z StorageErrorCode.authentication_failed]:
2021-09-21T00:15:31.6859468Z raise_error = ClientAuthenticationError
2021-09-21T00:15:31.6860256Z if error_code in [StorageErrorCode.resource_not_found,
2021-09-21T00:15:31.6861256Z StorageErrorCode.cannot_verify_copy_source,
2021-09-21T00:15:31.6861998Z StorageErrorCode.blob_not_found,
2021-09-21T00:15:31.6862716Z StorageErrorCode.queue_not_found,
2021-09-21T00:15:31.6863718Z StorageErrorCode.container_not_found,
2021-09-21T00:15:31.6864526Z StorageErrorCode.parent_not_found,
2021-09-21T00:15:31.6865432Z StorageErrorCode.share_not_found]:
2021-09-21T00:15:31.6866549Z raise_error = ResourceNotFoundError
2021-09-21T00:15:31.6867609Z if error_code in [StorageErrorCode.account_already_exists,
2021-09-21T00:15:31.6868499Z StorageErrorCode.account_being_created,
2021-09-21T00:15:31.6869383Z StorageErrorCode.resource_already_exists,
2021-09-21T00:15:31.6870728Z StorageErrorCode.resource_type_mismatch,
2021-09-21T00:15:31.6871688Z StorageErrorCode.blob_already_exists,
2021-09-21T00:15:31.6872429Z StorageErrorCode.queue_already_exists,
2021-09-21T00:15:31.6873252Z StorageErrorCode.container_already_exists,
2021-09-21T00:15:31.6874124Z StorageErrorCode.container_being_deleted,
2021-09-21T00:15:31.6874922Z StorageErrorCode.queue_being_deleted,
2021-09-21T00:15:31.6875857Z StorageErrorCode.share_already_exists,
2021-09-21T00:15:31.6876608Z StorageErrorCode.share_being_deleted]:
2021-09-21T00:15:31.6877289Z raise_error = ResourceExistsError
2021-09-21T00:15:31.6877807Z except ValueError:
2021-09-21T00:15:31.6878259Z # Got an unknown error code
2021-09-21T00:15:31.6878648Z pass
2021-09-21T00:15:31.6878942Z
2021-09-21T00:15:31.6879424Z # Error message should include all the error properties
2021-09-21T00:15:31.6879908Z try:
2021-09-21T00:15:31.6880416Z error_message += "\nErrorCode:{}".format(error_code.value)
2021-09-21T00:15:31.6881289Z except AttributeError:
2021-09-21T00:15:31.6883082Z error_message += "\nErrorCode:{}".format(error_code)
2021-09-21T00:15:31.6883807Z for name, info in additional_data.items():
2021-09-21T00:15:31.6884446Z error_message += "\n{}:{}".format(name, info)
2021-09-21T00:15:31.6884932Z
2021-09-21T00:15:31.6885741Z # No need to create an instance if it has already been serialized by the generated layer
2021-09-21T00:15:31.6886538Z if serialized:
2021-09-21T00:15:31.6887019Z storage_error.message = error_message
2021-09-21T00:15:31.6887526Z error = storage_error
2021-09-21T00:15:31.6887908Z else:
2021-09-21T00:15:31.6888499Z error = raise_error(message=error_message, response=storage_error.response)
2021-09-21T00:15:31.6889380Z # Ensure these properties are stored in the error instance as well (not just the error message)
2021-09-21T00:15:31.6890078Z error.error_code = error_code
2021-09-21T00:15:31.6890630Z error.additional_info = additional_data
2021-09-21T00:15:31.6891956Z # error.args is what's surfaced on the traceback - show error message in all cases
2021-09-21T00:15:31.6892672Z error.args = (error.message,)
2021-09-21T00:15:31.6893112Z try:
2021-09-21T00:15:31.6893784Z # `from None` prevents us from double printing the exception (suppresses generated layer error context)
2021-09-21T00:15:31.6894873Z > exec("raise error from None") # pylint: disable=exec-used # nosec
2021-09-21T00:15:31.6896126Z E azure.core.exceptions.HttpResponseError: The value for one of the HTTP headers is not in the correct format.
2021-09-21T00:15:31.6897385Z E RequestId:d83b8102-bde1-4a77-aa7a-b67bbc2c4a80
2021-09-21T00:15:31.6898080Z E Time:2021-09-21T00:09:53.815Z
2021-09-21T00:15:31.6898578Z E ErrorCode:InvalidHeaderValue
2021-09-21T00:15:31.6899259Z E headername:x-ms-version
2021-09-21T00:15:31.6900066Z E headervalue:2020-10-02
2021-09-21T00:15:31.6900355Z
2021-09-21T00:15:31.6901781Z /opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/site-packages/azure/storage/blob/_shared/response_handlers.py:177: HttpResponseError
```
|Banner| `Website <https://dvc.org>`_ • `Docs <https://dvc.org/doc>`_ • `Blog <http://blog.dataversioncontrol.com>`_ • `Twitter <https://twitter.com/DVCorg>`_ • `Chat (Community & Support) <https://dvc.org/chat>`_ • `Tutorial <https://dvc.org/doc/get-started>`_ • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_ |CI| |Maintainability| |Coverage| |Donate| |DOI| |PyPI| |Packages| |Brew| |Conda| |Choco| |Snap| | **Data Version Control** or **DVC** is an **open-source** tool for data science and machine learning projects. Key features: #. Simple **command line** Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services. #. Management and versioning of **datasets** and **machine learning models**. Data can be saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID. #. Makes projects **reproducible** and **shareable**; helping to answer questions about how a model was built. #. Helps manage experiments with Git tags/branches and **metrics** tracking. **DVC** aims to replace spreadsheet and document sharing tools (such as Excel or Google Docs) frequently used as both knowledge repositories and team ledgers. DVC also replaces both ad-hoc scripts to track, move, and deploy different model versions and ad-hoc data file suffixes and prefixes. .. contents:: **Contents** :backlinks: none How DVC works ============= We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ guide to better understand what DVC is and how it can fit your scenarios. The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-LFS to be precise) & Makefiles made right and tailored specifically for ML and Data Science scenarios. #. ``Git/Git-LFS`` part - DVC helps store and share data artifacts and models, connecting them with a Git repository. #. ``Makefile``\ s part - DVC describes how one data or model artifact was built from other data and code. DVC usually runs along with Git. Git is used as usual to store and version code (including DVC meta-files). DVC helps to store data and model files seamlessly out of Git, while preserving almost the same user experience as if they were stored in Git itself. To store and share the data cache, DVC supports multiple remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise network storage (via SSH, for example). |Flowchart| The DVC pipelines (computational graph) feature connects code and data together. It is possible to explicitly specify all steps required to produce a model: input dependencies including data, commands to run, and output information to be saved. See the quick start section below or the `Get Started <https://dvc.org/doc/get-started>`_ tutorial to learn more. Quick start =========== Please read `Get Started <https://dvc.org/doc/get-started>`_ guide for a full version. Common workflow commands include: +-----------------------------------+----------------------------------------------------------------------------+ | Step | Command | +===================================+============================================================================+ | Track data | | ``$ git add train.py`` | | | | ``$ dvc add images.zip`` | +-----------------------------------+----------------------------------------------------------------------------+ | Connect code and data by commands | | ``$ dvc run -n prepare -d images.zip -o images/ unzip -q images.zip`` | | | | ``$ dvc run -n train -d images/ -d train.py -o model.p python train.py`` | +-----------------------------------+----------------------------------------------------------------------------+ | Make changes and reproduce | | ``$ vi train.py`` | | | | ``$ dvc repro model.p.dvc`` | +-----------------------------------+----------------------------------------------------------------------------+ | Share code | | ``$ git add .`` | | | | ``$ git commit -m 'The baseline model'`` | | | | ``$ git push`` | +-----------------------------------+----------------------------------------------------------------------------+ | Share data and ML models | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn`` | | | | ``$ dvc push`` | +-----------------------------------+----------------------------------------------------------------------------+ Installation ============ There are four options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package. Full instructions are `available here <https://dvc.org/doc/get-started/install>`_. Snap (Snapcraft/Linux) ---------------------- |Snap| .. code-block:: bash snap install dvc --classic This corresponds to the latest tagged release. Add ``--beta`` for the latest tagged release candidate, or ``--edge`` for the latest ``master`` version. Choco (Chocolatey/Windows) -------------------------- |Choco| .. code-block:: bash choco install dvc Brew (Homebrew/Mac OS) ---------------------- |Brew| .. code-block:: bash brew install dvc Conda (Anaconda) ---------------- |Conda| .. code-block:: bash conda install -c conda-forge mamba # installs much faster than conda mamba install -c conda-forge dvc Depending on the remote storage type you plan to use to keep and share your data, you might need to install optional dependencies: `dvc-s3`, `dvc-azure`, `dvc-gdrive`, `dvc-gs`, `dvc-oss`, `dvc-ssh`. pip (PyPI) ---------- |PyPI| .. code-block:: bash pip install dvc Depending on the remote storage type you plan to use to keep and share your data, you might need to specify one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all`` to include them all. The command should look like this: ``pip install dvc[s3]`` (in this case AWS S3 dependencies such as ``boto3`` will be installed automatically). To install the development version, run: .. code-block:: bash pip install git+git://github.com/iterative/dvc Package ------- |Packages| Self-contained packages for Linux, Windows, and Mac are available. The latest version of the packages can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_. Ubuntu / Debian (deb) ^^^^^^^^^^^^^^^^^^^^^ .. code-block:: bash sudo wget https://dvc.org/deb/dvc.list -O /etc/apt/sources.list.d/dvc.list sudo apt-get update sudo apt-get install dvc Fedora / CentOS (rpm) ^^^^^^^^^^^^^^^^^^^^^ .. code-block:: bash sudo wget https://dvc.org/rpm/dvc.repo -O /etc/yum.repos.d/dvc.repo sudo yum update sudo yum install dvc Comparison to related technologies ================================== #. Data Engineering tools such as `AirFlow <https://airflow.apache.org/>`, `Luigi <https://github.com/spotify/luigi>`, and others - in DVC data, model and ML pipelines represent a single ML project focused on data scientists' experience. Data engineering tools orchestrate multiple data projects and focus on efficient execution. A DVC project can be used from existing data pipelines as a single execution step. #. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (which should not be in a Git repository) in a local key-value store, and uses file hardlinks/symlinks instead of copying/duplicating files. #. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with many remote storage services (S3, Google Cloud, Azure, SSH, etc). DVC also uses reflinks or hardlinks to avoid copy operations on checkouts; thus handling large data files much more efficiently. #. Makefile (and analogues including ad-hoc scripts) - DVC tracks dependencies (in a directed acyclic graph). #. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_ - DVC is a workflow management system designed specifically to manage machine learning experiments. DVC is built on top of Git. #. `DAGsHub <https://dagshub.com/>`_ - online service to host DVC projects. It provides a useful UI around DVC repositories and integrates other tools. #. `DVC Studio <https://studio.iterative.ai/>`_ - official online platform for DVC projects. It can be used to manage data and models, run and track experiments, and visualize and share results. Also, it integrates with `CML (CI/CD for ML) <https://cml.dev/>` for training models in the cloud or Kubernetes. Contributing ============ |Maintainability| |Donate| Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more details. Thanks to all our contributors! |Contribs| Mailing List ============ Want to stay up to date? Want to help improve DVC by participating in our occasional polls? Subscribe to our `mailing list <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_. No spam, really low traffic. Copyright ========= This project is distributed under the Apache license version 2.0 (see the LICENSE file in the project root). By submitting a pull request to this project, you agree to license your contribution under the Apache license version 2.0 to this project. Citation ======== |DOI| Iterative, *DVC: Data Version Control - Git for Data & Models* (2020) `DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_. Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and Source Code - Empirical Study of DVC Projects <https://mcis.cs.queensu.ca/publications/2021/saner.pdf>`_ , in Proceedings of the 28th IEEE International Conference on Software Analysis, Evolution, and Reengineering, SANER 2021. Hawaii, USA. .. |Banner| image:: https://dvc.org/img/logo-github-readme.png :target: https://dvc.org :alt: DVC logo .. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=master :target: https://github.com/iterative/dvc/actions :alt: GHA Tests .. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg :target: https://codeclimate.com/github/iterative/dvc :alt: Code Climate .. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg :target: https://codecov.io/gh/iterative/dvc :alt: Codecov .. |Donate| image:: https://img.shields.io/badge/patreon-donate-green.svg?logo=patreon :target: https://www.patreon.com/DVCorg/overview :alt: Donate .. |Snap| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft :target: https://snapcraft.io/dvc :alt: Snapcraft .. |Choco| image:: https://img.shields.io/chocolatey/v/dvc?label=choco :target: https://chocolatey.org/packages/dvc :alt: Chocolatey .. |Brew| image:: https://img.shields.io/homebrew/v/dvc?label=brew :target: https://formulae.brew.sh/formula/dvc :alt: Homebrew .. |Conda| image:: https://img.shields.io/conda/v/conda-forge/dvc.svg?label=conda&logo=conda-forge :target: https://anaconda.org/conda-forge/dvc :alt: Conda-forge .. |PyPI| image:: https://img.shields.io/pypi/v/dvc.svg?label=pip&logo=PyPI&logoColor=white :target: https://pypi.org/project/dvc :alt: PyPI .. |Packages| image:: https://img.shields.io/github/v/release/iterative/dvc?label=deb|pkg|rpm|exe&logo=GitHub :target: https://github.com/iterative/dvc/releases/latest :alt: deb|pkg|rpm|exe .. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.3677553-blue.svg :target: https://doi.org/10.5281/zenodo.3677553 :alt: DOI .. |Flowchart| image:: https://dvc.org/img/flow.gif :target: https://dvc.org/img/flow.gif :alt: how_dvc_works .. |Contribs| image:: https://contrib.rocks/image?repo=iterative/dvc :target: https://github.com/iterative/dvc/graphs/contributors :alt: Contributors