mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-05 23:09:32 +01:00
nanaya-tachibana
7ceaf88c54
* remote: add support for aliyun oss Usage: $ dvc remote add myremote oss://my-bucket.endpoint/path Set key id and key secret using modify command $ dvc remote modify myremote oss_key_id my-key-id $ dvc remote modify myremote oss_key_secret my-key-secret or environment variables $ export OSS_ACCESS_KEY_ID="my-key-id" $ export OSS_ACCESS_KEY_SECRET="my-key-secret" Ref: oss python SDK: https://www.alibabacloud.com/help/doc-detail/32026.htm * Not needed, since we don't support external dependencies/outputs on OSS. See https://github.com/iterative/dvc/pull/1961. * Add a way to test oss storage using docker. Start a container running an oss emulator. $ git clone https://github.com/nanaya-tachibana/oss-emulator.git $ docker image build -t oss:1.0 oss-emulator $ docker run --detach -p 8880:8880 --name oss-emulator oss:1.0 Setup environment variables. $ export OSS_BUCKET='my-bucket' $ export OSS_ENDPOINT='localhost:8880' $ export OSS_ACCESS_KEY_ID='AccessKeyID' $ export OSS_ACCESS_KEY_SECRET='AccessKeySecret' * Use default key id and key secret when they are not given, which gives read access to public read bucket and public bucket. * test: add oss tests to appveyor. * remove unneeded tests. * remote: use s3 style url for oss storage and make endpoint a configurable value. Usage: $ dvc remote add myremote oss://my-bucket/path Set key id, key secret and endpoint using modify command $ dvc remote modify myremote oss_key_id my-key-id $ dvc remote modify myremote oss_key_secret my-key-secret $ dvc remote modify myremote oss_endpoint endpoint or environment variables $ export OSS_ACCESS_KEY_ID="my-key-id" $ export OSS_ACCESS_KEY_SECRET="my-key-secret" $ export OSS_ENDPOINT="endpoint" * remote: fallback to [] if there are no cinfos Signed-off-by: Ruslan Kuprieiev <[email protected]> * test: remote: add oss CLI test Signed-off-by: Ruslan Kuprieiev <[email protected]> * travis: use iterative's fork of oss-emulator Just to keep things in-house. Signed-off-by: Ruslan Kuprieiev <[email protected]>
67 lines
1.8 KiB
Bash
67 lines
1.8 KiB
Bash
#!/bin/bash
|
|
|
|
set -x
|
|
set -e
|
|
|
|
if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
|
|
ulimit -a
|
|
sudo sysctl -w kern.maxproc=2048
|
|
sudo sysctl -w kern.maxprocperuid=2048
|
|
echo '\nulimit -u 2048' >> ~/.bash_profile
|
|
ulimit -a
|
|
brew update
|
|
brew upgrade pyenv
|
|
# NOTE: used to install and run our style checking tools
|
|
osx_python_ver=3.6.2
|
|
eval "$(pyenv init -)"
|
|
pyenv install --skip-existing --keep --verbose $osx_python_ver &> pyenv.log || tail -n 50 pyenv.log
|
|
pyenv shell $osx_python_ver
|
|
python --version
|
|
fi
|
|
|
|
pip install -r tests/requirements.txt
|
|
|
|
# stop the build if there are any readme formatting errors
|
|
python setup.py checkdocs
|
|
|
|
# stop the build if there are any black or flake8 errors
|
|
black ./ --check
|
|
flake8 ./
|
|
|
|
#NOTE: ssh keys for ssh test to be able to ssh to the localhost
|
|
ls -la ~/.ssh/
|
|
|
|
ssh-keygen -t rsa -N "" -f mykey
|
|
mkdir -p ~/.ssh
|
|
cp mykey ~/.ssh/id_rsa
|
|
cp mykey.pub ~/.ssh/id_rsa.pub
|
|
cat mykey.pub >> ~/.ssh/authorized_keys
|
|
ssh-keyscan localhost >> ~/.ssh/known_hosts
|
|
ssh-keyscan 127.0.0.1 >> ~/.ssh/known_hosts
|
|
ssh-keyscan 0.0.0.0 >> ~/.ssh/known_hosts
|
|
ssh 0.0.0.0 ls &> /dev/null
|
|
ssh 127.0.0.1 ls &> /dev/null
|
|
ssh localhost ls &> /dev/null
|
|
|
|
scriptdir="$(dirname $0)"
|
|
|
|
echo > env.sh
|
|
if [ -n "$TRAVIS_OS_NAME" ] && [ "$TRAVIS_OS_NAME" != "osx" ] \
|
|
&& [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
|
|
bash "$scriptdir/install_azurite.sh"
|
|
bash "$scriptdir/install_oss.sh"
|
|
bash "$scriptdir/install_hadoop.sh"
|
|
fi
|
|
|
|
if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
|
|
brew install openssl
|
|
brew cask install google-cloud-sdk
|
|
fi
|
|
|
|
if [[ -n "$PYTHON_VER" ]]; then
|
|
eval "$(pyenv init -)"
|
|
pyenv install --skip-existing --keep --verbose "$PYTHON_VER" &> pyenv.log || tail -n 50 pyenv.log
|
|
echo 'eval "$(pyenv init -)"' >> env.sh
|
|
echo "pyenv shell $PYTHON_VER" >> env.sh
|
|
fi
|