Skip to content

Commit 4d67941

Browse files
zimingdzimingd
authored
Merge pull request Sage-Bionetworks#781 from jkiang13/github-actions-build
SYNPY-1063 GitHub Actions build
2 parents 401dfd4 + 2194cce commit 4d67941

3 files changed

Lines changed: 273 additions & 103 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
# GitHub Action workflow for testing, building, and releasing the Synapse Python Client.
2+
3+
# - all pushes, releases, and pull requests are tested against unit tests, and additionally
4+
# integration tests if account configuration secrets are available.
5+
# - releases are additionally packaged, uploaded as build and release artifacts,
6+
# and deployed to pypi servers (test.pypi.org for prereleases, and pypi.org for releases)
7+
# Release tags must conform to our semver versioning, e.g. v1.2.3 in order to be packaged
8+
# for pypi deployment.
9+
10+
name: build
11+
12+
on:
13+
push:
14+
# we test all pushed branches, but not tags.
15+
# we only push tags with releases, and we handle releases explicitly
16+
branches:
17+
- '**'
18+
tags-ignore:
19+
- '**'
20+
21+
pull_request:
22+
23+
release:
24+
types:
25+
- 'published'
26+
27+
jobs:
28+
29+
# run unit (and integration tests if account secrets available) on our build matrix
30+
test:
31+
32+
strategy:
33+
matrix:
34+
os: [ubuntu-18.04, macos-10.15, windows-2019]
35+
python: [3.6, 3.7, 3.8]
36+
37+
runs-on: ${{ matrix.os }}
38+
39+
steps:
40+
- uses: actions/checkout@v2
41+
42+
- uses: actions/setup-python@v2
43+
with:
44+
python-version: ${{ matrix.python }}
45+
46+
- name: install-py-dependencies
47+
shell: bash
48+
run: |
49+
pip install -e ".[boto3,pandas,pysftp,tests]"
50+
51+
# ensure that numpy c extensions are installed on windows
52+
# https://stackoverflow.com/a/59346525
53+
if [ "${{startsWith(runner.os, 'Windows')}}" == "true" ]; then
54+
pip uninstall -y numpy
55+
pip uninstall -y setuptools
56+
pip install setuptools
57+
pip install numpy
58+
fi
59+
60+
- name: run-unit-tests
61+
shell: bash
62+
run: |
63+
pytest -sv tests/unit
64+
65+
# run integration tests iff the decryption keys for the test configuration are available.
66+
# they will not be available in pull requests from forks.
67+
- name: run-integration-tests
68+
shell: bash
69+
run: |
70+
if [ -z "${{ secrets.encrypted_d17283647768_key }}" ] || [ -z "${{ secrets.encrypted_d17283647768_key }}" ]; then
71+
echo "No test configuration decryption keys available, skipping integration tests"
72+
73+
else
74+
# host used in pysftp tests
75+
mkdir -p ~/.ssh
76+
ssh-keyscan -H ec2-18-209-45-78.compute-1.amazonaws.com >> ~/.ssh/known_hosts
77+
78+
# decrypt the encrypted test synapse configuration
79+
openssl aes-256-cbc -K ${{ secrets.encrypted_d17283647768_key }} -iv ${{ secrets.encrypted_d17283647768_iv }} -in test.synapseConfig.enc -out test.synapseConfig -d
80+
mv test.synapseConfig ~/.synapseConfig
81+
82+
pytest -sv tests/integration
83+
fi
84+
85+
# on a GitHub release, build the pip package and upload it as a GitHub release asset
86+
package:
87+
needs: test
88+
89+
runs-on: ubuntu-18.04
90+
91+
if: github.event_name == 'release'
92+
93+
steps:
94+
- uses: actions/checkout@v2
95+
96+
- uses: actions/setup-python@v2
97+
with:
98+
python-version: 3.6
99+
100+
- name: set-release-env
101+
shell: bash
102+
run: |
103+
RELEASE_TAG="${{ github.event.release.tag_name }}"
104+
if [[ $RELEASE_TAG =~ ^v?([[:digit:]\.]+)(-rc)? ]]; then
105+
VERSION="${BASH_REMATCH[1]}"
106+
if [[ "${{ github.event.release.prerelease}}" == "true" ]]; then
107+
if [[ -z "${BASH_REMATCH[2]}" ]]; then
108+
echo "A test release tag should end with \"-rc\""
109+
exit 1
110+
fi
111+
112+
# for staging builds we append the build number so we have
113+
# distinct version numbers between prod and test pypi.
114+
VERSION="$VERSION.$GITHUB_RUN_NUMBER"
115+
fi
116+
117+
else
118+
echo "Unable to parse deployment version from $RELEASE_TAG"
119+
exit 1
120+
fi
121+
122+
echo ::set-env name=VERSION::$VERSION
123+
124+
# ensure that the version file in the package will have the correct version
125+
# matching the name of the tag
126+
- name: update-version
127+
shell: bash
128+
run: |
129+
if [[ -n "$VERSION" ]]; then
130+
sed "s|\"latestVersion\":.*$|\"latestVersion\":\"$VERSION\",|g" synapseclient/synapsePythonClient > temp
131+
rm synapseclient/synapsePythonClient
132+
mv temp synapseclient/synapsePythonClient
133+
fi
134+
135+
- id: build-package
136+
shell: bash
137+
run: |
138+
python3 -m pip install setuptools
139+
140+
# install synapseclient
141+
python3 setup.py install
142+
143+
# create distribution
144+
python3 setup.py sdist
145+
146+
echo ::set-env name=PACKAGE_NAME::"synapseclient-${{env.VERSION}}.tar.gz"
147+
148+
# upload the artifact to the GitHub Action
149+
- name: upload-build-artifact
150+
uses: actions/upload-artifact@v2
151+
with:
152+
name: synapseclient
153+
path: dist/${{ env.PACKAGE_NAME }}
154+
155+
# upload the artifact as an asset of the GitHub Release
156+
- name: upload-release-asset
157+
uses: actions/upload-release-asset@v1
158+
env:
159+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
160+
with:
161+
upload_url: https://uploads.github.com/repos/${{ github.event.repository.full_name }}/releases/${{ github.event.release.id }}/assets?name=${{ env.PACKAGE_NAME }}
162+
asset_name: ${{ env.PACKAGE_NAME }}
163+
asset_path: dist/${{ env.PACKAGE_NAME }}
164+
asset_content_type: application/gzip
165+
166+
# re-download the built package to the appropriate pypi server.
167+
# we upload prereleases to test.pypi.org and releases to pypi.org.
168+
deploy:
169+
needs: package
170+
171+
runs-on: ubuntu-18.04
172+
173+
steps:
174+
175+
- uses: actions/setup-python@v2
176+
with:
177+
python-version: 3.6
178+
179+
- uses: actions/download-artifact@v2
180+
with:
181+
name: synapseclient
182+
path: dist
183+
184+
- name: deploy-to-pypi
185+
shell: bash
186+
run: |
187+
python3 -m pip install twine
188+
189+
if [[ "${{ github.event.release.prerelease}}" == "false" ]]; then
190+
# production deploy to prod pypi server
191+
192+
PYPI_NAME="pypi"
193+
PYPI_URL="https://pypi.org"
194+
PYPI_REPO="https://upload.pypi.org/legacy/"
195+
PYPI_USERNAME="${{ secrets.PYPI_PROD_USERNAME }}"
196+
PYPI_PASSWORD="${{ secrets.PYPI_PROD_PASSWORD }}"
197+
else
198+
# staging deploy to test pypi server
199+
200+
PYPI_NAME=testpypi
201+
PYPI_URL="https://test.pypi.org"
202+
PYPI_REPO=https://test.pypi.org/legacy/
203+
PYPI_USERNAME="${{ secrets.PYPI_TEST_USERNAME }}"
204+
PYPI_PASSWORD="${{ secrets.PYPI_TEST_PASSWORD }}"
205+
PYPI_INDEX_URL="https://test.pypi.org/simple/"
206+
fi
207+
208+
# create .pypirc file
209+
echo "[distutils]" > ~/.pypirc
210+
echo "index-servers=$PYPI_NAME" >> ~/.pypirc
211+
echo >> ~/.pypirc
212+
echo "[$PYPI_NAME]" >> ~/.pypirc
213+
echo "repository: $PYPI_REPO" >> ~/.pypirc
214+
echo "username:$PYPI_USERNAME" >> ~/.pypirc
215+
echo "password:$PYPI_PASSWORD" >> ~/.pypirc
216+
217+
twine upload --repository $PYPI_NAME dist/*
218+
219+
# on each of our matrix platforms, download the newly uploaded package from pypi and confirm its version
220+
check-deploy:
221+
needs: deploy
222+
223+
strategy:
224+
matrix:
225+
os: [ubuntu-18.04, macos-10.15, windows-2019]
226+
python: [3.6, 3.7, 3.8]
227+
228+
runs-on: ${{ matrix.os }}
229+
230+
steps:
231+
- uses: actions/setup-python@v2
232+
with:
233+
python-version: ${{ matrix.python }}
234+
235+
- name: check-pypi
236+
shell: bash
237+
run: |
238+
if [[ "${{ github.event.release.prerelease}}" == "false" ]]; then
239+
PYPI_INDEX_URL="https://pypi.org/simple/"
240+
else
241+
PYPI_INDEX_URL="https://test.pypi.org/simple/"
242+
fi
243+
244+
RELEASE_TAG="${{ github.event.release.tag_name }}"
245+
if [[ $RELEASE_TAG =~ ^v?([[:digit:]\.]+)(-rc)? ]]; then
246+
VERSION="${BASH_REMATCH[1]}"
247+
if [[ "${{ github.event.release.prerelease}}" == "true" ]]; then
248+
VERSION="$VERSION.$GITHUB_RUN_NUMBER"
249+
fi
250+
else
251+
echo "Unrecognized release tag"
252+
exit 1
253+
fi
254+
255+
# it can take some time for the packages to become available in pypi after uploading
256+
for i in 5 10 20 40; do
257+
if pip3 install --index-url $PYPI_INDEX_URL --extra-index-url https://pypi.org/simple "synapseclient==$VERSION"; then
258+
ACTUAL_VERSION=$(synapse --version)
259+
260+
if [ -n "$(echo "$ACTUAL_VERSION" | grep -oF "$VERSION")" ]; then
261+
echo "Successfully installed version $VERSION"
262+
exit 0
263+
else
264+
echo "Expected version $VERSION, found $ACTUAL_VERSION instead"
265+
exit 1
266+
fi
267+
fi
268+
269+
sleep $i
270+
done
271+
272+
echo "Failed to install expected version $VERSION"
273+
exit 1

‎.travis.yml‎

Lines changed: 0 additions & 80 deletions
This file was deleted.

‎appveyor.yml‎

Lines changed: 0 additions & 23 deletions
This file was deleted.

0 commit comments

Comments
 (0)