CI: run all test flavors even on failure
[tinc] / .github / workflows / test.yml
1 name: Test
2
3 on:
4   push:
5   pull_request:
6     types:
7       - opened
8       - synchronize
9
10 jobs:
11   cross:
12     runs-on: ubuntu-latest
13     timeout-minutes: 30
14     strategy:
15       fail-fast: false
16       matrix:
17         arch:
18           - armhf
19           - mipsel
20           - mingw
21
22     container:
23       image: debian:bullseye
24       options: --privileged
25
26     steps:
27       - name: Checkout code
28         uses: actions/checkout@v1
29
30       - name: Install deps
31         run: HOST=${{ matrix.arch }} sh .ci/deps.sh
32
33       - name: Prepare the system
34         run: HOST=${{ matrix.arch }} sh .ci/test/prepare.sh
35
36       - name: Run tests with default settings
37         run: sudo -u build CI=1 HOST=${{ matrix.arch }} sh .ci/test/run.sh default
38
39       - name: Run tests without legacy protocol
40         run: sudo -u build CI=1 HOST=${{ matrix.arch }} sh .ci/test/run.sh nolegacy
41         if: always()
42
43       - name: Run tests with libgcrypt
44         run: sudo -u build CI=1 HOST=${{ matrix.arch }} sh .ci/test/run.sh gcrypt
45         if: always()
46
47       - name: Upload test results
48         uses: actions/upload-artifact@v2
49         with:
50           name: tests_cross_${{ matrix.arch }}
51           path: /tmp/logs/tests.*.tar.gz
52         if: always()
53
54   static-analysis:
55     runs-on: ubuntu-latest
56     timeout-minutes: 30
57     steps:
58       - name: Checkout code
59         uses: actions/checkout@v1
60
61       - name: Install tools
62         run: |
63           sudo apt-get install -y astyle clang-tidy-$CLANG
64           sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-$CLANG 100
65           curl -OL "https://github.com/koalaman/shellcheck/releases/download/v$SHELLCHECK/shellcheck-v${SHELLCHECK}.linux.x86_64.tar.xz"
66           tar -C ~ --strip-components=1 --wildcards -xf ./shellcheck-*.tar.xz 'shellcheck-*/shellcheck'
67           curl -o ~/shfmt -L "https://github.com/mvdan/sh/releases/download/v$SHFMT/shfmt_v${SHFMT}_linux_amd64"
68           chmod 755 ~/shfmt ~/shellcheck
69           python3 -m venv /tmp/venv
70           . /tmp/venv/bin/activate
71           pip3 install black pylint mypy
72         env:
73           CLANG: 11
74           SHELLCHECK: 0.7.2
75           SHFMT: 3.3.0
76
77       - name: Install deps
78         run: sudo SKIP_OPENSSL3=1 sh .ci/deps.sh
79
80       - name: Lint/typecheck/check formatting on C/shell/Python code
81         run: |
82           . /tmp/venv/bin/activate
83           PATH=$PATH:$HOME ./lint.py
84         if: always()
85
86       - name: Run clang-tidy
87         run: sh .ci/tidy/run.sh
88         if: always()
89
90       - name: Check warnings (gcc)
91         run: bash .ci/warn/run.sh
92         env:
93           CC: gcc-10
94         if: always()
95
96       - name: Check warnings (clang)
97         run: bash .ci/warn/run.sh
98         env:
99           CC: clang-12
100         if: always()
101
102   sanitizer:
103     runs-on: ubuntu-latest
104     timeout-minutes: 30
105     strategy:
106       fail-fast: false
107       matrix:
108         sanitizer:
109           - address
110           - thread
111           - undefined
112     env:
113       SANITIZER: "${{ matrix.sanitizer }}"
114
115     steps:
116       - name: Checkout code
117         uses: actions/checkout@v1
118
119       - name: Install deps
120         run: sudo sh .ci/deps.sh
121
122       - name: Run tests with OpenSSL 3
123         run: bash .ci/sanitizers/run.sh openssl3
124
125       - name: Sanitize tests with default settings
126         run: bash .ci/sanitizers/run.sh default
127         if: always()
128
129       - name: Sanitize tests without legacy protocol
130         run: bash .ci/sanitizers/run.sh nolegacy
131         if: always()
132
133       - name: Upload test results
134         uses: actions/upload-artifact@v2
135         with:
136           name: tests_sanitizer_${{ matrix.sanitizer }}
137           path: /tmp/logs/tests.*.tar.gz
138         if: always()
139
140   linux:
141     runs-on: ubuntu-latest
142     timeout-minutes: 30
143     strategy:
144       fail-fast: false
145       matrix:
146         os:
147           - alpine
148           - centos:7 # aka RHEL 7
149           - almalinux:8 # aka RHEL 8
150           - fedora
151           - debian:buster
152           - debian:bullseye
153           - debian:testing
154           - ubuntu # current LTS
155           - ubuntu:rolling # latest
156     container:
157       image: ${{ matrix.os }}
158       options: --privileged
159       env:
160         CI: 1
161     steps:
162       - name: Checkout code
163         uses: actions/checkout@v1
164
165       - name: Install deps
166         run: sh .ci/deps.sh
167
168       - name: Assign name for test results artifact
169         run: echo ARTIFACT="$(echo '${{ matrix.os }}' | sed 's|[:/]|_|g')" >>"$GITHUB_ENV"
170
171       - name: Create a non-privileged user
172         run: sh .ci/test/prepare.sh
173
174       - name: Run tests with OpenSSL 3
175         run: sudo -u build CI=1 sh .ci/test/run.sh openssl3
176
177       - name: Run tests with default settings
178         run: sudo -u build CI=1 sh .ci/test/run.sh default
179         if: always()
180
181       - name: Run tests without legacy protocol
182         run: sudo -u build CI=1 sh .ci/test/run.sh nolegacy
183         if: always()
184
185       - name: Run tests with libgcrypt
186         run: sudo -u build CI=1 sh .ci/test/run.sh gcrypt
187         if: always()
188
189       - name: Upload test results
190         uses: actions/upload-artifact@v2
191         with:
192           name: tests_${{ env.ARTIFACT }}
193           path: /tmp/logs/tests.*.tar.gz
194         if: always()
195
196       - name: Build package
197         run: sh .ci/package/build.sh
198         if: github.ref == 'refs/heads/1.1' || startsWith(github.ref, 'refs/tags/release-')
199         continue-on-error: true
200
201       - name: Upload package
202         uses: actions/upload-artifact@v2
203         with:
204           name: pkg-${{ env.ARTIFACT }}
205           path: |
206             *.deb
207             ~/rpmbuild/RPMS/*/*.rpm
208         continue-on-error: true
209
210   pkg-publish:
211     if: always() && (github.ref == 'refs/heads/1.1' || startsWith(github.ref, 'refs/tags/release-'))
212     runs-on: ubuntu-latest
213     continue-on-error: true
214     needs:
215       - linux
216       - mingw
217
218     steps:
219       - name: Create artifact directory
220         run: mkdir -p /tmp/artifacts
221
222       - name: Download packages
223         uses: actions/download-artifact@v2
224         with:
225           path: /tmp/artifacts
226
227       - name: Publish packages (dev)
228         uses: marvinpinto/action-automatic-releases@latest
229         with:
230           repo_token: ${{ secrets.GITHUB_TOKEN }}
231           automatic_release_tag: latest
232           title: Development release
233           prerelease: true
234           files: /tmp/artifacts/**/*.(deb|rpm|exe)
235         if: startsWith(github.ref, 'refs/heads/')
236
237       - name: Publish packages (release)
238         uses: softprops/action-gh-release@v1
239         with:
240           files: |
241             /tmp/artifacts/**/*.deb
242             /tmp/artifacts/**/*.rpm
243             /tmp/artifacts/**/*.exe
244         if: startsWith(github.ref, 'refs/tags/')
245
246   macos:
247     runs-on: macos-latest
248     timeout-minutes: 20
249
250     steps:
251       - name: Checkout code
252         uses: actions/checkout@v1
253
254       - name: Install build deps
255         run: sh .ci/deps.sh
256
257       - name: Run tests with default settings
258         run: sh .ci/test/run.sh default
259
260       - name: Run tests without legacy protocol
261         run: sh .ci/test/run.sh nolegacy
262         if: always()
263
264       - name: Run tests with libgcrypt
265         run: sh .ci/test/run.sh gcrypt
266         if: always()
267
268       - name: Upload test results
269         uses: actions/upload-artifact@v2
270         with:
271           name: tests_macos
272           path: /tmp/logs/tests.*.tar.gz
273         if: always()
274
275   mingw:
276     runs-on: windows-latest
277     timeout-minutes: 30
278
279     steps:
280       - name: Install msys2
281         uses: msys2/setup-msys2@v2
282         with:
283           update: true
284           # https://packages.msys2.org/package/
285           install: >-
286             base-devel
287             mingw-w64-x86_64-meson
288             mingw-w64-x86_64-pkgconf
289             mingw-w64-x86_64-gcc
290             mingw-w64-x86_64-openssl
291             mingw-w64-x86_64-libgcrypt
292             mingw-w64-x86_64-zlib
293             mingw-w64-x86_64-lzo2
294             mingw-w64-x86_64-lz4
295             mingw-w64-x86_64-ncurses
296             mingw-w64-x86_64-miniupnpc
297             mingw-w64-x86_64-nsis
298             git
299             openbsd-netcat
300             procps
301
302       - name: Checkout code
303         uses: actions/checkout@v1
304
305       - name: Run tests with default settings
306         shell: msys2 {0}
307         run: sh .ci/test/run.sh default
308
309       - name: Create installer
310         shell: msys2 {0}
311         run: sh .ci/package/build.sh
312         if: github.ref == 'refs/heads/1.1' || startsWith(github.ref, 'refs/tags/release-')
313         continue-on-error: true
314
315       - name: Upload package
316         uses: actions/upload-artifact@v2
317         with:
318           name: pkg-windows
319           path: .ci/package/win/tinc-*.exe
320         continue-on-error: true
321
322       - name: Run tests without legacy protocol
323         shell: msys2 {0}
324         run: sh .ci/test/run.sh nolegacy
325         if: always()
326
327       - name: Run tests with libgcrypt
328         shell: msys2 {0}
329         run: sh .ci/test/run.sh gcrypt
330         if: always()
331
332       - name: Upload test results
333         uses: actions/upload-artifact@v2
334         with:
335           name: tests_windows
336           path: /tmp/logs/tests.*.tar.gz
337         if: always()
338
339   msvc:
340     runs-on: windows-latest
341     timeout-minutes: 30
342
343     strategy:
344       fail-fast: false
345       matrix:
346         target:
347           - { build: amd64, host: amd64, test: test }
348           - { build: amd64, host: x86, test: test }
349           - { build: amd64, host: arm64, test: notest }
350
351     env:
352       HOST_ARCH: ${{ matrix.target.host }}
353       BUILD_ARCH: ${{ matrix.target.build }}
354
355     steps:
356       - name: Install meson
357         run: pip3 install meson
358
359       - name: Checkout code
360         uses: actions/checkout@v1
361
362       - name: Activate dev environment
363         uses: ilammy/msvc-dev-cmd@v1
364         with:
365           arch: ${{ matrix.target.build == matrix.target.host && matrix.target.host || format('{0}_{1}', matrix.target.build, matrix.target.host) }}
366
367       - name: Build (nolegacy)
368         run: .ci\windows\build.cmd nolegacy
369
370       - name: Test (nolegacy)
371         run: .ci\windows\test.cmd nolegacy
372         if: always() && matrix.target.test == 'test'
373
374       - name: Build (OpenSSL)
375         run: .ci\windows\build.cmd openssl
376         if: always()
377
378       - name: Test (OpenSSL)
379         run: .ci\windows\test.cmd openssl
380         if: always() && matrix.target.test == 'test'