CI: run all test flavors even on failure
[tinc] / .github / workflows / test.yml
index 171329c..cb72c66 100644 (file)
@@ -10,16 +10,17 @@ on:
 jobs:
   cross:
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
         arch:
           - armhf
-          - mips
+          - mipsel
+          - mingw
 
     container:
-      image: debian:stable
+      image: debian:bullseye
       options: --privileged
 
     steps:
@@ -30,26 +31,29 @@ jobs:
         run: HOST=${{ matrix.arch }} sh .ci/deps.sh
 
       - name: Prepare the system
-        run: |
-          sh .ci/test/prepare.sh
-          rm -f /dev/net/tun
+        run: HOST=${{ matrix.arch }} sh .ci/test/prepare.sh
 
       - name: Run tests with default settings
         run: sudo -u build CI=1 HOST=${{ matrix.arch }} sh .ci/test/run.sh default
 
       - name: Run tests without legacy protocol
         run: sudo -u build CI=1 HOST=${{ matrix.arch }} sh .ci/test/run.sh nolegacy
+        if: always()
+
+      - name: Run tests with libgcrypt
+        run: sudo -u build CI=1 HOST=${{ matrix.arch }} sh .ci/test/run.sh gcrypt
+        if: always()
 
       - name: Upload test results
         uses: actions/upload-artifact@v2
         with:
-          name: tests_cross_${{ env.ARTIFACT }}
+          name: tests_cross_${{ matrix.arch }}
           path: /tmp/logs/tests.*.tar.gz
         if: always()
 
   static-analysis:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 30
     steps:
       - name: Checkout code
         uses: actions/checkout@v1
@@ -62,55 +66,42 @@ jobs:
           tar -C ~ --strip-components=1 --wildcards -xf ./shellcheck-*.tar.xz 'shellcheck-*/shellcheck'
           curl -o ~/shfmt -L "https://github.com/mvdan/sh/releases/download/v$SHFMT/shfmt_v${SHFMT}_linux_amd64"
           chmod 755 ~/shfmt ~/shellcheck
-          pip3 install --user compiledb
+          python3 -m venv /tmp/venv
+          . /tmp/venv/bin/activate
+          pip3 install black pylint mypy
         env:
           CLANG: 11
           SHELLCHECK: 0.7.2
           SHFMT: 3.3.0
 
       - name: Install deps
-        run: sudo sh .ci/deps.sh
+        run: sudo SKIP_OPENSSL3=1 sh .ci/deps.sh
 
-      - name: Configure and compile
+      - name: Lint/typecheck/check formatting on C/shell/Python code
         run: |
-          autoreconf -fsi
-          ./configure $(sh .ci/conf.sh)
+          . /tmp/venv/bin/activate
+          PATH=$PATH:$HOME ./lint.py
+        if: always()
 
       - name: Run clang-tidy
         run: sh .ci/tidy/run.sh
         if: always()
 
-      - name: Check code formatting
-        run: "! astyle -r --options=.astylerc --dry-run --formatted '*.c' '*.h' | grep '^Formatted'"
-        if: always()
-
-      - name: Check scripts formatting
-        run: find -type f -regextype egrep -regex '.+\.(sh|sh\.in|test)$' -exec ~/shfmt -d -i 2 -s '{}' +
-        if: always()
-
-      - name: Run static analysis on scripts
-        run: find -type f -regextype egrep -regex '.+\.sh(\.in)?$' -exec shellcheck -x '{}' +
-        if: always()
-
-      - name: Run static analysis on tests
-        run: find -type f -name '*.test' -execdir shellcheck -x '{}' +
-        if: always()
-
       - name: Check warnings (gcc)
         run: bash .ci/warn/run.sh
         env:
-          CC: gcc
+          CC: gcc-10
         if: always()
 
       - name: Check warnings (clang)
         run: bash .ci/warn/run.sh
         env:
-          CC: clang
+          CC: clang-12
         if: always()
 
   sanitizer:
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
@@ -128,29 +119,27 @@ jobs:
       - name: Install deps
         run: sudo sh .ci/deps.sh
 
-      - name: Configure and compile
-        shell: bash
-        run: bash .ci/sanitizers/build.sh
-        env:
-          CC: clang-12
+      - name: Run tests with OpenSSL 3
+        run: bash .ci/sanitizers/run.sh openssl3
 
-      - name: Run tests
-        run: bash .ci/sanitizers/run.sh
+      - name: Sanitize tests with default settings
+        run: bash .ci/sanitizers/run.sh default
+        if: always()
 
-      - name: Archive test results
-        run: sudo tar -c -z -f test-results.tar.gz test/ sanitizer/
+      - name: Sanitize tests without legacy protocol
+        run: bash .ci/sanitizers/run.sh nolegacy
         if: always()
 
       - name: Upload test results
         uses: actions/upload-artifact@v2
         with:
           name: tests_sanitizer_${{ matrix.sanitizer }}
-          path: test-results.tar.gz
+          path: /tmp/logs/tests.*.tar.gz
         if: always()
 
   linux:
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
@@ -159,7 +148,8 @@ jobs:
           - centos:7 # aka RHEL 7
           - almalinux:8 # aka RHEL 8
           - fedora
-          - debian:stable
+          - debian:buster
+          - debian:bullseye
           - debian:testing
           - ubuntu # current LTS
           - ubuntu:rolling # latest
@@ -181,11 +171,20 @@ jobs:
       - name: Create a non-privileged user
         run: sh .ci/test/prepare.sh
 
+      - name: Run tests with OpenSSL 3
+        run: sudo -u build CI=1 sh .ci/test/run.sh openssl3
+
       - name: Run tests with default settings
         run: sudo -u build CI=1 sh .ci/test/run.sh default
+        if: always()
 
       - name: Run tests without legacy protocol
         run: sudo -u build CI=1 sh .ci/test/run.sh nolegacy
+        if: always()
+
+      - name: Run tests with libgcrypt
+        run: sudo -u build CI=1 sh .ci/test/run.sh gcrypt
+        if: always()
 
       - name: Upload test results
         uses: actions/upload-artifact@v2
@@ -197,6 +196,7 @@ jobs:
       - name: Build package
         run: sh .ci/package/build.sh
         if: github.ref == 'refs/heads/1.1' || startsWith(github.ref, 'refs/tags/release-')
+        continue-on-error: true
 
       - name: Upload package
         uses: actions/upload-artifact@v2
@@ -205,13 +205,15 @@ jobs:
           path: |
             *.deb
             ~/rpmbuild/RPMS/*/*.rpm
+        continue-on-error: true
 
   pkg-publish:
     if: always() && (github.ref == 'refs/heads/1.1' || startsWith(github.ref, 'refs/tags/release-'))
     runs-on: ubuntu-latest
+    continue-on-error: true
     needs:
       - linux
-      - windows
+      - mingw
 
     steps:
       - name: Create artifact directory
@@ -243,7 +245,7 @@ jobs:
 
   macos:
     runs-on: macos-latest
-    timeout-minutes: 15
+    timeout-minutes: 20
 
     steps:
       - name: Checkout code
@@ -255,13 +257,13 @@ jobs:
       - name: Run tests with default settings
         run: sh .ci/test/run.sh default
 
-      - name: Run clang-tidy
-        run: |
-          export PATH="$PATH:$(brew --prefix llvm)/bin:$HOME/Library/Python/3.9/bin"
-          sh .ci/tidy/run.sh
-
       - name: Run tests without legacy protocol
         run: sh .ci/test/run.sh nolegacy
+        if: always()
+
+      - name: Run tests with libgcrypt
+        run: sh .ci/test/run.sh gcrypt
+        if: always()
 
       - name: Upload test results
         uses: actions/upload-artifact@v2
@@ -270,9 +272,9 @@ jobs:
           path: /tmp/logs/tests.*.tar.gz
         if: always()
 
-  windows:
+  mingw:
     runs-on: windows-latest
-    timeout-minutes: 20
+    timeout-minutes: 30
 
     steps:
       - name: Install msys2
@@ -282,8 +284,11 @@ jobs:
           # https://packages.msys2.org/package/
           install: >-
             base-devel
+            mingw-w64-x86_64-meson
+            mingw-w64-x86_64-pkgconf
             mingw-w64-x86_64-gcc
             mingw-w64-x86_64-openssl
+            mingw-w64-x86_64-libgcrypt
             mingw-w64-x86_64-zlib
             mingw-w64-x86_64-lzo2
             mingw-w64-x86_64-lz4
@@ -291,7 +296,7 @@ jobs:
             mingw-w64-x86_64-miniupnpc
             mingw-w64-x86_64-nsis
             git
-            netcat
+            openbsd-netcat
             procps
 
       - name: Checkout code
@@ -305,16 +310,24 @@ jobs:
         shell: msys2 {0}
         run: sh .ci/package/build.sh
         if: github.ref == 'refs/heads/1.1' || startsWith(github.ref, 'refs/tags/release-')
+        continue-on-error: true
 
       - name: Upload package
         uses: actions/upload-artifact@v2
         with:
           name: pkg-windows
           path: .ci/package/win/tinc-*.exe
+        continue-on-error: true
 
       - name: Run tests without legacy protocol
         shell: msys2 {0}
         run: sh .ci/test/run.sh nolegacy
+        if: always()
+
+      - name: Run tests with libgcrypt
+        shell: msys2 {0}
+        run: sh .ci/test/run.sh gcrypt
+        if: always()
 
       - name: Upload test results
         uses: actions/upload-artifact@v2
@@ -322,3 +335,46 @@ jobs:
           name: tests_windows
           path: /tmp/logs/tests.*.tar.gz
         if: always()
+
+  msvc:
+    runs-on: windows-latest
+    timeout-minutes: 30
+
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - { build: amd64, host: amd64, test: test }
+          - { build: amd64, host: x86, test: test }
+          - { build: amd64, host: arm64, test: notest }
+
+    env:
+      HOST_ARCH: ${{ matrix.target.host }}
+      BUILD_ARCH: ${{ matrix.target.build }}
+
+    steps:
+      - name: Install meson
+        run: pip3 install meson
+
+      - name: Checkout code
+        uses: actions/checkout@v1
+
+      - name: Activate dev environment
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: ${{ matrix.target.build == matrix.target.host && matrix.target.host || format('{0}_{1}', matrix.target.build, matrix.target.host) }}
+
+      - name: Build (nolegacy)
+        run: .ci\windows\build.cmd nolegacy
+
+      - name: Test (nolegacy)
+        run: .ci\windows\test.cmd nolegacy
+        if: always() && matrix.target.test == 'test'
+
+      - name: Build (OpenSSL)
+        run: .ci\windows\build.cmd openssl
+        if: always()
+
+      - name: Test (OpenSSL)
+        run: .ci\windows\test.cmd openssl
+        if: always() && matrix.target.test == 'test'