From 452738883d2b954ac623052d1ebe8ce91db6bfba Mon Sep 17 00:00:00 2001 From: Stuart Prescott Date: Sat, 15 Oct 2022 16:55:06 +1100 Subject: [PATCH] Rebuild github actions - reduce duplication - easier maintenance of test matrix - test more things --- .github/workflows/ci.yml | 482 +++++++++++++++++++++++++++++++ .github/workflows/docs.yml | 111 ------- .github/workflows/installers.yml | 202 ------------- .github/workflows/matrix.py | 89 ++++++ .github/workflows/release.yml | 255 ---------------- .github/workflows/test.yml | 131 --------- 6 files changed, 571 insertions(+), 699 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/installers.yml create mode 100644 .github/workflows/matrix.py delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..ad097b91aa --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,482 @@ +name: CI + +on: + [push, pull_request] + +defaults: + run: + shell: bash + +env: + RELEASE: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/test-v') }} + # The test-installer job can use an old existing image rather than the most + # recently built image; the 'needs' can be commented out to allow the + # test-installer job to run independently (and therefore faster) + INSTALLER_USE_OLD_IMAGE: false + INSTALLER_OLD_RUNID: 3204825457 + # The test-installer job can run with the pyinstaller debug bundle + INSTALLER_USE_DEBUG: false + +jobs: + matrix: + name: Generate test matrix + + #if: false # uncomment to prevent the rest of the workflow running + + runs-on: ubuntu-latest + + outputs: + matrix-json: ${{ steps.set-matrix.outputs.matrix }} + + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - uses: actions/checkout@v3 + - id: set-matrix + run: python .github/workflows/matrix.py >> $GITHUB_OUTPUT + + build-matrix: + needs: [matrix] + + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.matrix.outputs.matrix-json) }} + + name: "${{ matrix.job_name }}" + + runs-on: ${{ matrix.os }} + + steps: + - name: Configuration of this job + run: | + echo '${{ toJson(matrix) }}' + + - name: Obtain SasView source from git + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + **/ci.yml + **/requirements*.txt + + ### Installation of build-dependencies + + - name: Install X11 libraries (Linux) + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + sudo apt install libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 x11-utils xvfb + + - name: Install lxml and MPL (Windows + Linux) + if: ${{ !startsWith(matrix.os, 'macos') }} + run: | + python -m pip install matplotlib~=3.5.2 + python -m pip install lxml + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install wheel setuptools + + python -m pip install numpy scipy==1.7.3 pytest pytest-mock pytest_qt + python -m pip install tinycc h5py pyparsing html5lib reportlab==3.6.6 pybind11 appdirs + python -m pip install six numba mako ipython qtconsole xhtml2pdf pylint + python -m pip install qt5reactor periodictable uncertainties dominate importlib_resources + python -m pip install html2text + + - name: Install PyQt (Windows + Linux) + if: ${{ !startsWith(matrix.os, 'macos') }} + run: | + python -m pip install PyQt5 + + - name: Install lxml, matplotlib and PyQt (OSX) + if: ${{ startsWith(matrix.os, 'macos') }} + run: | + python -m pip install PyQt5==5.13 + python -m pip install matplotlib~=3.5.2 + python -m pip install --no-binary lxml lxml + + - name: Install pyopencl (Windows) + if: ${{ startsWith(matrix.os, 'windows') }} + run: | + python -m pip install pytools mako cffi + choco install opencl-intel-cpu-runtime + choco install innosetup + python -m pip install --only-binary=pyopencl --find-links http://www.silx.org/pub/wheelhouse/ --trusted-host www.silx.org pyopencl + + - name: Install pyopencl (Linux + macOS) + if: ${{ !startsWith(matrix.os, 'windows') }} + run: | + python -m pip install pyopencl + + - name: Fetch sources for sibling projects + run: | + git clone --depth=50 --branch=master /~https://github.com/SasView/sasdata.git ../sasdata + git clone --depth=50 --branch=master /~https://github.com/SasView/sasmodels.git ../sasmodels + git clone --depth=50 --branch=master /~https://github.com/bumps/bumps.git ../bumps + + - name: Build and install sasdata + run: | + cd ../sasdata + rm -rf build + rm -rf dist + python setup.py clean + python setup.py build + python -m pip install --no-deps . + + - name: Build and install sasmodels + run: | + cd ../sasmodels + rm -rf build + rm -rf dist + python setup.py clean + python setup.py build + python -m pip install --no-deps . + + - name: Build and install bumps + run: | + cd ../bumps + rm -rf build + rm -rf dist + python setup.py clean + python setup.py build + python -m pip install --no-deps . + + ### Document the build environment + + - name: Python package version list + run: | + python -m pip freeze + + ### Actual building of sasview + + - name: Build sasview + run: | + # SET SASVIEW GITHASH + githash=$( git rev-parse HEAD ) + sed -i.bak s/GIT_COMMIT/$githash/g src/sas/sasview/__init__.py + # BUILD SASVIEW + python setup.py clean + python setup.py build + python -m pip install --no-deps . + + ### Run tests (if enabled) + + - name: Test with pytest + if: ${{ matrix.tests }} + env: + PYOPENCL_COMPILER_OUTPUT: 1 + run: | + python -m pytest -v -s test + + - name: Test GUI (Linux) + if: ${{ matrix.tests && matrix.os == 'ubuntu-latest' }} + env: + PYOPENCL_COMPILER_OUTPUT: 1 + run: | + # Suppress SIGSEGV from the tests until they can be fixed + retval=0 + xvfb-run -a --server-args="-screen 0 1600x900x24" python -m pytest -rsx -v src/sas/qtgui/ || retval=$? + if [ $retval -eq 139 ]; then echo "WARNING: Python interpreter exited with Segmentation Fault. This normally indicates that Qt objects were not correctly deleted. This error is currently suppressed in SasView's test suite."; retval=0; fi + exit $retval + + ### Build documentation (if enabled) + + - name: Install doc-building dependencies + if: ${{ matrix.docs }} + run: | + python -m pip install sphinx + + - name: Build sasmodels and bumps docs + if: ${{ matrix.docs }} + run: | + make -C ../bumps/doc html || true + mkdir -p ~/.sasmodels/compiled_models + make -j4 -C ../sasmodels/doc html || true + + - name: Build sasview docs + if: ${{ matrix.docs }} + run: | + cd docs/sphinx-docs/ + python build_sphinx.py || true + + ### Build the installer (if enabled) + + - name: Install utilities to build installer + if: ${{ matrix.installer }} + run: | + python -m pip install pyinstaller + + - name: Build sasview with pyinstaller + if: ${{ matrix.installer }} + run: | + cd installers + rm -rf build/ dist/ + mkdir -p dist + pyinstaller sasview.spec + cd dist + tar zcf sasview-pyinstaller-dist.tar.gz sasview + + - name: Build sasview installer with INNO (Windows) + if: ${{ matrix.installer && startsWith(matrix.os, 'windows') }} + run: | + iscc installers/installer.iss + mv installers/Output/setupSasView.exe installers/dist + + - name: Build sasview installer dmg file (OSX) + if: ${{ matrix.installer && startsWith(matrix.os, 'macos') }} + run: | + cd installers/dist + hdiutil create SasView5.dmg -srcfolder SasView5.app -ov -format UDZO + + - name: Build sasview installer tarball (Linux) + if: ${{ matrix.installer && startsWith(matrix.os, 'ubuntu') }} + run: | + cd installers/dist + tar zcf sasview5.tar.gz sasview + + - name: Collect a debug tarball of the installer package + if: ${{ matrix.installer }} + uses: actions/upload-artifact@v3 + with: + name: Debug-SasView-Installer-${{ matrix.os }}-${{ matrix.python-version }} + path: | + installers/dist/sasview-pyinstaller-dist.tar.gz + if-no-files-found: ignore + + - name: Publish installer package + if: ${{ matrix.installer }} + uses: actions/upload-artifact@v3 + with: + name: SasView-Installer-${{ matrix.os }}-${{ matrix.python-version }} + path: | + installers/dist/setupSasView.exe + installers/dist/SasView5.dmg + installers/dist/sasview5.tar.gz + if-no-files-found: error + + - name: Sign executable and create dmg (OSX) + if: ${{ env.RELEASE == 'true' && matrix.installer && startsWith(matrix.os, 'macos') }} + env: + MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} + MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }} + run: | + echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12 + security create-keychain -p DloaAcYP build.keychain + security default-keychain -s build.keychain + security unlock-keychain -p DloaAcYP build.keychain + security import certificate.p12 -k build.keychain -P $MACOS_CERTIFICATE_PWD -T /usr/bin/codesign + security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k DloaAcYP build.keychain + + cd installers/dist + python ../../build_tools/code_sign_osx.py + codesign --verify --options=runtime --entitlements ../../build_tools/entitlements.plist --timestamp --deep --verbose=4 --force --sign "Developer ID Application: European Spallation Source Eric (W2AG9MPZ43)" SasView5.app + hdiutil create SasView5.dmg -srcfolder SasView5.app -ov -format UDZO + codesign -s "Developer ID Application: European Spallation Source Eric (W2AG9MPZ43)" SasView5.dmg + + # - name: Notarize Release Build (OSX) + # if: ${{ env.RELEASE == 'true' && matrix.installer && startsWith(matrix.os, 'macos') }} + # uses: devbotsxyz/xcode-notarize@v1 + # with: + # product-path: "installers/dist/SasView5.dmg" + # primary-bundle-id: "org.sasview.SasView5" + # appstore-connect-username: ${{ secrets.NOTARIZATION_USERNAME }} + # appstore-connect-password: ${{ secrets.NOTARIZATION_PASSWORD }} + + # - name: Staple Release Build (OSX) + # if: ${{ startsWith(matrix.os, 'macos') }} + # uses: devbotsxyz/xcode-staple@v1 + # with: + # product-path: "installers/dist/SasView5.dmg" + + - name: Declare env variables on push only + if: ${{ env.RELEASE == 'true' && matrix.installer && github.event_name == 'push' }} + shell: bash + run: echo "BRANCH_NAME=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + + - name: Declare env variables on pull_request only + if: ${{ env.RELEASE == 'true' && matrix.installer && github.event_name == 'pull_request' }} + shell: bash + run: echo "BRANCH_NAME=$GITHUB_HEAD_REF" >> $GITHUB_ENV + + - name: Upload installer to GitHub releases + if: ${{ env.RELEASE == 'true' && matrix.installer }} + uses: ncipollo/release-action@v1 + with: + draft: true + prerelease: true + allowUpdates: true + replacesArtifacts: true + token: ${{ secrets.GITHUB_TOKEN }} + artifacts: "installers/dist/SasView5.dmg, installers/Output/setupSasView.exe" + name: ${{ env.BRANCH_NAME }} + tag: ${{ env.BRANCH_NAME }} + + + test-installer: + needs: [ build-matrix ] + + strategy: + matrix: + os: [ windows-latest, ubuntu-18.04 ] + python-version: [ 3.8 ] + fail-fast: false + + name: Test installer + + runs-on: ${{ matrix.os }} + + steps: + - name: Set variables (Windows) + if: ${{ startsWith(matrix.os, 'windows') }} + run: | + echo "INSTALL_PATH=$PWD/sasview/" >> $GITHUB_ENV + echo "RUN_SASVIEW=$PWD/sasview/sasview.exe" >> $GITHUB_ENV + + - name: Set variables (Linux) + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + echo "INSTALL_PATH=$PWD/sasview/" >> $GITHUB_ENV + echo "RUN_SASVIEW=$PWD/sasview/sasview" >> $GITHUB_ENV + + - name: Set variables (Windows Powershell) + if: ${{ startsWith(matrix.os, 'windows') }} + shell: powershell + # unfortunately we end up tripping over ourselves with paths + # later on and need a PS version of the install path + run: | + "INSTALL_WIN_PATH=$(Get-Location)/sasview/" >> $env:GITHUB_ENV + + ### Obtain the installer, either from the previous step + ### or from a previous run of the workflow + + - name: Retrieve installer (workflow image) + if: ${{ env.INSTALLER_USE_OLD_IMAGE != 'true' }} + uses: actions/download-artifact@v3 + id: download + with: + name: SasView-Installer-${{ matrix.os }}-${{ matrix.python-version }} + + - name: Set variables for download (workflow image) + if: ${{ env.INSTALLER_USE_OLD_IMAGE != 'true' }} + run: | + echo "DL_PATH=${{steps.download.outputs.download-path}}" >> $GITHUB_ENV + + - name: Set variables for download (workflow image) (Windows) + if: ${{ env.INSTALLER_USE_OLD_IMAGE != 'true' && startsWith(matrix.os, 'windows') }} + shell: powershell + run: | + "DL_WIN_PATH=${{steps.download.outputs.download-path}}" >> $env:GITHUB_ENV + + - name: Retrieve installer (old image) + if: ${{ env.INSTALLER_USE_OLD_IMAGE == 'true' }} + id: download-old-installer + uses: dawidd6/action-download-artifact@v2 + with: + # Select a specific installer image based on the GitHub Actions run id + run_id: ${{ env.INSTALLER_OLD_RUNID }} + name: SasView-Installer-${{ matrix.os }}-${{ matrix.python-version }} + path: downloads + if_no_artifact_found: fail + + - name: Set variables for download (old image) + if: ${{ env.INSTALLER_USE_OLD_IMAGE == 'true' }} + run: | + echo "DL_PATH=$PWD/downloads" >> $GITHUB_ENV + + - name: Set variables for download (old image) (Windows) + if: ${{ env.INSTALLER_USE_OLD_IMAGE == 'true' && startsWith(matrix.os, 'windows') }} + shell: powershell + run: | + "DL_WIN_PATH=$(Get-Location)/downloads" >> $env:GITHUB_ENV + + - name: Check downloads + run: | + echo "Sasview downloaded to: $DL_PATH" + find "$DL_PATH" + + ### Run the installer and check that the installation looks OK + + - name: Run installer (Windows) + if: ${{ startsWith(matrix.os, 'windows') }} + # The installer spawns a new process and exits immediately, meaning + # that the shell thinks it has finishes, tries to exit to move onto + # the next step in the job, and kills the child process that is + # trying to do the installation. + # Wrapping the installer in "Start-Process ... -Wait" means that + # this step doesn't exit until the installer has finished. + shell: powershell + run: | + Start-Process -FilePath "${{ env.DL_WIN_PATH }}/setupSasView.exe" -ArgumentList "/SP- /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /ALLUSERS /SKIPLICENSE=true /LANG=english /NOICONS /TASKS='' /LOG=${{ env.DL_WIN_PATH }}/install.log /DIR=${{ env.INSTALL_WIN_PATH }}" -Wait + + - name: Check installation log (Windows) + if: ${{ startsWith(matrix.os, 'windows') }} + run: | + echo "Contents of installation log '$DL_PATH/install.log':" + cat $DL_PATH/install.log + + - name: Run installer (Linux) + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + # $INSTALL_PATH is already the base directory + tar xf "$DL_PATH/sasview5.tar.gz" + + - name: Check installation files + run: | + echo "File listing of SasView installed in '$INSTALL_PATH'" + find "$INSTALL_PATH" + + - name: Try running the installation (Windows) + if: ${{ startsWith(matrix.os, 'windows') }} + # If sasview has crashed on its own, then this should return an error + run: | + $RUN_SASVIEW & + sleep 60s + kill %1 + + - name: Try running the installation (Linux) + if: ${{ startsWith(matrix.os, 'ubuntu' )}} + # If sasview has crashed on its own, then this should return an error + run: | + xvfb-run -a --server-args="-screen 0 1024x768x24" $RUN_SASVIEW & + sleep 60s + kill %1 + + ### Optionally attempt to work with the debug tarball from pyinstaller + + - name: Retrieve installer debug tarball + if: ${{ env.INSTALLER_USE_DEBUG == 'true' }} + id: download-old-debug + uses: dawidd6/action-download-artifact@v2 + with: + # Select a specific installer image based on the GitHub Actions run id + run_id: ${{ env.INSTALLER_OLD_RUNID }} + name: Debug-SasView-Installer-${{ matrix.os }}-${{ matrix.python-version }} + path: downloads + if_no_artifact_found: fail + + - name: Extract debug tarball + if: ${{ env.INSTALLER_USE_DEBUG == 'true' }} + run: | + mkdir unpack + cd unpack + tar zxf ../downloads/sasview-pyinstaller-dist.tar.gz + + - name: Check debug installation files + if: ${{ env.INSTALLER_USE_DEBUG == 'true' && startsWith(matrix.os, 'windows') }} + run: | + echo "File listing of SasView install bundle unpacked in 'unpack'" + find unpack + + - name: Try running debug unpack + if: ${{ env.INSTALLER_USE_DEBUG == 'true' && startsWith(matrix.os, 'windows') }} + run: | + cd unpack/sasview + ./sasview.exe diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index ba524e15bc..0000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: Docs - -on: - [push, pull_request] - -defaults: - run: - shell: bash - -jobs: - doc-build: - - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - python-version: [3.9] - - steps: - - - name: Obtain SasView source from git - uses: actions/checkout@v1 - - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - ### Caching of pip downloads and local wheel builds - - - name: Obtain pip cache - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ matrix.OS }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/test.yml') }} - restore-keys: | - ${{ matrix.OS }}-pip-${{ matrix.python-version }}- - ${{ matrix.OS }}-pip- - - ### Installation of build-dependencies - - - name: Install packaged dependencies - run: | - sudo apt-get update - sudo apt-get install opencl-headers ocl-icd-opencl-dev libpocl2 xvfb pyqt5-dev-tools - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install wheel setuptools - python -m pip install numpy scipy matplotlib docutils "pytest<6" sphinx unittest-xml-reporting tinycc lxml h5py sphinx pyparsing html5lib reportlab pybind11 appdirs six numba mako ipython qtconsole xhtml2pdf unittest-xml-reporting pylint qt5reactor periodictable PyQt5 html2text importlib_resources - - - name: Install pyopencl - run: | - python -m pip install pyopencl - - - name: Fetch sources for sibling projects - run: | - git clone --depth=50 --branch=master /~https://github.com/SasView/sasdata.git ../sasdata - git clone --depth=50 --branch=master /~https://github.com/SasView/sasmodels.git ../sasmodels - git clone --depth=50 --branch=master /~https://github.com/bumps/bumps.git ../bumps - - - name: Build and install sasdata - run: | - cd ../sasdata - rm -rf build - rm -rf dist - python setup.py clean - python setup.py build - python -m pip install . - - - name: Build and install sasmodels - run: | - cd ../sasmodels - rm -rf build - rm -rf dist - python setup.py clean - python setup.py build - python -m pip install . - - - name: Build and install bumps - run: | - cd ../bumps - rm -rf build - rm -rf dist - python setup.py clean - python setup.py build - python -m pip install . - - ### Actual building/testing of sasview - - - name: Build sasview - run: | - # SET SASVIEW GITHASH - githash=$( git rev-parse HEAD ) - sed -i.bak s/GIT_COMMIT/$githash/g src/sas/system/version.py - # BUILD SASVIEW - python setup.py clean - python setup.py build - python -m pip install . - - - name: Build sasmodels and bumps docs - run: | - make -C ../bumps/doc html || true - mkdir -p ~/.sasmodels/compiled_models - make -j4 -C ../sasmodels/doc html || true - - - name: Build sasview docs - run: | - cd docs/sphinx-docs/ - xvfb-run -a --server-args="-screen 0 1024x768x24" python build_sphinx.py || true diff --git a/.github/workflows/installers.yml b/.github/workflows/installers.yml deleted file mode 100644 index 999b818e8d..0000000000 --- a/.github/workflows/installers.yml +++ /dev/null @@ -1,202 +0,0 @@ -name: Installer - -#Build all branches except release -on: - push: - branches-ignore: - - release* - pull_request: - branches-ignore: - - release* - -defaults: - run: - shell: bash - -jobs: - installer: - - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-latest, windows-latest] - python-version: [3.8] - - steps: - - name: Obtain SasView source from git - uses: actions/checkout@v1 - - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - ### Caching of pip downloads and local wheel builds - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Obtain pip cache (macOS) - uses: actions/cache@v2 - if: startsWith(runner.os, 'macOS') - with: - path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/test.yml') }} - restore-keys: | - ${{ runner.os }}-pip-${{ matrix.python-version }}- - ${{ runner.os }}-pip- - - - name: Obtain pip cache (Windows) - uses: actions/cache@v2 - if: startsWith(runner.os, 'Windows') - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/test.yml') }} - restore-keys: | - ${{ runner.os }}-pip-${{ matrix.python-version }}- - ${{ runner.os }}-pip- - - ### Installation of build-dependencies - - name: Install lxml and MPL (Linux + Win) - if: ${{ matrix.os != 'macos-latest' }} - run: | - python -m pip install matplotlib~=3.5.2 - python -m pip install lxml - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install wheel setuptools - - python -m pip install numpy scipy==1.7.3 docutils "pytest<6" sphinx unittest-xml-reporting - python -m pip install tinycc h5py sphinx pyparsing html5lib reportlab==3.6.6 pybind11 appdirs - python -m pip install six numba mako ipython qtconsole xhtml2pdf unittest-xml-reporting pylint - python -m pip install qt5reactor periodictable uncertainties dominate importlib_resources - python -m pip install html2text - - - - name: Install PyQt (Windows + Linux) - if: ${{ matrix.os != 'macos-latest' }} - run: | - python -m pip install PyQt5 - - - name: Install lxml, matplotlib and PyQt (OSX) - if: ${{ matrix.os == 'macos-latest' }} - run: | - python -m pip install PyQt5==5.13 - python -m pip install matplotlib~=3.5.2 - python -m pip install --no-binary lxml lxml - - - name: Install pyopencl (Windows) - if: ${{ matrix.os == 'windows-latest' }} - run: | - python -m pip install pytools mako cffi - choco install opencl-intel-cpu-runtime - choco install innosetup - python -m pip install --only-binary=pyopencl --find-links http://www.silx.org/pub/wheelhouse/ --trusted-host www.silx.org pyopencl - - - name: Install pyopencl (Linux + macOS) - if: ${{ matrix.os != 'windows-latest' }} - run: | - python -m pip install pyopencl - - - name: Install utilities to build installer - run: | - python -m pip install pyinstaller - - - name: Fetch sources for sibling projects - run: | - git clone --depth=50 --branch=master /~https://github.com/SasView/sasdata.git ../sasdata - git clone --depth=50 --branch=master /~https://github.com/SasView/sasmodels.git ../sasmodels - git clone --depth=50 --branch=master /~https://github.com/bumps/bumps.git ../bumps - - - name: Build and install sasdata - run: | - cd ../sasdata - rm -rf build - rm -rf dist - python setup.py clean - python setup.py build - python -m pip install . - - - name: Build and install sasmodels - run: | - cd ../sasmodels - rm -rf build - rm -rf dist - python setup.py clean - python setup.py build - python -m pip install . - - - name: Build and install bumps - run: | - cd ../bumps - rm -rf build - rm -rf dist - python setup.py clean - python setup.py build - python -m pip install . - - ### Document the build environment - - - name: Python package version list - run: | - python -m pip freeze - - ### Actual building/testing of sasview - - - name: Build sasview - run: | - # SET SASVIEW GITHASH - githash=$( git rev-parse HEAD ) - sed -i.bak s/GIT_COMMIT/$githash/g src/sas/system/version.py - # BUILD SASVIEW - python setup.py clean - python setup.py build - python -m pip install . - - - name: Build sasmodels and bumps docs - run: | - make -C ../bumps/doc html || true - mkdir -p ~/.sasmodels/compiled_models - make -j4 -C ../sasmodels/doc html || true - - - name: Build sasview docs - run: | - cd docs/sphinx-docs/ - python build_sphinx.py || true - - ### Build the installer - - - name: Build sasview with pyinstaller - run: | - cd installers - pyinstaller sasview.spec - - - name: Build sasview with INNO setup package (Windows) - if: ${{ matrix.os == 'windows-latest' }} - run: | - iscc installers/installer.iss - - - name: Publish installer package (Windows) - if: ${{ matrix.os == 'windows-latest' }} - uses: actions/upload-artifact@v2 - with: - name: SasView-Installer-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.sha }} - path: installers/Output/setupSasView.exe - if-no-files-found: error - - - name: Create dmg file (OSX) - if: ${{ matrix.os == 'macos-latest' }} - run: | - cd installers/dist - hdiutil create SasView5.dmg -srcfolder SasView5.app -ov -format UDZO - - - name: Publish installer package (OSX) - if: ${{ matrix.os == 'macos-latest' }} - uses: actions/upload-artifact@v2 - with: - name: SasView-Installer-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.sha }} - path: installers/dist/SasView5.dmg - if-no-files-found: error diff --git a/.github/workflows/matrix.py b/.github/workflows/matrix.py new file mode 100644 index 0000000000..2405567229 --- /dev/null +++ b/.github/workflows/matrix.py @@ -0,0 +1,89 @@ +# Generate the matrix of jobs to run tests, build docs, build installers +# +# run `python .github/workflows/matrix.py --json` to see the output + +import json +import sys + +pretty = "--json" in sys.argv + + +jobs = [] + +# List of OS images to use for release builds +# Notes on OS selection: +# - macos +# - windows +# - linux: the release that is used defines the oldest distro on which +# the Linux pyinstaller output will run as libc is still +# dynamically linked by pyinstaller. +# https://pyinstaller.readthedocs.io/en/stable/usage.html#making-gnu-linux-apps-forward-compatible +os_release_list = [ + 'ubuntu-18.04', + 'windows-latest', + 'macos-latest', +] + +# List of OS images to use for release tests +os_list = os_release_list + [ + 'ubuntu-latest', +] + +# List of python versions to use for release builds +python_release_list = [ + '3.8', +] + +# List of python versions to use for tests +python_list = python_release_list + [ + '3.9', + '3.10' +] + + +def truthy(val): + return val if val is not None else 1 + +def entry(job_name="Job", os=None, pyver=None, tests=None, docs=None, installer=None): + return { + 'job_name': job_name, + 'os': os or os_release_list[0], + 'python-version': pyver or python_release_list[0], + 'tests': truthy(tests), + 'docs': truthy(docs), + 'installer': truthy(installer), + } + +# Construct the list of jobs for build+test +for os in os_list: + for pyver in python_list: + jobs.append( + entry( + job_name = f"Test ({os}, {pyver})", + os = os, + pyver = pyver, + tests = 1, + docs = 0, + installer = 0, + ) + ) + +# Construct the list of jobs for the installer +# Building the installer needs the docs but not the tests +for os in os_release_list: + for pyver in python_release_list: + jobs.append( + entry( + job_name = f"Installer ({os}, {pyver})", + os = os, + pyver = pyver, + tests = 0, + docs = 1, + installer = 1, + ) + ) + +if not pretty: + print("matrix=", end='') + +print(json.dumps(jobs, indent=4 if pretty else None)) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index ea151eadb9..0000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,255 +0,0 @@ -name: Create release - -on: - push: - branches: - - release* - pull_request: - branches: - - release* - -defaults: - run: - shell: bash - -jobs: - installer: - - timeout-minutes: 40 - - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-latest, windows-latest] - python-version: [3.8] - - steps: - - name: Obtain SasView source from git - uses: actions/checkout@v1 - - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - ### Caching of pip downloads and local wheel builds - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Obtain pip cache (macOS) - uses: actions/cache@v2 - if: startsWith(runner.os, 'macOS') - with: - path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/test.yml') }} - restore-keys: | - ${{ runner.os }}-pip-${{ matrix.python-version }}- - ${{ runner.os }}-pip- - - - name: Obtain pip cache (Windows) - uses: actions/cache@v2 - if: startsWith(runner.os, 'Windows') - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/test.yml') }} - restore-keys: | - ${{ runner.os }}-pip-${{ matrix.python-version }}- - ${{ runner.os }}-pip- - - ### Installation of build-dependencies - - - name: Install lxml and MPL (Linux + Win) - if: ${{ matrix.os != 'macos-latest' }} - run: | - python -m pip install matplotlib~=3.5.2 - python -m pip install lxml - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install wheel setuptools - - python -m pip install numpy scipy==1.7.3 docutils "pytest<6" sphinx unittest-xml-reporting - python -m pip install tinycc h5py sphinx pyparsing html5lib reportlab==3.6.6 pybind11 appdirs - python -m pip install six numba mako ipython qtconsole xhtml2pdf unittest-xml-reporting pylint - python -m pip install qt5reactor periodictable uncertainties dominate importlib_resources - python -m pip install html2text - - - - name: Install PyQt (Windows + Linux) - if: ${{ matrix.os != 'macos-latest' }} - run: | - python -m pip install PyQt5 - - - name: Install lxml, matplotlib and PyQt (OSX) - if: ${{ matrix.os == 'macos-latest' }} - run: | - python -m pip install PyQt5==5.13 - python -m pip install matplotlib~=3.5.2 - python -m pip install --no-binary lxml lxml - - name: Install pyopencl (Windows) - if: ${{ matrix.os == 'windows-latest' }} - run: | - python -m pip install pytools mako cffi - choco install opencl-intel-cpu-runtime - choco install innosetup - python -m pip install --only-binary=pyopencl --find-links http://www.silx.org/pub/wheelhouse/ --trusted-host www.silx.org pyopencl - - - name: Install pyopencl (Linux + macOS) - if: ${{ matrix.os != 'windows-latest' }} - run: | - python -m pip install pyopencl - - - name: Install utilities to build installer - run: | - python -m pip install pyinstaller - - ### TODO: Bumps, sasdata and sasmodels branches needs to be setup manually until we find better mechanism - - name: Fetch sources for sibling projects - run: | - git clone --depth=50 --branch=master /~https://github.com/SasView/sasdata.git ../sasdata - git clone --depth=1 --branch=v1.0.6 /~https://github.com/SasView/sasmodels.git ../sasmodels - git clone --depth=1 --branch=v0.9.0 /~https://github.com/bumps/bumps.git ../bumps - - - name: Build and install sasdata - run: | - cd ../sasdata - rm -rf build - rm -rf dist - python setup.py clean - python setup.py build - python -m pip install . - - - name: Build and install sasmodels - run: | - cd ../sasmodels - rm -rf build - rm -rf dist - python setup.py clean - python setup.py build - python -m pip install . - - - name: Build and install bumps - run: | - cd ../bumps - rm -rf build - rm -rf dist - python setup.py clean - python setup.py build - python -m pip install . - - ### Actual building/testing of sasview - - - name: Build sasview - run: | - # SET SASVIEW GITHASH - githash=$( git rev-parse HEAD ) - sed -i.bak s/GIT_COMMIT/$githash/g src/sas/system/version.py - # BUILD SASVIEW - python setup.py clean - python setup.py build - python -m pip install . - - - name: Build sasmodels and bumps docs - run: | - make -C ../bumps/doc html || true - mkdir -p ~/.sasmodels/compiled_models - make -j4 -C ../sasmodels/doc html || true - - - name: Build sasview docs - run: | - cd docs/sphinx-docs/ - python build_sphinx.py || true - - - ### Build the installer - - - name: Build sasview with pyinstaller - run: | - cd installers - pyinstaller sasview.spec - - - name: Build sasview with INNO setup package (Windows) - if: ${{ matrix.os == 'windows-latest' }} - run: | - iscc installers/installer.iss - - - name: Publish installer package (Windows) - if: ${{ matrix.os == 'windows-latest' }} - uses: actions/upload-artifact@v2 - with: - name: SasView-Installer-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.sha }} - path: installers/Output/setupSasView.exe - if-no-files-found: error - - - name: Create dmg file (OSX) - if: ${{ matrix.os == 'macos-latest' }} - run: | - cd installers/dist - hdiutil create SasView5.dmg -srcfolder SasView5.app -ov -format UDZO - - - name: Sign executable and create dmg (OSX) - if: ${{ matrix.os == 'macos-latest' }} - env: - MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} - MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }} - run: | - echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12 - security create-keychain -p DloaAcYP build.keychain - security default-keychain -s build.keychain - security unlock-keychain -p DloaAcYP build.keychain - security import certificate.p12 -k build.keychain -P $MACOS_CERTIFICATE_PWD -T /usr/bin/codesign - security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k DloaAcYP build.keychain - - cd installers/dist - python ../../build_tools/code_sign_osx.py - codesign --verify --options=runtime --entitlements ../../build_tools/entitlements.plist --timestamp --deep --verbose=4 --force --sign "Developer ID Application: European Spallation Source Eric (W2AG9MPZ43)" SasView5.app - hdiutil create SasView5.dmg -srcfolder SasView5.app -ov -format UDZO - codesign -s "Developer ID Application: European Spallation Source Eric (W2AG9MPZ43)" SasView5.dmg - - - name: Publish installer package (OSX) - if: ${{ matrix.os == 'macos-latest' }} - uses: actions/upload-artifact@v2 - with: - name: SasView-Installer-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.sha }} - path: installers/dist/SasView5.dmg - if-no-files-found: error - -# - name: Notarize Release Build (OSX) -# if: ${{ matrix.os == 'macos-latest' }} -# uses: devbotsxyz/xcode-notarize@v1 -# with: -# product-path: "installers/dist/SasView5.dmg" -# primary-bundle-id: "org.sasview.SasView5" -# appstore-connect-username: ${{ secrets.NOTARIZATION_USERNAME }} -# appstore-connect-password: ${{ secrets.NOTARIZATION_PASSWORD }} - -# - name: Staple Release Build (OSX) -# if: ${{ matrix.os == 'macos-latest' }} -# uses: devbotsxyz/xcode-staple@v1 -# with: -# product-path: "installers/dist/SasView5.dmg" - - - name: Declare env variables on push only - if: github.event_name == 'push' - shell: bash - run: echo "BRANCH_NAME=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - - name: Declare env variables on pull_request only - if: github.event_name == 'pull_request' - shell: bash - run: echo "BRANCH_NAME=$GITHUB_HEAD_REF" >> $GITHUB_ENV - - - name: Upload installer to GitHub releases - uses: ncipollo/release-action@v1 - with: - draft: true - prerelease: true - allowUpdates: true - replacesArtifacts: true - token: ${{ secrets.GITHUB_TOKEN }} - artifacts: "installers/dist/SasView5.dmg, installers/Output/setupSasView.exe" - name: ${{ env.BRANCH_NAME }} - tag: ${{ env.BRANCH_NAME }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index b7aabbb0a3..0000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,131 +0,0 @@ -name: Tests - -on: - [push, pull_request] - -defaults: - run: - shell: bash - -jobs: - unit-test: - - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - python-version: [3.7, 3.8, 3.9] - fail-fast: false - - steps: - - - name: Obtain SasView source from git - uses: actions/checkout@v1 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - ### Caching of pip downloads and local wheel builds - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Obtain pip cache (Linux) - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/test.yml') }} - restore-keys: | - ${{ runner.os }}-pip-${{ matrix.python-version }}- - ${{ runner.os }}-pip- - - ### Installation of build-dependencies - - - name: Install packaged dependencies (Linux) - if: ${{ matrix.os == 'ubuntu-latest' }} - run: | - sudo apt-get update - sudo apt-get install opencl-headers ocl-icd-opencl-dev libpocl2 xvfb pyqt5-dev-tools - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install wheel setuptools - python -m pip install numpy scipy matplotlib docutils pytest pytest_qt pytest-mock sphinx unittest-xml-reporting tinycc lxml h5py sphinx pyparsing html5lib reportlab pybind11 appdirs six numba mako ipython qtconsole xhtml2pdf unittest-xml-reporting pylint qt5reactor periodictable PyQt5 html2text importlib_resources uncertainties dominate html2text - - - name: Install pyopencl (Linux + macOS) - if: ${{ matrix.os != 'windows-latest' }} - run: | - python -m pip install pyopencl - - - name: Install pyopencl (Windows) - if: ${{ matrix.os == 'windows-latest' }} - run: | - python -m pip install pytools mako cffi - choco install opencl-intel-cpu-runtime - python -m pip install --only-binary=pyopencl --find-links http://www.silx.org/pub/wheelhouse/ --trusted-host www.silx.org pyopencl - - - name: Fetch sources for sibling projects - run: | - git clone --depth=50 --branch=master /~https://github.com/SasView/sasdata.git ../sasdata - git clone --depth=50 --branch=master /~https://github.com/SasView/sasmodels.git ../sasmodels - git clone --depth=50 --branch=master /~https://github.com/bumps/bumps.git ../bumps - - - name: Build and install sasdata - run: | - cd ../sasdata - rm -rf build - rm -rf dist - python setup.py clean - python setup.py build - python -m pip install . - - - name: Build and install sasmodels - run: | - cd ../sasmodels - rm -rf build - rm -rf dist - python setup.py clean - python setup.py build - python -m pip install . - - - name: Build and install bumps - run: | - cd ../bumps - rm -rf build - rm -rf dist - python setup.py clean - python setup.py build - python -m pip install . - - ### Actual building/testing of sasview - - - name: Build sasview - run: | - # SET SASVIEW GITHASH - githash=$( git rev-parse HEAD ) - sed -i.bak s/GIT_COMMIT/$githash/g src/sas/system/version.py - # BUILD SASVIEW - python setup.py clean - python setup.py build - python -m pip install . - - - name: Test with pytest - env: - PYOPENCL_COMPILER_OUTPUT: 1 - run: | - python -m pytest -v -s test - - - name: Test GUI (Linux) - if: ${{ matrix.os == 'ubuntu-latest' }} - env: - PYOPENCL_COMPILER_OUTPUT: 1 - run: | - # Suppress SIGSEGV from the tests until they can be fixed - retval=0 - xvfb-run -a --server-args="-screen 0 1024x768x24" python -m pytest -rsx -v src/sas/qtgui/ || retval=$? - if [ $retval -eq 139 ]; then echo "WARNING: Python interpreter exited with Segmentation Fault. This normally indicates that Qt objects were not correctly deleted. This error is currently suppressed in SasView's test suite."; retval=0; fi - exit $retval