From 6b0704f4fa51a915a777f093b24f4a064e69ca26 Mon Sep 17 00:00:00 2001 From: David Liptai <31463304+dliptai@users.noreply.github.com> Date: Fri, 6 Oct 2023 15:29:26 +1100 Subject: [PATCH] Release workflow (#68) * add release workflow * build hdf5 manually and link against that static lib * update cache key to reflect whether it contains HDF5 * build static libs on macos too and link to those * ensure GNU gcc is used, not clang * dont error if linking fails (this is to avoid gfrotran, which already exsists) * only need to link gcc and g++ * pin homebrew gcc-13 version * use read writeable MCFOST_INSTALL path for any user * fix gcc install * gfortran issue * single dependency action * do release as part of main workflow * xattr -dr com.apple.quarantine mcfost [skip ci] * link statically with libgfortran and libquadmath * command 'xattr -dr com.apple.quarantine mcfost' does nothing, needs to be done by user once file is downloaded * skip test step for releases. Master is protected so test should already have passed * use binary stored in artifact and upload to release upload for linux and macos, always using ifort build * only run release job on release * debug file permissions * fix bug caused by tar running as root * separate tar fix into its own step * remove debugging line * link statically with libgfortran and libquadmath * libstdc++ should come after libvoro++ when linking some linkers will only search for functions from left to right in the link line. This commit ensures the code compiles regardless of the linker behaviour see e.g.: https://www.linuxtopia.org/online_books/an_introduction_to_gcc/gccintro_18.html * fix incorrect artifact name * always run test, even for release * run mcfost -h to make sure it runs in a clean env * remove redundant release check * temporarily disable test * chmod +x mcfost * fix path * fix * just use gfortran for testing * ./ * use actual os for release runner * missing $ * correct filename * Revert "just use gfortran for testing" This reverts commit f406bf83e4cc01e6a30f26df30e704d79ce99815. * Revert "temporarily disable test" This reverts commit f6060c0aeccaba9589fe86d5f2a2176fda18aca8. * fix filenames * use march=native, unless it's a release * nicer name for build+test action * run CI for all PRs, not just on master --------- Co-authored-by: Conrad Chan Co-authored-by: Conrad Chan <8309215+conradtchan@users.noreply.github.com> --- .github/actions/dependencies/action.yml | 9 ++- .github/actions/release/action.yml | 17 +++++ .github/workflows/test-suite.yml | 57 ++++++++++++++-- src/Makefile | 91 +++++++++---------------- 4 files changed, 108 insertions(+), 66 deletions(-) create mode 100644 .github/actions/release/action.yml diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index a4e332ab3..a8dcbc998 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -25,7 +25,7 @@ runs: chmod +x l_fortran-compiler_p_2023.2.1.8_offline.sh ./l_fortran-compiler_p_2023.2.1.8_offline.sh -a --silent --eula accept echo "INTEL_PATH=/home/runner/intel/oneapi" >> "$GITHUB_ENV" - sudo chown root /bin/tar && sudo chmod u+s /bin/tar + sudo chown root /bin/tar && sudo chmod u+s /bin/tar # tar needs to run as root to extract when restoring the cache - name: fix gcc issue on the runner (GNU / macos) if: matrix.compiler == 'gfortran' && matrix.os == 'macos' @@ -58,6 +58,11 @@ runs: ${{ env.MCFOST_INSTALL }}/lib/${{ matrix.compiler }} key: mcfost-deps-${{ runner.os }}-${{ matrix.compiler }}-${{ hashFiles('lib/install.sh') }} + - name: unset setuid bit for tar if cache not found + if: matrix.compiler == 'ifort' && matrix.os == 'linux' && steps.cache-deps.outputs.cache-hit != 'true' + shell: bash + run: sudo chmod u-s /bin/tar + # Only do this (lengthy) setup if dependency cache not found - name: prepare mcfost environment if: ${{ steps.cache-deps.outputs.cache-hit != 'true' }} @@ -66,4 +71,4 @@ runs: run: | [ ! "$SETVARS_COMPLETED" == 1 ] && test -f "$INTEL_PATH"/setvars.sh && . "$INTEL_PATH"/setvars.sh sudo chmod o+w /usr/local/include /usr/local/lib - ./install.sh \ No newline at end of file + ./install.sh diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml new file mode 100644 index 000000000..210cbc721 --- /dev/null +++ b/.github/actions/release/action.yml @@ -0,0 +1,17 @@ +name: "release" +description: "release" + +runs: + using: "composite" + steps: + - name: prepare tarball + working-directory: src + shell: bash + run: tar -cvzf mcfost.tar.gz mcfost + + - name: Add binary to release + uses: shogo82148/actions-upload-release-asset@v1.6.6 + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: src/mcfost.tar.gz + asset_name: ${{ format('mcfost_{0}-{1}.tar.gz', runner.os, runner.arch) }} diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index ff4aa998f..d6dabab50 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -4,7 +4,9 @@ on: push: branches: [ master ] pull_request: - branches: [ master ] + release: + types: + - published # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -22,6 +24,7 @@ jobs: os: [linux, macos] compiler: [gfortran, ifort] openmp: [yes, no] + release: [yes, no] include: # Associate specific os versions with our os tags @@ -36,6 +39,7 @@ jobs: compiler: gfortran container: "quay.io/pypa/manylinux_2_28_x86_64" + name: build+test (${{ matrix.os }}, ${{ matrix.compiler }}, openmp=${{ matrix.openmp }}, release=${{ matrix.release }}) runs-on: ${{ matrix.os-version }} container: ${{ matrix.container }} env: @@ -57,17 +61,18 @@ jobs: shell: bash -e {0} run: | [ ! "$SETVARS_COMPLETED" == 1 ] && test -f "$INTEL_PATH"/setvars.sh && . "$INTEL_PATH"/setvars.sh - make static=yes all + make release=${{ matrix.release }} static=yes all - name: test uses: ./.github/actions/test # Saving the binaries from the macos builds since the runner is the same for both compilers -> might be more consistent - - name: save binary for benchmarking - if: matrix.os == 'macos' && matrix.openmp == 'yes' + # Only benchmark+release the 'release' builds, never the 'march=native' builds + - name: save binary for benchmarking and release + if: matrix.openmp == 'yes' && matrix.release == 'yes' uses: actions/upload-artifact@v3 with: - name: mcfost-binary-${{ matrix.compiler }} + name: mcfost-binary-${{ matrix.os }}-${{ matrix.compiler }} path: src/mcfost benchmark: @@ -86,8 +91,8 @@ jobs: - name: restore binary artifact uses: actions/download-artifact@v3 - with: - name: mcfost-binary-${{ matrix.compiler }} + with: + name: mcfost-binary-macos-${{ matrix.compiler }} path: src/ - name: benchmark @@ -96,3 +101,41 @@ jobs: - name: report benchmark results run: | echo ${{ matrix.compiler }} $(cat benchmark-${{ matrix.compiler }}.tim) + + + release: + needs: build-and-test + strategy: + matrix: + os: [linux, macos] + + include: + # Associate specific os versions with our os tags + - os: linux + os-version: ubuntu-22.04 + - os: macos + os-version: macos-12 + + runs-on: ${{ matrix.os-version }} + if: ${{ github.event_name == 'release' }} + + steps: + - uses: actions/checkout@v2 + + - name: restore binary artifact + uses: actions/download-artifact@v3 + with: + name: mcfost-binary-${{ matrix.os }}-ifort # use ifort for binary release + path: src/ + + - name: simple check to make sure the binary runs in a clean environment + shell: bash + working-directory: src + env: + MCFOST_UTILS: ${{ github.workspace }}/utils + run: | + chmod +x mcfost + ./mcfost -h + + - name: upload binary + uses: ./.github/actions/release diff --git a/src/Makefile b/src/Makefile index e464052e9..57c579281 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,6 +1,9 @@ #------------------------------------------------------------------------------# # Makefile for Monte Carlo radiative transfer code MCFOST #------------------------------------------------------------------------------# + +IS_DARWIN := $(shell uname | tr '[a-z]' '[A-Z]' 2>&1 | grep -c DARWIN) + # Linux compiling server LINUX_SERVER=gadi IPAG_SERVER=ipag-nfs @@ -25,14 +28,14 @@ ifeq ($(ifx), yes) SYSTEM=ifx endif +ifeq ($(release), yes) + RELEASE=yes +endif + ifndef OPENMP OPENMP=yes endif -# Forcing static flags when compiling the releases -release : LDFLAGS+= $(STATIC_FLAGS) -release_mac : LDFLAGS+= $(STATIC_FLAGS) - # faster compilation ifeq ($(dev), yes) DEV=yes @@ -83,15 +86,15 @@ endif # System dependent compiler flags ifeq ($(SYSTEM), ifort) - ifeq ($(shell uname | tr '[a-z]' '[A-Z]' 2>&1 | grep -c DARWIN),1) - ARCH= -axSSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2 -mmacosx-version-min=10.13 -mdynamic-no-pic - STATIC_FLAGS= -static-intel -qopenmp-link static + STATIC_FLAGS = -static-intel -qopenmp-link static + ifeq ($(IS_DARWIN),1) + ARCH = -axSSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2 -mmacosx-version-min=10.13 -mdynamic-no-pic else - ARCH= -axSSE2,SSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2,CORE-AVX512 - STATIC_FLAGS= -static-intel -qopenmp-link static -static-libstdc++ + ARCH = -axSSE2,SSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2,CORE-AVX512 + STATIC_FLAGS += -static-libstdc++ endif FC= ifort - FFLAGS= -fpp -O3 -no-prec-div -fp-model fast=2 -traceback $(ARCH) + FFLAGS= -fpp -O3 -no-prec-div -fp-model fast=2 -traceback DBLFLAG= -r8 DEBUGFLAG= -check all -C -g -fpe0 -traceback -no-ftz \ -warn uninitialized -warn unused -warn truncated_source @@ -100,22 +103,22 @@ ifeq ($(SYSTEM), ifort) IPOFLAGS= -ipo CC= icc CXX=icpc -std=c++11 - CCFLAGS= -O3 -no-prec-div -fp-model fast=2 -traceback $(ARCH) + CCFLAGS= -O3 -no-prec-div -fp-model fast=2 -traceback COMPFLAGS= -fopenmp LIBCXX= -cxxlib LIBS= $(LIB)/ifort endif ifeq ($(SYSTEM), ifx) - ifeq ($(shell uname | tr '[a-z]' '[A-Z]' 2>&1 | grep -c DARWIN),1) - ARCH= -axSSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2 -mmacosx-version-min=10.13 -mdynamic-no-pic - STATIC_FLAGS= -static-intel -qopenmp-link static + STATIC_FLAGS = -static-intel -qopenmp-link static + ifeq ($(IS_DARWIN),1) + ARCH = -axSSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2 -mmacosx-version-min=10.13 -mdynamic-no-pic else - ARCH= -axSSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2,CORE-AVX512 - STATIC_FLAGS= -static-intel -qopenmp-link static -static-libstdc++ + ARCH = -axSSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2,CORE-AVX512 + STATIC_FLAGS += -static-libstdc++ endif FC= ifx - FFLAGS= -fpp -O3 -fp-model fast=2 -traceback $(ARCH) + FFLAGS= -fpp -O3 -fp-model fast=2 -traceback DBLFLAG= -r8 DEBUGFLAG= -check all -C -g -fpe0 -traceback -no-ftz \ -warn uninitialized -warn unused -warn truncated_source @@ -124,7 +127,7 @@ ifeq ($(SYSTEM), ifx) IPOFLAGS= -ipo CC= icx CXX=icpx -std=c++11 - CCFLAGS= -O3 -fp-model fast=2 -traceback $(ARCH) + CCFLAGS= -O3 -fp-model fast=2 -traceback COMPFLAGS= -qopenmp LIBCXX= -cxxlib LIBS= $(LIB)/ifx @@ -134,7 +137,7 @@ ifeq ($(SYSTEM), gfortran) FC= gfortran FFLAGS+= -m64 -cpp -O3 -funroll-loops -ffinite-math-only -fno-trapping-math \ -ffree-line-length-132 -fbacktrace - STATIC_FLAGS= -static-libgcc -static-libstdc++ + STATIC_FLAGS= -static-libgcc -static-libstdc++ -static-libgfortran -static-libquadmath DBLFLAG= -fdefault-real-8 -fdefault-double-8 DEBUGFLAG= -g -fbounds-check -Wunused-variable -Wunused-function -Wuninitialized -fbacktrace KNOWN_SYSTEM= yes @@ -153,6 +156,16 @@ endif #------------------------------------------------------------------------------# # Set other optional flags depending on system settings +# Force static flags when compiling the releases, and march=native if not a release +ifeq ($(RELEASE), yes) + STATIC = yes + FFLAGS += $(ARCH) + CCFLAGS += $(ARCH) +else + FFLAGS += -march=native + CCFLAGS += -march=native +endif + ifeq ($(DEBUG), yes) FFLAGS+= ${DEBUGFLAG} FFLAGS:= $(FFLAGS:-O3=-O0) @@ -164,7 +177,7 @@ endif ifeq ($(DEV), yes) - FFLAGS:= $(FFLAGS:-O3=-O1) + FFLAGS:= $(FFLAGS:-O3=-O1) endif ifeq ($(OPENMP), yes) @@ -302,47 +315,11 @@ mcfost2prodimo : mcfost mcfost2prodimo.o #------------------------------------------------------------------------------# # Release -.PHONY : release release_mac release_utils fosti dmz wardlaw - -release : main - - git tag -d release - git tag release - git push github - git push github --tags --force - mkdir -p $(MCFOST_INSTALL)/bin/ - \cp -f mcfost $(MCFOST_INSTALL)/bin/ - rm -rf web - mkdir web web/linux web/mac - cp mcfost mcfost_update - tar czvf mcfost_bin.tgz mcfost_update ; rm -rf mcfost_update - openssl sha1 mcfost_bin.tgz | awk '{print $$2}' > web/mac/mcfost.sha1 - mv mcfost_bin.tgz web/mac/ - ssh $(LINUX_SERVER) "cd mcfost/src && git pull origin master && git fetch --tags --force && git checkout release && export MCFOST_GIT=1 && export MAKEFLAGS= && make clean mcfost static=yes && git checkout master && \cp -f mcfost ~/mcfost_cigri && \cp -f mcfost mcfost_update && rm -f mcfost.sha1 && tar czf mcfost_bin.tgz mcfost_update --remove-files && sha1sum mcfost_bin.tgz | awk '{print $$1}' > mcfost.sha1" - scp $(LINUX_SERVER):mcfost/src/mcfost_bin.tgz web/linux/ - scp $(LINUX_SERVER):mcfost/src/mcfost.sha1 web/linux/ - cp `grep mcfost_version mcfost_env.f90 | awk '{print "ref"$$6".para ref"$$6"_multi.para ref"$$6"_3D.para"}' ` web - grep mcfost_release mcfost_env.f90 | awk '{print $$6}' | sed s/\"//g > web/version.txt - git log --pretty=format:"%cd %h %d %s" --date=short --graph | sed s/"dmz\/master, phare\/master, github\/master, bitbucket\/master"// | sed s/"()"// | sed s/", ,"/","/ > web/history.txt - rsync -Pur web/* $(IPAG_SERVER):/user/publicdir/pintec/mcfost - ssh $(IPAG_SERVER) "cd /user/publicdir/pintec/mcfost/mac ; tar xzf mcfost_bin.tgz ; mv mcfost_update mcfost ; rm -f mcfost.tgz ; tar czf mcfost.tgz mcfost --remove-files ; cd /user/publicdir/pintec/mcfost/linux ; tar xzf mcfost_bin.tgz ; mv mcfost_update mcfost ; rm -f mcfost.tgz ; tar czf mcfost.tgz mcfost --remove-files" +.PHONY : release_utils fosti dmz wardlaw release_doc: pushd . ; cd ../docs ; make html ; cd _build ; tar czf html.tgz html ; scp -r html.tgz $(IPAG_SERVER):/user/publicdir/pintec/mcfost/docs ; ssh $(IPAG_SERVER) "cd /user/publicdir/pintec/mcfost/docs ; tar xzf html.tgz ; rm -f html.tgz" ; popd -release_mac : main - \cp -f mcfost $(MCFOST_INSTALL)/bin/ - rm -rf web - mkdir web web/mac - cp mcfost mcfost_update - tar czvf mcfost_bin.tgz mcfost_update ; rm -rf mcfost_update - openssl sha1 mcfost_bin.tgz | awk '{print $$2}' > web/mac/mcfost.sha1 - mv mcfost_bin.tgz web/mac/ - cp `grep mcfost_version mcfost_env.f90 | awk '{print "ref"$$6".para ref"$$6"_multi.para ref"$$6"_3D.para"}' ` web - grep mcfost_release mcfost_env.f90 | awk '{print $$6}' | sed s/\"//g > web/version.txt - git log --pretty=format:"%cd %h %d %s" --date=short --graph --branches > web/history.txt - rsync -Pur -r web/* $(IPAG_SERVER):/user/publicdir/pintec/mcfost - ssh $(IPAG_SERVER) "cd /user/publicdir/pintec/mcfost/mac ; tar xzf mcfost_bin.tgz ; mv mcfost_update mcfost ; rm -f mcfost.tgz ; tar czf mcfost.tgz mcfost --remove-files" - release_utils : tar czvf mcfost_utils.tgz -C../utils . openssl sha1 mcfost_utils.tgz | awk '{print $2}' > mcfost_utils.sha1