From 0d754094442a62b1ec1b30819b4edf22090d9496 Mon Sep 17 00:00:00 2001 From: Benjamin Morgan Date: Fri, 30 Oct 2020 18:30:03 +0100 Subject: [PATCH] tooling: Simplify and streamline Makefiles In particular, you can now do everything with all the packages as you can with any individual packages. These targets are in the Makefile.all file, which is included by Makefile. This makes it more clear, what Makefile itself brings to the table. --- Makefile | 222 +++++++------------------------ Makefile.all | 157 ++++++++++++++++++++++ Makefile.setup | 11 +- README.md | 9 +- dist/docker/Dockerfile.alpine | 3 +- dist/docker/Dockerfile.archlinux | 5 +- dist/docker/Dockerfile.ubuntu | 5 +- 7 files changed, 226 insertions(+), 186 deletions(-) create mode 100644 Makefile.all diff --git a/Makefile b/Makefile index ca679e2b9..2f4d53421 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,8 @@ # Makefile +# # This file contains Makefile targets for the cloe project. # -# Make configuration: -SHELL := /bin/bash -GNUMAKEFLAGS := --no-print-directory - # Set the clang-format command line to use: CLANG_FORMAT := $(shell command -v clang-format 2>/dev/null) CLANG_FORMAT_ARGS := -style=file @@ -16,201 +13,84 @@ AG := $(or \ "grep -r" \ ) +define print_header + @printf "________________________________________" + @printf "________________________________________\n" + @printf ":: %s\n" ${1} +endef + CLOE_ROOT := . CLOE_LAUNCH := PYTHONPATH="${CLOE_ROOT}/cli" python3 -m cloe_launch # Build configuration: -SOURCE_DIR := . -BUILD_DIR := build -INSTALL_DIR := /usr/local - -# These packages can all be built in parallel: -PACKAGES := engine $(wildcard plugins/*) - -# This variable contains packages that should not be built. -# This should only be provided from the command line, for example: -# make NOBUILD_PKGS="plugins/vtd plugins/demo_printer" package -NOBUILD_PKGS := - -# These options are passed to conan when it is working with the cloe metapackage. +BUILD_DIR := build +INSTALL_DIR := /usr/local CONAN_OPTIONS := -# Options: -BUILD_TESTS := 1 -ifeq (${BUILD_TESTS},0) - CONAN_OPTIONS += -o cloe:test=False - CONAN_OPTIONS += -o fable:test=False - CONAN_OPTIONS += -o cloe-runtime:test=False - CONAN_OPTIONS += -o cloe-models:test=False - CONAN_OPTIONS += -o cloe-oak:test=False - ifeq (${WITH_ENGINE},1) - CONAN_OPTIONS += -o cloe-engine:test=False - endif - ifeq (${WITH_VTD},1) - CONAN_OPTIONS += -o cloe-plugin-vtd:test=False - endif - CONAN_OPTIONS += -o cloe-plugin-basic:test=False -endif - -WITH_ENGINE := 1 -ifeq (${WITH_ENGINE},0) - PACKAGES := $(filter-out engine, ${PACKAGES}) - CONAN_OPTIONS += -o cloe:with_engine=False -endif - -WITH_VTD := 1 -ifeq (${WITH_VTD},0) - PACKAGES := $(filter-out plugins/vtd, ${PACKAGES}) - CONAN_OPTIONS += -o cloe:with_vtd=False -endif - -# Take NOBUILD_PKGS into account to assemble a final list of -# packages that we shall build (in addition to fable, runtime, and models). -# This can also be provided from the command line. -BUILD_PKGS := $(filter-out ${NOBUILD_PKGS}, ${PACKAGES}) - .PHONY: help .DEFAULT: help +.SILENT: help help:: - @echo "Usage: make " - @echo + echo "Usage: make " + echo -# Setup Targets ---------------------------------------------------------------- include Makefile.setup +include Makefile.all -# Build Targets ----------------------------------------------------------------------------- +# Workspace targets ----------------------------------------------------------- +.PHONY: deploy doxygen docker docker-test smoketest help:: - @echo "Available workspace build targets:" - @echo " smoketest to run BATS system tests" - @echo " clean to clean all generated artifacts" - @echo " git-clean to clean all extraneous artifacts from repository" - @echo " doc to generate Doxygen documentation" - @echo - @echo "Available Conan build targets:" - @echo " export to export all recipes to Conan cache" - @echo " package to build and package all Cloe Conan packages" - @echo " purge to remove all Cloe Conan packages from cache" - @echo " deploy to install all Cloe Conan packages locally" - @echo " docker to build all Docker images" - @echo - @echo "Available development targets:" - @echo " format to format Cloe source code with clang-format" - @echo " todos to show all TODOs in Cloe source code" - @echo - @echo "Options:" - @echo " BUILD_TESTS=(0|1) to build and run unit tests" - @echo " WITH_ENGINE=(0|1) to build and deploy cloe-engine (default=1)" - @echo " WITH_VTD=(0|1) to build and deploy cloe-plugin-vtd (default=1)" - @echo - @echo "Defines:" - @echo " CONAN_OPTIONS = ${CONAN_OPTIONS}" - @echo " NOBUILD_PKGS = ${NOBUILD_PKGS}" - @echo " SOURCE_DIR = ${SOURCE_DIR}" - @echo " BUILD_DIR = ${BUILD_DIR}" - @echo - -.PHONY: clean clean-all test test-all package package-all deploy purge purge-all export smoketest git-clean docker - -${BUILD_PKGS}:: - ${MAKE} -C $@ $(shell echo ${MAKECMDGOALS} | sed 's/-all//') - -export: - conan export vendor/cpp-netlib cloe/stable - conan export vendor/incbin cloe/stable - conan export vendor/open-simulation-interface cloe/stable + echo "Available workspace targets:" + echo " deploy to deploy Cloe to INSTALL_DIR [=${INSTALL_DIR}]" + echo " docker to build all Docker images" + echo " docker-test to build only a single Docker image" + echo " doxygen to generate Doxygen documentation" + echo " smoketest to run BATS system tests" + echo -smoketest: - @echo ___________________________________________________________________________________ - @echo "Running smoketests..." - @${CLOE_LAUNCH} clean -P conanfile.py - @\time -f "\nTotal smoketest time (real) = %e sec" bats tests - -doc: - @echo ___________________________________________________________________________________ - @echo "Generating Doxygen documentation..." - mkdir -p build/doxygen - doxygen Doxyfile +deploy: + $(call print_header, "Deploying binaries to ${INSTALL_DIR}...") + conan install ${CONAN_OPTIONS} --install-folder ${BUILD_DIR}/deploy -g deploy . + mkdir -p ${INSTALL_DIR} + cp -r ${BUILD_DIR}/deploy/cloe-*/* ${INSTALL_DIR}/ docker: + $(call print_header, "Building all Docker images...") ${MAKE} -C dist/docker all docker-test: + $(call print_header, "Building ubuntu-18.04 Docker image...") ${MAKE} -C dist/docker ubuntu-18.04 -clean-all: ${BUILD_PKGS} - -clean: - @echo ___________________________________________________________________________________ - @echo "Cleaning build files..." - rm -rf ${BUILD_DIR} - rm -f compile_commands.json - ${MAKE} -C runtime clean - ${MAKE} -C models clean - ${MAKE} clean-all - -git-clean: - -git clean -xdf -e .gtm - -purge-all: ${BUILD_PKGS} - -purge: - @echo ___________________________________________________________________________________ - @echo "Removing Conan packages..." - ${MAKE} -C fable purge - ${MAKE} -C oak purge - ${MAKE} -C runtime purge - ${MAKE} -C models purge - ${MAKE} purge-all - -package-all: ${BUILD_PKGS} - -package: - @echo ___________________________________________________________________________________ - @echo "Creating Conan packages..." - conan export fable cloe/develop - conan export oak cloe/develop - ${MAKE} -C runtime package - ${MAKE} -C models package - ${MAKE} package-all - ${MAKE} -f Makefile.package package - -package-outdated-all: ${BUILD_PKGS} - -package-outdated: - @echo ___________________________________________________________________________________ - @echo "Creating outdated Conan packages..." - ${MAKE} -C runtime package-outdated - ${MAKE} -C models package-outdated - ${MAKE} package-outdated-all - -test-all: ${BUILD_PKGS} - -test: - @echo ___________________________________________________________________________________ - @echo "Running CMake tests..." - ${MAKE} -C runtime test - ${MAKE} -C models test - ${MAKE} test-all +doxygen: + $(call print_header, "Generating Doxygen documentation...") + mkdir -p ${BUILD_DIR}/doxygen + doxygen Doxyfile -deploy: - @echo ___________________________________________________________________________________ - @echo "Deploying binaries to ${INSTALL_DIR}..." - conan install ${CONAN_OPTIONS} --install-folder ${BUILD_DIR}/deploy -g deploy . - mkdir -p ${INSTALL_DIR} - cp -r ${BUILD_DIR}/deploy/cloe-*/* ${INSTALL_DIR}/ +smoketest: + $(call print_header, "Running smoke tests...") + @${CLOE_LAUNCH} clean -P conanfile.py + @\time -f "\nTotal smoketest time (real) = %e sec" bats tests + + +# Development targets --------------------------------------------------------- +.PHONY: format todos find-missing-eol sanitize-files +help:: + echo "Available development targets:" + echo " format to format Cloe source code with clang-format" + echo " todos to show all TODOs in Cloe source code" + echo + +format: + find . -type f -not -path '*/\.git/*' -and \( -name '*.cpp' -o -name '*.hpp' \) -exec ${CLANG_FORMAT} ${CLANG_FORMAT_ARGS} -i {} \; -# Development targets ----------------------------------------------------------------------- -.PHONY: todos format find-missing-eol sanitize-files todos: ${AG} TODO ${AG} FIXME ${AG} XXX -format: - find . -type f -not -path '*/\.git/*' -and \( -name '*.cpp' -o -name '*.hpp' \) -exec ${CLANG_FORMAT} ${CLANG_FORMAT_ARGS} -i {} \; +find-missing-eol: + find . -type f -size +0 -exec gawk 'ENDFILE{if ($0 == "") print FILENAME}' {} \; sanitize-files: git grep --cached -Ilz '' | while IFS= read -rd '' f; do tail -c1 < "$$f" | read -r _ || echo >> "$$f"; done - -find-missing-eol: - find . -type f -size +0 -exec gawk 'ENDFILE{if ($0 == "") print FILENAME}' {} \; diff --git a/Makefile.all b/Makefile.all new file mode 100644 index 000000000..b3fa06cf3 --- /dev/null +++ b/Makefile.all @@ -0,0 +1,157 @@ +# Makefile.all +# +# This file defines all packages available for building and defines their +# dependencies, so that make can build them in parallel effectively. +# +# Use make USE_NPROC=1 for parallel execution of the targets. +# Current tests show a ~50% speed-up. +# + +# Make configuration: +SHELL := /bin/bash +GNUMAKEFLAGS := --no-print-directory +SUBMAKEFLAGS := + +USE_NPROC := 0 +ifeq (${USE_NPROC},1) + NPROC := $(shell nproc 2>/dev/null) + ifdef NPROC + SUBMAKEFLAGS += -j${NPROC} + endif +endif + +ALL_VENDOR_PKGS := $(wildcard vendor/*) +ALL_RUNTIME_PKGS := fable runtime models +ALL_ENGINE_PKGS := oak engine +ALL_PLUGIN_PKGS := $(wildcard plugins/*) +ALL_PKGS := ${ALL_RUNTIME_PKGS} ${ALL_ENGINE_PKGS} ${ALL_PLUGIN_PKGS} cloe +.PHONY: ${ALL_PKGS} + +# Define dependencies for each package: +fable: +runtime: fable +models: runtime + +oak: runtime +engine: models oak + +${ALL_PLUGIN_PKGS}: runtime models + +# This variable contains packages that should not be select for the action. +# This should only be provided from the command line, for example: +# make UNSELECT_PKGS="plugins/vtd plugins/demo_printer" package +UNSELECT_PKGS := + +CONAN_OPTIONS := + +# Options: +VENDOR_ONLY := 0 +ifeq (${VENDOR_ONLY},1) + ALL_PKGS := ${ALL_VENDOR_PKGS} +endif + +BUILD_TESTS := 1 +ifeq (${BUILD_TESTS},0) + CONAN_OPTIONS += -o cloe:test=False + CONAN_OPTIONS += -o fable:test=False + CONAN_OPTIONS += -o cloe-runtime:test=False + CONAN_OPTIONS += -o cloe-models:test=False + CONAN_OPTIONS += -o cloe-oak:test=False + ifeq (${WITH_ENGINE},1) + CONAN_OPTIONS += -o cloe-engine:test=False + endif + ifeq (${WITH_VTD},1) + CONAN_OPTIONS += -o cloe-plugin-vtd:test=False + endif + CONAN_OPTIONS += -o cloe-plugin-basic:test=False +endif + +WITH_ENGINE := 1 +ifeq (${WITH_ENGINE},0) + ALL_PKGS := $(filter-out oak, ${ALL_PKGS}) + ALL_PKGS := $(filter-out engine, ${ALL_PKGS}) + CONAN_OPTIONS += -o cloe:with_engine=False +endif + +WITH_VTD := 1 +ifeq (${WITH_VTD},0) + ALL_PKGS := $(filter-out plugins/vtd, ${ALL_PKGS}) + CONAN_OPTIONS += -o cloe:with_vtd=False +endif + +# Take UNSELECT_PKGS into account to assemble a final list of +# packages that we shall build (in addition to fable, runtime, and models). +# This can also be provided from the command line. +SELECT_PKGS := $(filter-out ${UNSELECT_PKGS}, ${ALL_PKGS}) +${SELECT_PKGS}: + ${MAKE} -C $@ $(shell echo ${MAKECMDGOALS} | sed -re 's/-all//' ) + +# Re-define cloe target to use Makefile.package, and only run for targets +# where it makes sense, since "cloe" is a Conan meta-package. +cloe: + for case in export package package-outdated list purge clean; do \ + if [[ "${MAKECMDGOALS}" == "$${case}-all" ]]; then \ + ${MAKE} -f Makefile.package CONAN_OPTIONS="${CONAN_OPTIONS}" $${case}; \ + fi \ + done + +define make_rule +help:: + @printf " % -16s to %s\n" ${1} ${2} + +${1}-all: ${SELECT_PKGS} +${1}: + @printf "________________________________________" + @printf "________________________________________\n" + @printf ":: Proceeding to%s\n" "$(shell echo "${2}" | sed -re 's/ +/ /g')" + ${MAKE} ${SUBMAKEFLAGS} ${1}-all +endef + +define make_target + $(eval $(call make_rule,${1},${2})) +endef + +.PHONY: help +.DEFAULT: help +.SILENT: help +help:: + echo "Available package targets:" + +$(call make_target, export, "export selected package recipes [conan-cache]") +$(call make_target, package, "create selected packages [conan-cache]") +$(call make_target, package-outdated, "create selected packages if outdated [conan-cache]") +$(call make_target, list, "list selected package install files [conan-cache]") +$(call make_target, purge, "remove selected packages from cache [conan-cache]") + +help:: + echo + +$(call make_target, editable, "instruct Conan to use in-source build for selected packages") +$(call make_target, uneditable, "instruct Conan to use local cache for selected packages") + +help:: + echo + +$(call make_target, all, "build selected packages [in-source]") +$(call make_target, conan, "configure Conan and get dependencies [in-source]") +$(call make_target, configure, "configure CMake packages [in-source]") +$(call make_target, test, "run CMake tests if they are available [in-source]") +$(call make_target, export-pkg, "export build artifacts to Conan cache [in-source]") +$(call make_target, clean, "remove build artifacts [in-source]") + +help:: + echo + echo " Options:" + echo " BUILD_TESTS=(0|1) to build and run unit tests (default=1)" + echo " WITH_ENGINE=(0|1) to build and deploy cloe-engine (default=1)" + echo " WITH_VTD=(0|1) to build and deploy cloe-plugin-vtd (default=1)" + echo " WITH_CLOE=(0|1) to build cloe metapackage (default=0)" + echo " VENDOR_ONLY=(0|1) to only select vendor packages (default=0)" + echo " (note that vendor packages are normally unselected)" + echo + echo " Defines:" + echo " CONAN_OPTIONS = ${CONAN_OPTIONS}" + echo " UNSELECT_PKGS = ${UNSELECT_PKGS}" + echo " SELECT_PKGS =" + echo "$$(echo " ${SELECT_PKGS}" | fmt -70)" + echo diff --git a/Makefile.setup b/Makefile.setup index 57b158baf..db954b806 100644 --- a/Makefile.setup +++ b/Makefile.setup @@ -32,12 +32,13 @@ endif .PHONY: help .DEFAULT: help +.SILENT: help help:: - @echo "Available setup targets:" - @echo " setup to perform Git repository setup" - @echo " install-system-deps to install build (and development) system requirements" - @echo " install-python-deps to install build (and development) Python requirements" - @echo + echo "Available setup targets:" + echo " setup to perform Git repository setup" + echo " install-system-deps to install build (and development) system requirements" + echo " install-python-deps to install build (and development) Python requirements" + echo .PHONY: setup setup: diff --git a/README.md b/README.md index df7397d54..cabc9588c 100644 --- a/README.md +++ b/README.md @@ -87,10 +87,14 @@ See the Conan documentation for more information on how to do this. To build all packages, you should run the following: - make export packages + make export VENDOR_ONLY=1 + make package This will export Conan recipes for some of our dependencies and build the -packages that are contained in the repository. +packages that are contained in the repository. If you want to make use of +all your 64 CPU cores, you can run the last command like so: + + make package USE_NPROC=1 For more details on how this is done, have a look at the Makefiles in the repository root. @@ -105,6 +109,7 @@ functionality for you in one convenient place. You can install an editable instance with `pipx` (or `pip`): + cd cli pipx install -e . This has the advantage that any updates to the repository will be transparently diff --git a/dist/docker/Dockerfile.alpine b/dist/docker/Dockerfile.alpine index 05a499fc7..030389eb6 100644 --- a/dist/docker/Dockerfile.alpine +++ b/dist/docker/Dockerfile.alpine @@ -43,10 +43,9 @@ RUN pip3 install --upgrade pip && \ # Build and Install Cloe WORKDIR /cloe COPY . /cloe -RUN make export-deps && \ +RUN make VENDOR_ONLY=1 export && \ make BUILD_TESTS=0 WITH_ENGINE=0 WITH_VTD=0 package && \ make BUILD_TESTS=0 WITH_ENGINE=0 WITH_VTD=0 INSTALL_DIR="/deploy" deploy && \ - make clean && \ conan remove \* -b -f # Finalize Image diff --git a/dist/docker/Dockerfile.archlinux b/dist/docker/Dockerfile.archlinux index d9350f2d2..e4806c632 100644 --- a/dist/docker/Dockerfile.archlinux +++ b/dist/docker/Dockerfile.archlinux @@ -18,7 +18,7 @@ RUN pacman -Syu --noconfirm --needed make && \ RUN pip3 install --upgrade pip && \ make -f /cloe/Makefile.setup \ - WITHOUT_DEV_DEPS=yes \ + WITHOUT_DEV_DEPS=yes PIP_INSTALL_ARGS="" \ install-python-deps # Install and Setup Conan @@ -32,10 +32,9 @@ RUN conan profile new --detect default && \ # Build and Install Cloe WORKDIR /cloe COPY . /cloe -RUN make export && \ +RUN make VENDOR_ONLY=1 export && \ make WITH_VTD=0 package && \ make WITH_VTD=0 INSTALL_DIR="/deploy" deploy && \ - make clean && \ conan remove \* -b -f # Create Deploy Image diff --git a/dist/docker/Dockerfile.ubuntu b/dist/docker/Dockerfile.ubuntu index 349e32f0c..45554fb7c 100644 --- a/dist/docker/Dockerfile.ubuntu +++ b/dist/docker/Dockerfile.ubuntu @@ -26,7 +26,7 @@ RUN apt-get update && \ RUN pip3 install --upgrade pip && \ make -f /cloe/Makefile.setup \ - WITHOUT_DEV_DEPS=yes \ + WITHOUT_DEV_DEPS=yes PIP_INSTALL_ARGS="" \ install-python-deps # Install and Setup Conan @@ -51,7 +51,7 @@ COPY . /cloe RUN \ # Export our own Conan recipes, in case they are not available in the # CONAN_REMOTE specified above. - make export && \ + make VENDOR_ONLY=1 export && \ # Build all the packages, except for vtd, because that currently requires # dependencies we don't have in the Docker container. # You can specify more than one package here, see the Makefile for more @@ -64,7 +64,6 @@ RUN \ # In the above commands, Conan downloads and creates packages into its # local cache. We don't need the build directories, since we have deployed # Cloe, so we should clean up to keep the Docker image down. - make clean && \ conan remove \* -b -f # Create Deploy Image