diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 5d3cab74..00000000 --- a/.appveyor.yml +++ /dev/null @@ -1,47 +0,0 @@ -environment: - matrix: - - julia_version: 1.0 - - julia_version: 1.3 - - julia_version: nightly - -platform: - - x86 # 32-bit - - x64 # 64-bit - -matrix: - allow_failures: - - julia_version: nightly - -# # Uncomment the following lines to allow failures on nightly julia -# # (tests will run but not make your overall status red) -# matrix: -# allow_failures: -# - julia_version: latest - -branches: - only: - - master - - /release-.*/ - -notifications: - - provider: Email - on_build_success: false - on_build_failure: false - on_build_status_changed: false - -install: - - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1")) - -build_script: - - echo "%JL_BUILD_SCRIPT%" - - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%" - -test_script: - - echo "%JL_TEST_SCRIPT%" - - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%" - -# # Uncomment to support code coverage upload. Should only be enabled for packages -# # which would have coverage gaps without running on Windows -# on_success: -# - echo "%JL_CODECOV_SCRIPT%" -# - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%" diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 69cb7601..00000000 --- a/.codecov.yml +++ /dev/null @@ -1 +0,0 @@ -comment: false diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index cc27b731..00000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1 +0,0 @@ -custom: https://numfocus.salsalabs.org/donate-to-julia/index.html diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml new file mode 100644 index 00000000..ad15d36f --- /dev/null +++ b/.github/workflows/CompatHelper.yml @@ -0,0 +1,24 @@ +name: CompatHelper + +on: + schedule: + - cron: '00 * * * *' + +jobs: + CompatHelper: + runs-on: ${{ matrix.os }} + strategy: + matrix: + julia-version: [1.2.0] + julia-arch: [x86] + os: [ubuntu-latest] + steps: + - uses: julia-actions/setup-julia@latest + with: + version: ${{ matrix.julia-version }} + - name: Pkg.add("CompatHelper") + run: julia -e 'using Pkg; Pkg.add("CompatHelper")' + - name: CompatHelper.main() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: julia -e 'using CompatHelper; CompatHelper.main( (; registries) -> CompatHelper._update_manifests(pwd(); registries = registries) )' diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml deleted file mode 100644 index d77d3a0c..00000000 --- a/.github/workflows/TagBot.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: TagBot -on: - schedule: - - cron: 0 * * * * -jobs: - TagBot: - runs-on: ubuntu-latest - steps: - - uses: JuliaRegistries/TagBot@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/documenter-workflow.yml b/.github/workflows/documenter-workflow.yml new file mode 100644 index 00000000..26d15524 --- /dev/null +++ b/.github/workflows/documenter-workflow.yml @@ -0,0 +1,29 @@ +name: Documentation + +on: + push: + branches: master + + pull_request: + branches: master + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + julia-version: [1.3.0] + julia-arch: [x86] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v1.0.0 + - uses: julia-actions/setup-julia@latest + with: + version: ${{ matrix.julia-version }} + - name: Install dependencies + run: julia --project=docs/ -e 'using Pkg; Pkg.instantiate()' + - name: Build and deploy + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} + run: julia --project=docs/ docs/make.jl deploy diff --git a/.gitignore b/.gitignore index ee7e4b0e..c3d31939 100644 --- a/.gitignore +++ b/.gitignore @@ -1,20 +1,11 @@ -*.jl.cov *.jl.*.cov +*.jl.cov *.jl.mem -deps.jl -*.csv -snoopy.jl -precompile.jl -sysimg -*.ji -*.o +.DS_Store +/Manifest.toml +/examples/MyApp/MyApp +/dev/ *.so -*.so.* -*.dylib *.dll -hello -hello.exe -packages -deps/usr/ -deps/build.log -Manifest.toml +*.dylib +*.o diff --git a/.mailmap b/.mailmap deleted file mode 100644 index af815b8d..00000000 --- a/.mailmap +++ /dev/null @@ -1,8 +0,0 @@ -Luca Trevisani -Luca Trevisani - -Simon Danisch - -Viral B. Shah -Viral B. Shah -Viral B. Shah diff --git a/.travis.yml b/.travis.yml index 70e59b3c..7ed91e20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,46 +1,39 @@ -## Documentation: http://docs.travis-ci.com/user/languages/julia/ - language: julia - os: - linux - osx - + - windows julia: - - 1.0 - 1.3 - - nightly - -matrix: +before_install: + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update; fi + - if [ "$TRAVIS_OS_NAME" = "linux" ] && [ $TRAVIS_CPU_ARCH != arm64 ]; then sudo apt-get install gcc-multilib; fi +arch: + - amd64 + - x86 + - arm64 +jobs: allow_failures: - - julia: nightly - + - arch: x86 + exclude: + - os: osx + arch: x86 + - os: osx + arch: arm64 + - os: windows + arch: arm64 + include: + - stage: "Documentation" + julia: 1.3 + os: linux + script: + - julia --project=docs/ -e 'using Pkg; Pkg.instantiate()' + - julia --project=docs/ docs/make.jl + after_success: skip +branches: + only: + - master + - /^release-.*/ + - /^v[0-9]+\.[0-9]+\.[0-9]+$/ # version tags notifications: email: false - -git: - depth: 99999999 - -## uncomment the following lines to allow failures on nightly julia -## (tests will run but not make your overall status red) -# matrix: -# allow_failures: -# - julia: nightly - -## uncomment and modify the following lines to manually install system packages -#addons: -# apt: # apt-get for linux -# packages: -# - gfortran -#before_script: # homebrew for mac -# - if [ $TRAVIS_OS_NAME = osx ]; then brew install gcc; fi - -## uncomment the following lines to override the default test script -#script: -# - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("PackageCompiler"); Pkg.test("PackageCompiler"; coverage=true)' - -after_success: - # push coverage results to Coveralls - - julia -e 'cd(normpath(Base.find_package("PackageCompiler"), "..", "..")); using Pkg; Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())' - # push coverage results to Codecov - - julia -e 'cd(normpath(Base.find_package("PackageCompiler"), "..", "..")); using Pkg; Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())' diff --git a/Artifacts.toml b/Artifacts.toml new file mode 100644 index 00000000..3f86a9c3 --- /dev/null +++ b/Artifacts.toml @@ -0,0 +1,9 @@ +[[x86_64-w64-mingw32]] +git-tree-sha1 = "572b61b5075459e3ed62317e674398166ca98dd4" +os = "windows" +arch = "x86_64" +lazy = true + + [[x86_64-w64-mingw32.download]] + sha256 = "fe3f401bc936fbe6af940b26c5e0f266f762a3416f979c706e599b24082dc5c7" + url = "/~https://github.com/JuliaComputing/PackageCompilerX.jl/releases/download/v0.1/x86_64-8.1.0-release-posix-seh-rt_v6-rev0.tar.gz" diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..cc1ebfbf --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2015-2020 Kristoffer Carlsson, Simon Danisch, Luca Trevisani, Julia Computing, and contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100644 index adcc4f70..00000000 --- a/LICENSE.md +++ /dev/null @@ -1,22 +0,0 @@ -The PackageCompiler.jl package is licensed under the MIT "Expat" License: - -> Copyright (c) 2017: SimonDanisch. -> -> Permission is hereby granted, free of charge, to any person obtaining a copy -> of this software and associated documentation files (the "Software"), to deal -> in the Software without restriction, including without limitation the rights -> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -> copies of the Software, and to permit persons to whom the Software is -> furnished to do so, subject to the following conditions: -> -> The above copyright notice and this permission notice shall be included in all -> copies or substantial portions of the Software. -> -> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -> SOFTWARE. -> diff --git a/Project.toml b/Project.toml index a585ba7a..c244d5ca 100644 --- a/Project.toml +++ b/Project.toml @@ -1,30 +1,18 @@ name = "PackageCompiler" uuid = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" -version = "0.6.5" +version = "1.0.0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" -Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" -WinRPM = "c17dfb99-b4f7-5aad-8812-456da1ad7187" [compat] -julia = "1" -WinRPM = "0.4.3, 1" +julia = "1.3.1" [extras] -ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" -DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -FixedPointNumbers = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" -JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Example = "7876af07-990d-54b4-ab0e-23690620f79a" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -UnicodePlots = "b8865327-cd53-5732-bb35-84acbb429228" -ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" - [targets] -test = ["OffsetArrays", "JSON", "Pkg", "DataStructures", "ColorTypes", "Test", "UnicodePlots", "FixedPointNumbers", "ArgParse"] +test = ["Test", "Example"] diff --git a/README.md b/README.md index 7ccb81b8..c70d9f4f 100644 --- a/README.md +++ b/README.md @@ -1,306 +1,16 @@ # PackageCompiler -[![Build Status](https://travis-ci.org/JuliaLang/PackageCompiler.jl.svg?branch=master)](https://travis-ci.org/JuliaLang/PackageCompiler.jl) -[![Coverage Status](https://coveralls.io/repos/JuliaLang/PackageCompiler.jl/badge.svg?branch=master&service=github)](https://coveralls.io/github/JuliaLang/PackageCompiler.jl?branch=master) +[![Build Status](https://travis-ci.com/JuliaComputing/PackageCompiler.jl.svg?branch=master)](https://travis-ci.com/JuliaComputing/PackageCompiler.jl) +[![Codecov](https://codecov.io/gh/JuliaComputing/PackageCompiler.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/JuliaComputing/PackageCompiler.jl) +[![][docs-stable-img]][docs-stable-url] -[![codecov.io](http://codecov.io/github/JuliaLang/PackageCompiler.jl/coverage.svg?branch=master)](http://codecov.io/github/JuliaLang/PackageCompiler.jl?branch=master) +PackageCompiler is a Julia package with two main purposes: -Remove just-in-time compilation overhead from your package and compile it into a system image. + 1. Creating custom sysimages for reduced latency when working locally with packages that has a high startup time. -## Usage example + 2. Creating "apps" which are a bundle of files including an executable that can be sent and run on other machines without Julia being installed on that machine. -One can try ahead of time compiled images online with nextjournal! -Here are some images for a few popular packages: +For installation and usage instructions, see the [documentation][docs-stable-url]. -[dataframes + query](https://nextjournal.com/sdanisch/data-remix) - -[plots & gr backend](https://nextjournal.com/sdanisch/plots-remix) - -[makie & opengl backend](https://nextjournal.com/sdanisch/glmakie-remix) - -[makie & cairo backend](https://nextjournal.com/sdanisch/cairomakie-remix) - -If you find a package to be missing, anyone can create these images and share them here! -One can also download the docker images for local usage: -[instructions](https://nextjournal.com/sdanisch/static-cairomakie) - -(signup code for nextjournal: `julia1.0`) - - -# compile_package -```Julia -using PackageCompiler - -# This command will use the `runtest.jl` of `ColorTypes` + `FixedPointNumbers` to find out what functions to precompile! -# `force = false` to not force overwriting Julia's current system image -compile_package("ColorTypes", "FixedPointNumbers", force = false) - -# force = false is the default and recommended, since overwriting your standard system image can make Julia unusable. - -# If you used force and want your old system image back (force will overwrite the default system image Julia uses) you can run: -revert() - -``` - -# compile_incremental - -This function works like the above, but incrementally adds the newly cached binary to your old system image. -That means that all precompiled code in the system image (e.g. REPL code) is preserved and therefore one gets a lag free start of the Julia REPL. -Also, the compilation times are much faster: - -help?> compile_incremental -``` -compile_incremental( - toml_path::String, snoopfile::String; - force = false, precompile_file = nothing, verbose = true, - debug = false, cc_flags = nothing -) - -Extract all calls from `snoopfile` and ahead of time compiles them -incrementally into the current system image. -`force = true` will replace the old system image with the new one. -The argument `toml_path` should contain a project file of the packages that `snoopfile` explicitly uses. -Implicitly used packages & modules don't need to be contained! - -To compile just a single package, see the simpler version `compile_incremental(package::Symbol)`: -``` - -``` -compile_incremental( - packages::Symbol...; - force = false, reuse = false, verbose = true, - debug = false, cc_flags = nothing -) - -Incrementally compile `package` into the current system image. -`force = true` will replace the old system image with the new one. -`compile_incremental` will run the `Package/test/runtests.jl` file to -record the functions getting compiled. The coverage of the Package's tests will -thus determine what is getting ahead of time compiled. -For a more explicit version of compile_incremental, see: -`compile_incremental(toml_path::String, snoopfile::String)` -``` - - -# more functionality - -```julia -# Or if you simply want to get a native system image e.g. when you have downloaded the generic Julia install: -force_native_image!() - -# Build an executable -build_executable( - "hello.jl", # Julia script containing a `julia_main` function, e.g. like `examples/hello.jl` - snoopfile = "call_functions.jl", # Julia script which calls functions that you want to make sure to have precompiled [optional] - builddir = "path/to/builddir" # that's where the compiled artifacts will end up [optional] -) - -# Build a shared library -build_shared_lib("hello.jl") -``` - - -# Static Julia Compiler - -Build shared libraries and executables from Julia code. - -Run `juliac.jl -h` for help: - -``` -usage: juliac.jl [-v] [-q] [-d ] [-n ] [-p ] [-c] - [-a] [-o] [-s] [-i] [-e] [-t] [-j] [-f ] [-r] - [-R] [-J ] [-H ] [--startup-file {yes|no}] - [--handle-signals {yes|no}] - [--sysimage-native-code {yes|no}] - [--compiled-modules {yes|no}] - [--depwarn {yes|no|error}] - [--warn-overwrite {yes|no}] - [--compile {yes|no|all|min}] [-C ] - [-O {0,1,2,3}] [-g ] [--inline {yes|no}] - [--check-bounds {yes|no}] [--math-mode {ieee,fast}] - [--cc ] [--cc-flag ] [--version] [-h] - juliaprog [cprog] - -Static Julia Compiler - -positional arguments: - juliaprog Julia program to compile - cprog C program to compile (required only when - building an executable, if not provided a - minimal driver program is used) - -optional arguments: - -v, --verbose increase verbosity - -q, --quiet suppress non-error messages - -d, --builddir build directory - -n, --outname output files basename - -p, --snoopfile - specify script calling functions to precompile - -c, --clean remove build directory - -a, --autodeps automatically build required dependencies - -o, --object build object file - -s, --shared build shared library - -i, --init-shared add `init_jl_runtime` and `exit_jl_runtime` to - shared library for runtime initialization - -e, --executable build executable file - -t, --rmtemp remove temporary build files - -j, --copy-julialibs copy Julia libraries to build directory - -f, --copy-file - copy file to build directory, can be repeated - for multiple files - -r, --release build in release mode, implies `-O3 -g0` - unless otherwise specified - -R, --Release perform a fully automated release build, - equivalent to `-atjr` - -J, --sysimage - start up with the given system image file - -H, --home set location of `julia` executable - --startup-file {yes|no} - load `~/.julia/config/startup.jl` - --handle-signals {yes|no} - enable or disable Julia's default signal - handlers - --sysimage-native-code {yes|no} - use native code from system image if available - --compiled-modules {yes|no} - enable or disable incremental precompilation - of modules - --depwarn {yes|no|error} - enable or disable syntax and method - deprecation warnings - --warn-overwrite {yes|no} - enable or disable method overwrite warnings - --compile {yes|no|all|min} - enable or disable JIT compiler, or request - exhaustive compilation - -C, --cpu-target - limit usage of CPU features up to - (implies default `--sysimage-native-code=no`) - -O, --optimize {0,1,2,3} - set the optimization level (type: Int64) - -g, --debug enable / set the level of debug info - generation (type: Int64) - --inline {yes|no} control whether inlining is permitted - --check-bounds {yes|no} - emit bounds checks always or never - --math-mode {ieee,fast} - disallow or enable unsafe floating point - optimizations - --cc system C compiler - --cc-flag pass custom flag to the system C compiler when - building a shared library or executable, can - be repeated for multiple flags - --version show version information and exit - -h, --help show this help message and exit - -examples: - juliac.jl -vae hello.jl # verbose, build executable and deps - juliac.jl -vae hello.jl prog.c # embed into user defined C program - juliac.jl -qo hello.jl # quiet, build object file only - juliac.jl -vosej hello.jl # build all and copy Julia libs - juliac.jl -vRe hello.jl # fully automated release build -``` - -## Building a shared library -`PackageCompiler` can compile a julia library into a linkable shared library, -built for a specific architecture, with a `C`-compatible ABI which can be -linked against from another program. This can be done either from the julia -api, `build_shared_lib("src/HelloLib.jl", "hello")`, or on the command line, -`$ juliac.jl -vas src/HelloLib.jl`. This will generate a shared library called -`builddir/libhello.{so,dylib,dll}` depending on your system. - -The provided julia file, `src/HelloLib.jl`, is `PackageCompiler`'s entry point -into the library, so it should be the "top level" library file. Any julia code -that it `include`s or `import`s will be compiled into the shared library. - -Note that for a julia function to be callable from `C`, it must be defined with -`Base.@ccallable`, e.g. `Base.@ccallable foo()::Cint = 3`. - -## Building an executable -To compile a Julia program into an executable, you can use either the julia -api, `build_executable("hello.jl", "hello")`, or the command line, `$ -juliac.jl -vae hello.jl`. - -The provided julia file, `hello.jl`, is `PackageCompiler`'s entry point into the -program, and should be the program's "main" file. Any julia code that it -`include`s or `import`s will be compiled into the shared library, which will be -linked against the provided `C` program to create an executable at -`builddir/hello`. - -If you choose to use the default `C` program, your julia code _must_ define -`julia_main` as its entry point. The resultant executable will start by calling -that function, so all of your program's logic should proceed from that -function. For example: - -``` -Base.@ccallable function julia_main(ARGS::Vector{String})::Cint - hello_main(ARGS) # call your program's logic. - return 0 -end -``` - -Please see -[examples/hello.jl](/~https://github.com/JuliaLang/PackageCompiler.jl/blob/master/examples/hello.jl) -for an example Julia program. - -### Notes - -1. The `juliac.jl` script is located in the `PackageCompiler` root - folder (`normpath(Base.find_package("PackageCompiler"), "..", "..")`). - -2. A shared library containing the system image `hello.so`, and a - driver binary `hello` are created in the `builddir` directory. - Running `hello` produces the following output: - -``` - hello, world - sin(0.0) = 0.0 - ┌─────────────────────────────────────────────────┐ - 1 │⠀⠀⠀⠀⠀⠀⠀⡠⠊⠉⠉⠉⠢⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠀⠀⠀⠀⠀⢠⠎⠀⠀⠀⠀⠀⠀⠘⢆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠀⠀⠀⠀⢠⠃⠀⠀⠀⠀⠀⠀⠀⠀⠀⠳⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠀⠀⠀⢠⠃⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠱⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠀⠀⢠⠃⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠳⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠀⢀⠇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢣⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠀⡎⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠼⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠬⢦⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⢤│ - │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⡆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠇│ - │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠘⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡎⠀│ - │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠱⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡞⠀⠀│ - │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠱⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡜⠀⠀⠀│ - │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠱⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡞⠀⠀⠀⠀│ - │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠘⢆⠀⠀⠀⠀⠀⠀⢠⠎⠀⠀⠀⠀⠀│ - -1 │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠑⢄⣀⣀⣀⠔⠁⠀⠀⠀⠀⠀⠀│ - └─────────────────────────────────────────────────┘ - 0 100 -``` - -3. Currently, before another program can call any of the functions defined in - the created shared library, that program must first initialize the julia - runtime. (See - [#53](/~https://github.com/JuliaLang/PackageCompiler.jl/issues/53) for - details.) - - -## Under the hood - -The `juliac.jl` script uses the `--output-o` switch to compile the user -script into object code, and then builds it into the system image -specified by the `-J` switch. This prepares an object file, which is -then linked into a shared library containing the system image and user -code. A driver script such as the one in `program.c` can then be used -to build a binary that runs the Julia code. - -Instead of a driver script, the generated system image can be embedded -into a larger program, see the -[Embedding Julia](https://docs.julialang.org/en/stable/manual/embedding/) -section of the Julia manual. Note that the name of the generated system -image (`"libhello"` for `hello.jl`) is accessible from C in the -preprocessor macro `JULIAC_PROGRAM_LIBNAME`. - -For more information on static Julia compilation see:\ -https://juliacomputing.com/blog/2016/02/09/static-julia.html - -## Side effects - -1. Using `PackageCompiler` makes it impossible to load changed package code automatically - it must be `eval`'ed in from the current session. This becomes a problem when developing packages. +[docs-stable-img]: https://img.shields.io/badge/docs-stable-blue.svg +[docs-stable-url]: https://JuliaComputing.github.io/PackageCompiler.jl/dev diff --git a/deps/build.jl b/deps/build.jl deleted file mode 100644 index bd8254b6..00000000 --- a/deps/build.jl +++ /dev/null @@ -1,73 +0,0 @@ -function verify_gcc(gcc) - try - return success(`$gcc --version`) - catch - return false - end -end - -if Sys.iswindows() - using WinRPM -end - -function build() - gccpath = "" - if isfile("deps.jl") - include("deps.jl") - if verify_gcc(gcc) - @info "GCC already installed and package already built" - return - else - rm("deps.jl") - end - end - - if haskey(ENV, "CC") - if !verify_gcc(`$(ENV["CC"]) -v`) - error("Using compiler override from environment variable CC = $(ENV["CC"]), but unable to run `$(ENV["CC"]) -v`.") - end - gccpath = ENV["CC"] - @info "Using `$gccpath` as C compiler from environment variable CC" - end - - @info "Looking for GCC" - - gccargs = `` - if verify_gcc("cc") - gccpath = "cc" - @info "Using `cc` as C compiler" - elseif Sys.iswindows() - sysroot = joinpath(WinRPM.installdir, "usr", "$(Sys.ARCH)-w64-mingw32", "sys-root") - gccpath = joinpath(sysroot, "mingw", "bin", "gcc.exe") - if !isfile(gccpath) - @info "Could not find GCC, installing using WinRPM" - WinRPM.install("gcc", yes = true) - end - if !isfile(gccpath) - error("Couldn't install gcc via WinRPM") - end - gccargs = `$gccargs --sysroot $sysroot` - @info "Using `gcc` from WinRPM as C compiler" - elseif Sys.isunix() && verify_gcc("gcc") - gccpath = "gcc" - @info "Using `gcc` as C compiler" - end - - if isempty(gccpath) - error(""" - Please make sure to provide a working gcc in your path! - You may need to install GCC. - """ - ) - Sys.isapple() && @info "You can install GCC using Homebrew by `brew install gcc`" - Sys.islinux() && @info """ - You can install GCC using `sudo apt-get install gcc` on Debian, - or use your distro's package manager. - """ - end - open("deps.jl", "w") do io - println(io, "const gcc = ", repr(`$gccpath $gccargs`)) - end -end - -build() diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..567609b1 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/docs/Manifest.toml b/docs/Manifest.toml new file mode 100644 index 00000000..4e5e74bc --- /dev/null +++ b/docs/Manifest.toml @@ -0,0 +1,99 @@ +# This file is machine-generated - editing it directly is not advised + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[DocStringExtensions]] +deps = ["LibGit2", "Markdown", "Pkg", "Test"] +git-tree-sha1 = "88bb0edb352b16608036faadcc071adda068582a" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.8.1" + +[[Documenter]] +deps = ["Base64", "Dates", "DocStringExtensions", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] +git-tree-sha1 = "d497bcc45bb98a1fbe19445a774cfafeabc6c6df" +uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +version = "0.24.5" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.0" + +[[LibGit2]] +deps = ["Printf"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[PackageCompiler]] +deps = ["Libdl", "Pkg", "UUIDs"] +path = ".." +uuid = "dffaa6cc-da53-48e5-b007-4292dfcc27f1" +version = "0.1.0" + +[[Parsers]] +deps = ["Dates", "Test"] +git-tree-sha1 = "d112c19ccca00924d5d3a38b11ae2b4b268dda39" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "0.3.11" + +[[Pkg]] +deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[Test]] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/docs/Project.toml b/docs/Project.toml new file mode 100644 index 00000000..e8d80518 --- /dev/null +++ b/docs/Project.toml @@ -0,0 +1,3 @@ +[deps] +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +PackageCompiler = "dffaa6cc-da53-48e5-b007-4292dfcc27f1" diff --git a/docs/make.jl b/docs/make.jl new file mode 100644 index 00000000..2e0b76c5 --- /dev/null +++ b/docs/make.jl @@ -0,0 +1,35 @@ +using Documenter, PackageCompiler + +makedocs( + format = Documenter.HTML( + prettyurls = "deploy" in ARGS, + ), + sitename = "PackageCompiler", + pages = [ + "Home" => "index.md", + + "Manual" => [ + "sysimages.md" + "apps.md" + ], + + "Examples" => [ + "examples/ohmyrepl.md", + "examples/plots.md", + ], + + "PackageCompiler - the manual way" => [ + "devdocs/intro.md", + "devdocs/sysimages_part_1.md", + "devdocs/binaries_part_2.md", + "devdocs/relocatable_part_3.md", + ], + + "References" => "refs.md", + ] +) + +deploydocs( + repo = "github.com/JuliaLang/PackageCompiler.jl.git", + push_preview = true, +) diff --git a/docs/src/apps.md b/docs/src/apps.md new file mode 100644 index 00000000..01a62bd5 --- /dev/null +++ b/docs/src/apps.md @@ -0,0 +1,255 @@ +# Apps + +With an "app" we here mean a "bundle" of files where one of these files is an +executable and where this bundle can be sent to another machine while still allowing +the executable to run. + +Use-cases for Julia-apps are for example when one wants to provide some kind of +functionality where the fact that the code was written in Julia is just an +implementation detail and where requiring the user to download and use Julia to +run the code would be a distraction. There is also no need to provide the +original Julia source code for apps since everything gets baked into the +sysimage. + + +## Relocatability + +Since we want to send the app to other machines the app we create must be +"relocatable". With an app being relocatable we mean it does not rely on +specifics of the machine where the app was created. Relocatability is not an +absolute measure, most apps assume some properties of the machine they will run +on, like what operating system is installed and the presence of graphics +drivers if one wants to show graphics. On the other hand, embedding things into +the app that is most likely unique to the machine, such as absolute paths to +libraries, means that the application almost surely will not run properly on +another machine. + +For something to be relocatable, everything that it depends on must also be +relocatable. In the case of an app, the app itself and all the Julia packages +it depends on must also relocatable. This is a bit of an issue because the +Julia package ecosystem has rarely given much thought to relocatability +since creating "apps" has not been common. + +The main problem with relocatability of Julia packages is that many packages +are encoding fundamentally non-relocatable information *into the source code*. +As an example, many packages tend to use a `build.jl` file (which runs when the +package is first installed) that looks something like: + +```julia +lib_path = find_library("libfoo") +write("deps.jl", "const LIBFOO_PATH = $(repr(lib_path))") +``` + +The main package file then contains: + +```julia +module Package + +if !isfile("../build/deps.jl") + error("run Pkg.build(\"Package\") to re-build Package") +end +include("../build/deps.jl") +function __init__() + libfoo = Libdl.dlopen(LIBFOO_PATH) +end + +... + +end # module +``` + +The absolute path to `lib_path` that `find_library` found is thus effectively +included into the source code of the package. Arguably, the whole build system +in Julia is inherently non-relocatable because it runs when the package is +being installed which is a concept that does not make sense when distributing +an app. + +Some packages do need to call into external libraries and use external binaries +so the question then arises: "how are these packages supposed to do this in a +relocatable way?" The answer is to use the "artifact system" introduced in +Julia 1.3, and described in the following [blog +post](https://julialang.org/blog/2019/11/artifacts). The artifact system is a +declarative way of downloading and using "external files" like binaries and +libraries. How this is used in practice is described later. + + +## Creating an app + +The source of an app is a package with a project and manifest file. +It should define a function with the signature + +```julia +function julia_main()::Cint + # do something based on ARGS? + return 0 # if things finished successfully +end +``` + +which will be the entry point of the app (the function that runs when the +executable in the app is run). A skeleton of an app to start working from can +be found at +/~https://github.com/JuliaLang/PackageCompiler.jl/tree/master/examples/MyApp. + +Regarding relocatability, PackageCompiler provides a function +[`audit_app(app_dir::String)`](@ref) that tries to find common problems with +relocatability in the app. + +The app is then compiled using the [`create_app`](@ref) function that takes a +path to the source code of the app and the destination where the app should be +compiled to. This will bundle all required libraries for the app to run on +another machine where the same Julia that created the app can run. As an +example, in the code snippet below, the example app linked above is compiled and run: + +``` +~/PackageCompiler.jl/examples +❯ julia -q --project + +julia> using PackageCompiler + +julia> create_app("MyApp", "MyAppCompiled") +[ Info: PackageCompiler: creating base system image (incremental=false), this might take a while... +[ Info: PackageCompiler: creating system image object file, this might take a while... + +julia> exit() + +~/PackageCompiler.jl/examples +❯ MyAppCompiled/bin/MyApp +ARGS = ["foo", "bar"] +Base.PROGRAM_FILE = "MyAppCompiled/bin/MyApp" +... +Hello, World! + +Running the artifact +The result of 2*5^2 - 10 == 40.000000 +unsafe_string((Base.JLOptions()).image_file) = "/Users/kristoffer/PackageCompiler.jl/examples/MyAppCompiled/bin/MyApp.dylib" +Example.domath(5) = 10 +``` + +The resulting executable is found in the `bin` folder in the compiled app +directory. The compiled app directory `MyAppCompiled` could now be put into an +archive and sent to another machine or an installer could be wrapped around the +directory, perhaps providing a better user experience than just an archive of +files. + +### Precompilation + +In the same way as files for precompilation could be given when creating +sysimages, the same keyword arguments are used to add precompilation to apps. + +### Incremental vs non-incremental sysimage + +In the section about creating sysimages, there was a short discussion about +incremental vs non-incremental sysimages. In short, an incremental sysimage is +built on top of another sysimage, while a non-incremental is created from +scratch. For sysimages, it makes sense to use an incremental sysimage built on +top of Julia's default sysimage since we wanted the benefit of having a responsive +REPL that it provides. For apps, this is no longer the case, the sysimage is +not meant to be used when working interactively, it only needs to be +specialized for the specific app. Therefore, by default, `incremental=false` is +used for `create_app`. If, for some reason, one wants an incremental sysimage, +`incremental=true` could be passed to `create_app`. With the example app, a +non-incremental sysimage is about 70MB smaller than the default sysimage. + +### Filtering stdlibs + +By default, all standard libraries are included in the sysimage. It is +possible to only include those standard libraries that the project needs. This +is done by passing the keyword argument `filter_stdlibs=true` to `create_app`. +This causes the sysimage to be smaller, and possibly load faster. The reason +this is not the default is that it is possible to "accidentally" depend on a +standard library without it being reflected in the Project file. For example, +it is possible to call `rand()` from a package without depending on Random, +even though that is where the method is defined. If Random was excluded from +the sysimage that call would then error. The same thing is true for e.g. matrix +multiplication, `rand(3,3) * rand(3,3)` requires both the standard libraries +`LinearAlgebra` and `Random` This is because these standard libraries do +"type-piracy" so just loading those packages can cause code to change behavior. + +Nevertheless, the option is there to use. Just make sure to properly test the +app with the resulting sysimage. + + +### Artifacts + +The way to depend on external libraries or binaries when creating apps is by +using the [artifact system](https://julialang.github.io/Pkg.jl/v1/artifacts/). +PackageCompiler will bundle all artifacts needed by the project, and set up +things so that they can be found during runtime on other machines. + +The example app uses the artifact system to depend on a very simple toy binary +that does some simple arithmetic. It is instructive to see how the [artifact +file](/~https://github.com/JuliaLang/PackageCompiler.jl/blob/master/examples/MyApp/Artifacts.toml) +is [used in the source](/~https://github.com/JuliaLang/PackageCompiler.jl/blob/d722a3d91abe328ebd239e2f45660be35263ebe1/examples/MyApp/src/MyApp.jl#L7-L8). + +### Reverse engineering the compiled app + +While the created app is relocatable and no source code is bundled with it, +there are still some things about the build machine and the source code that +can be "reverse engineered". + +#### Absolute paths of build machine + +Julia records the paths and line-numbers for methods when they are getting +compiled. These get cached into the sysimage and can be found e.g. by dumping +all strings in the sysimage: + +``` +~/PackageCompiler.jl/examples/MyAppCompiled/bin +❯ strings MyApp.so | grep MyApp +MyApp +/home/kc/PackageCompiler.jl/examples/MyApp/ +MyApp +/home/kc/PackageCompiler.jl/examples/MyApp/src/MyApp.jl +/home/kc/PackageCompiler.jl/examples/MyApp/src +MyApp.jl +/home/kc/PackageCompiler.jl/examples/MyApp/src/MyApp.jl +``` + +This is a problem that the Julia standard libraries themselves have: + +```julia-repl +julia> @which rand() +rand() in Random at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.3/Random/src/Random.jl:256 +``` + +#### Using reflection and finding lowered code + +There is nothing preventing someone from starting Julia with the sysimage that +comes with the app. And while the source code is not available one can read +the "lowered code" and use reflection to find things like the name of fields in +structs and global variables etc: + +```julia-repl +~/PackageCompiler.jl/examples/MyAppCompiled/bin kc/docs_apps* +❯ julia -q -JMyApp.so +julia> MyApp = Base.loaded_modules[Base.PkgId(Base.UUID("f943f3d7-887a-4ed5-b0c0-a1d6899aa8f5"), "MyApp")] +MyApp + +julia> names(MyApp; all=true) +10-element Array{Symbol,1}: + Symbol("#eval") + Symbol("#include") + Symbol("#julia_main") + Symbol("#real_main") + :MyApp + :eval + :include + :julia_main + :real_main + :socrates + +julia> @code_lowered MyApp.real_main() +CodeInfo( +1 ─ %1 = MyApp.ARGS +│ value@_2 = %1 +│ %3 = Base.repr(%1) +│ Base.println("ARGS = ", %3) +│ value@_2 +│ %6 = Base.PROGRAM_FILE +│ value@_3 = %6 +│ %8 = Base.repr(%6) +│ Base.println("Base.PROGRAM_FILE = ", %8) +│ value@_3 +│ %11 = MyApp.DEPOT_PATH +``` + diff --git a/docs/src/devdocs/app.png b/docs/src/devdocs/app.png new file mode 100644 index 00000000..fdcc7e40 Binary files /dev/null and b/docs/src/devdocs/app.png differ diff --git a/docs/src/devdocs/appexe.png b/docs/src/devdocs/appexe.png new file mode 100644 index 00000000..949f7b6d Binary files /dev/null and b/docs/src/devdocs/appexe.png differ diff --git a/docs/src/devdocs/binaries_part_2.md b/docs/src/devdocs/binaries_part_2.md new file mode 100644 index 00000000..889b03dd --- /dev/null +++ b/docs/src/devdocs/binaries_part_2.md @@ -0,0 +1,239 @@ +# [Creating a binary from Julia code](@id man-tutorial-binary) + +This section targets how to build an executable based on the custom sysimage so +that it can be run without having to explicitly start a Julia session. + +## Interacting with Julia through `libjulia`. + +The way to interact with Julia without using the Julia executable itself is by +calling into the Julia runtime library (`libjulia`) from a C program. A quite +detail set of docs for how this is done can be found at the [embedding chapter +in the Julia manual](https://docs.julialang.org/en/v1/manual/embedding/) and it +is recommended to read before reading on. Since this is supposed to highlight +the interals of PackageCompiler, will not use the conveniences shown in that +section (e.g. the `julia-config.jl` script) but it is good to know they exist. + +A rough outline of the steps we will take to create an executable are: + +- Create our Julia app with a `Base.@ccallable` entry-point which means the Julia + function can be called directly from C. +- Create a custom sysimage to reduce latency (this is pretty much just doing + part 1) and to hold the C-callable function from the first step. +- Write an embedding wrapper in C that loads our custom sysimage, does some + initialization and calls the entry point in the script. + +## A toy application + +To have something concrete to work with we will create a very simple +application. Keeping with the spirit of CSV parsing, we will create a small +app that parses a list of CSV files given as arguments to the app and prints +the size of the parsed result. The code for the app (`MyApp.jl`) is shown +below: + +```julia +module MyApp + +using CSV + +Base.@ccallable function julia_main()::Cint + try + real_main() + catch + Base.invokelatest(Base.display_error, Base.catch_stack()) + return 1 + end + return 0 +end + +function real_main() + for file in ARGS + if !isfile(file) + error("could not find file $file") + end + df = CSV.read(file) + println(file, ": ", size(df, 1), "x", size(df, 2)) + end +end + +if abspath(PROGRAM_FILE) == @__FILE__ + real_main() +end + +end # module +``` + +The function `julia_main` has been annotated with `Base.@ccallable` which means +that a function with the unmangled name will appear in the sysimage. This +function is just a small wrapper function that calls out to `real_main` which +does the actual work. All the code that is executed is put inside a try-catch +block since the error will otherwise happen in the C-code where the backtrace +is not very good + +To facilitate testing, we [check if the file was directly +executed](https://docs.julialang.org/en/v1/manual/faq/#How-do-I-check-if-the-current-file-is-being-run-as-the-main-script?-1) +and in that case, run the main function. We can test (and time) the script on +the sample CSV file [from the first tutorial](@ref man-tutorial-sysimage) + +``` +❯ time julia MyApp.jl FL_insurance_sample.csv +FL_insurance_sample.csv: 36634x18 +julia MyApp.jl FL_insurance_sample.csv 12.51s user 0.38s system 104% cpu 12.385 total +``` + +## Create the sysimage + +As in the previous tutorial, we do a "sample run" of our app to record what +functions end up getting compiled. Here, we simply run the app on the sample +CSV file since that should give good "coverage": + +``` +julia --startup-file=no --trace-compile=app_precompile.jl MyApp.jl "FL_insurance_sample.csv" +``` + +The `create_sysimage.jl` script look similar to before with the exception that +we added an include of the app file inside the anonymous module where the +precompiliation statements are evaluated in: + +```julia +Base.init_depot_path() +Base.init_load_path() + +@eval Module() begin + Base.include(@__MODULE__, "MyApp.jl") + for (pkgid, mod) in Base.loaded_modules + if !(pkgid.name in ("Main", "Core", "Base")) + eval(@__MODULE__, :(const $(Symbol(mod)) = $mod)) + end + end + for statement in readlines("app_precompile.jl") + try + Base.include_string(@__MODULE__, statement) + catch + # See julia issue #28808 + Core.println("failed to compile statement: ", statement) + end + end +end # module + +empty!(LOAD_PATH) +empty!(DEPOT_PATH) +``` + +The sysimage is then created as before: + +``` +❯ julia --startup-file=no -J"/home/kc/julia/lib/julia/sys.so" --output-o sys.o custom_sysimage.jl + +❯ gcc -shared -o sys.so -fPIC -Wl,--whole-archive sys.o -Wl,--no-whole-archive -L"/home/kc/julia/lib" -ljulia +``` + +### Windows-specific flags + +For Windows we need to tell the linker to export all symbols via the flag `-Wl,--export-all-symbols`. +Otherwise, the linker will fail to find `julia_main` when we build the executable. + +## Creating the executable + +### Embedding code + +The embedding script is the "driver" of the app. It initializes the julia +runtime, does some other initialization, calls into our `julia_main` and then +does some cleanup when it returns. We can borrow a lot for this embedding +script from the embedding manual there are however some things we ne +ed to set up +"manually" that Julia usually does by itself when starting Julia. This +includes assigning the `PROGRAM_FILE` variable as well as updating `Base.ARGS` +to contain the correct values. The script `MyApp.c` ends up looking like: + +```c +// Standard headers +#include +#include + +// Julia headers (for initialization and gc commands) +#include "uv.h" +#include "julia.h" + +JULIA_DEFINE_FAST_TLS() + +// Forward declare C prototype of the C entry point in our application +int julia_main(); + +int main(int argc, char *argv[]) +{ + uv_setup_args(argc, argv); + + // initialization + libsupport_init(); + + // JULIAC_PROGRAM_LIBNAME defined on command-line for compilation + jl_options.image_file = JULIAC_PROGRAM_LIBNAME; + julia_init(JL_IMAGE_JULIA_HOME); + + // Initialize Core.ARGS with the full argv. + jl_set_ARGS(argc, argv); + + // Set PROGRAM_FILE to argv[0]. + jl_set_global(jl_base_module, + jl_symbol("PROGRAM_FILE"), (jl_value_t*)jl_cstr_to_string(argv[0])); + + // Set Base.ARGS to `String[ unsafe_string(argv[i]) for i = 1:argc ]` + jl_array_t *ARGS = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("ARGS")); + jl_array_grow_end(ARGS, argc - 1); + for (int i = 1; i < argc; i++) { + jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); + jl_arrayset(ARGS, s, i - 1); + } + + // call the work function, and get back a value + int ret = julia_main(); + + // Cleanup and gracefully exit + jl_atexit_hook(ret); + return ret; +} +``` + +## Building the executable + +We now have all the pieces needed to build the executable; a sysimage and a driver script. +It is compiled as: + +``` +❯ gcc -DJULIAC_PROGRAM_LIBNAME=\"sys.so\" -o MyApp MyApp.c sys.so -O2 -fPIE \ + -I'/home/kc/julia/include/julia' \ + -L'/home/kc/julia/lib' \ + -ljulia \ + -Wl,-rpath,'/home/kc/julia/lib:$ORIGIN' +``` + +where we have added an `rpath` entry into the executable so that the julia +library can be found at runtime as well as the `sys.so` library ($ORIGIN means +to look in the same folder as the binary for shared libraries). + +``` +❯ time ./MyApp FL_insurance_sample.csv +FL_insurance_sample.csv: 36634x18 +./MyApp FL_insurance_sample.csv 0.19s user 0.09s system 242% cpu 0.115 total + +❯ ./MyApp non_existing.csv +ERROR: could not find file non_existing.csv +Stacktrace: + [1] error(::String) at ./error.jl:33 + [2] real_main() at /home/kc/MyApp/MyApp.jl:21 + [3] julia_main() at /home/kc/MyApp/MyApp.jl:7 +``` + +### macOS considerations + +On macOS, instead of `$ORIGIN` for the `rpath`, use `@executable_path`. + +### Windows considerations + +On Windows, it is recommended to increase the size of the stack from the +default 1 MB to 8MB which can be done by passing the `-Wl,--stack,8388608` +flag. Windows doesn't have (at least in an as simple way as Linux and macOS) +the concept of `rpath`. The goto solution is to either set the `PATH` +environment variable to the Julia `bin` folder or alternatively copy paste all +the libraries in the Julia `bin` folder so they sit next to the executable. + diff --git a/docs/src/devdocs/intro.md b/docs/src/devdocs/intro.md new file mode 100644 index 00000000..629cb31d --- /dev/null +++ b/docs/src/devdocs/intro.md @@ -0,0 +1,24 @@ +# Introduction + +This part of the documentation contains a set of tutorials aimed to teach how +PackageCompiler works internally. This is done by going through some examples +of manually creating sysimages and apps, mostly from the command line. +By knowing the internals of PackageCompiler you can more easily figure out +root causes of problems and help others. The inner functionality of PackageCompiler +is actually quite simple. There are a few julia commands and compiler invocations +that everything is built around, the rest is mostly scaffolding. + +[Part 1](@ref man-tutorial-sysimage) focuses on how to build a local system +image to reduce package load times and reduce the latency that can occur when +calling a function for the first time. [Part 2](@ref man-tutorial-binary) +targets how to build an executable based on the custom sysimage so that it can +be run without having to explicitly start a Julia session. [Part 3](@ref man-tutorial-reloc) +details how to bundle that executable together with the Julia libraries and +other files needed so that the bundle can be sent to and run on a different +system where Julia might not be installed. These functionalities are exposed +from PackageCompiler as [`create_sysimage`](@ref) and [`create_app`](@ref). + +It should be noted that there is some usage of non-documented Julia functions +and flags. They have not been changed for quite a long time (and are unlikely +to change too much in the future), but some care should be taken. + diff --git a/docs/src/devdocs/relocatable_part_3.md b/docs/src/devdocs/relocatable_part_3.md new file mode 100644 index 00000000..3670eb3a --- /dev/null +++ b/docs/src/devdocs/relocatable_part_3.md @@ -0,0 +1,357 @@ +# [Relocatable apps](@id man-tutorial-reloc) + +In the previous tutorials, we created a custom sysimage and a binary (app) that +did some simple CSV parsing with an (depending on the exact demands) acceptable +latency (time until the app starts doing real work). However, trying to send +this executable to another machine will fail spectacularly. This tutorial +outlines how to create and package a bundle of files into an app that we can +send to other machines and have them run, without for example, requiring Julia +itself to be installed, and without having to ship the source code of the app. + +The tutorial will not deal with any kind of file size optimization or "tree +shaking" as it is sometimes called. + +## Why is the built executable in the previous tutorial non-relocatable? + +With relocatability, we mean the ability of being able to send e.g. an +executable (or a bundle of files including an executable, here called an app) +to another machine and have it run there without too many assumptions of the +state of the other machine. Relocatability is not an absolute measure, most +apps assume some properties of the machine they will run on (like graphics +drivers if one want to show graphics) but other (implicit) assumptions, like +embedding absolute paths into source code would make the app almost completely +non-relocatable since that absolute path is unlikely to exist on another +machine. The goal here is to make our app relocatable enough such that if we +could install and run the same Julia as we use to build the app on the other +machine, then the app should also run on that machine (with exceptions if some +of our dependencies impose extra requirements on the machine). + +So what is causing our executable that we built in the previous tutorial to not +be relocatable? Firstly, our sysimage relies on `libjulia` which we currently +load from the Julia directory and, in addition, `libjulia` itself relies on +other libraries (like LLVM) to work. And secondly, the packages we embedded in +the sysimage might have encoded assumptions about the current system into their +code. + +The first problem is quite easy to fix while the second one is harder since +some popular packages that we might want to use as dependencies are inherently +non-relocatable. There is nothing to do about that except try to fix these +packages. + +For now, we will ignore the problem of packages not being relocatable by only +using a small dependency that we know does not have a relocatability problem. +Later in the blog post, we will revisit this and discuss more in-depth what +makes a package non-relocatable and how to fix this, even if the package needs +things like external libraries or binaries (spoiler alert: it is using the +artifact system presented in [the blog about +artifacts](https://julialang.org/blog/2019/11/artifacts). + +## A toy app + +The package we used in the previous examples to create a sysimage and +executable was CSV.jl. Now, to simplify things, we will only use a very simple +package with no relocatability problems that also has no dependencies. The +app will take some input on stdin and print it out with color to the terminal +using the [Crayons.jl](/~https://github.com/KristofferC/Crayons.jl) package. + +When we add the Crayons.jl package we use a separate project to encapsulate +things better by creating a new project in the app directory: + +``` +~/MyApp +❯ julia -q --project=. + +julia> using Pkg; Pkg.add("Crayons") + Updating registry at `~/.julia/registries/General` + Updating git-repo `/~https://github.com/JuliaRegistries/General.git` + Resolving package versions... + Updating `~/MyApp/Project.toml` + [a8cc5b0e] + Crayons v4.0.1 + Updating `~/MyApp/Manifest.toml` + [a8cc5b0e] + Crayons v4.0.1 +``` + +The code for the app itself is quite simple: + +```julia +module MyApp +using Crayons + +Base.@ccallable function julia_main()::Cint + try + real_main() + catch + Base.invokelatest(Base.display_error, Base.catch_stack()) + return 1 + end + return 0 +end + +function real_main() + Crayons.FORCE_COLOR[] = true + color = :red + for arg in ARGS + if !(arg in ["red", "green", "blue"]) + error("invalid color $arg") + end + color = Symbol(arg) + end + c = Crayon(foreground=color) + r = Crayon(reset=true) + while !eof(stdin) + txt = String(readavailable(stdin)) + print(r, c, txt, r) + end + return 0 +end +if abspath(PROGRAM_FILE) == @__FILE__ + real_main() +end +end # module +``` + +It got the same high-level structure as the previous app in the earlier parts. +The exact details are not so interesting but here a color is set based on the +command-line arguments and the `stdin` is written to `stdout` with that color. +We can see some usage of it: + +![](app.png) + + +## Precompilation and sysimage + +As in part 1 we generate precompilation statements and create a system image. +When recording precompilation statements and creating the sysimage, we make +sure to use the `--project` flag to use the packages declared in the local +project: + +``` +~/MyApp +❯ echo "Hello, this is some stdin" | julia --project --startup-file=no --trace-compile=app_precompile.jl MyApp.jl green +``` + +The `.o` file is then created with the same `generate_sysimage.jl` file as in part 2: + +``` +~/MyApp +❯ gcc -shared -o sys.so -Wl,--whole-archive sys.o -Wl,--no-whole-archive -L"/home/kc/julia/lib" -ljulia +``` + +And then the sysimage is linked: + +``` +~/MyApp +❯ gcc -shared -o sys.so -Wl,--whole-archive sys.o -Wl,--no-whole-archive -L"/home/kc/julia/lib" -ljulia +``` + +Before moving on and creating the executable, we need to think about what other +files we need for the app and the file structure we want. + +## File structure for our app bundle + +We already mentioned that `libjulia` has some dependencies. Using `ldd`, we +can see the dependencies and where the dynamic linker would load them from: + +``` +~/julia/lib +❯ ldd libjulia.so + linux-vdso.so.1 (0x00007ffec63c3000) + libLLVM-6.0.so => /home/kc/julia/lib/./julia/libLLVM-6.0.so (0x00007f925ef13000) + libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f925eeea000) + librt.so.1 => /lib/x86_64-linux-gnu/librt.so.1 (0x00007f925eedf000) + libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f925eebc000) + libstdc++.so.6 => /home/kc/julia/lib/./julia/libstdc++.so.6 (0x00007f925eb3e000) + libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f925e9ef000) + libgcc_s.so.1 => /home/kc/julia/lib/./julia/libgcc_s.so.1 (0x00007f925e7d5000) + libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f925e5e4000) + /lib64/ld-linux-x86-64.so.2 (0x00007f9262356000) +``` + +So some libraries would be loaded from the (`libdl`, `librt`) system itself, +and some are bundled with Julia (`libLLVM`, `libstdc++` etc) in the `julia` +folder inside `lib`. The reason the dynamic linker finds the libraries in the +subfolder is due to the `rpath` which can be seen with `objdump`: + +``` +❯ objdump -x libjulia.so |grep RPATH + RPATH $ORIGIN/julia:$ORIGIN +``` + +However, these are not the only libraries Julia (and its standard libraries) +need. Libraries can also be dynamically opened at runtime (with +[dlopen](https://linux.die.net/man/3/dlopen)). For now, we will just bring all +the libraries in `lib/julia` along (excluding the sysimage since we will use +our sysimage). + +The plan is that on macOS and Linux the files are structured as: + +``` +├── bin +│   └── MyApp [executable] +│   └── sys.so +└── lib + ├── julia + │   ├── libamd.so -> libamd.so.2.4.6 + │   ├── libamd.so.2 -> libamd.so.2.4.6 + │   ├── libamd.so.2.4.6 + │   ├── libcamd.so -> libcamd.so.2.4.6 + ... ... + │   └── libz.so.1.2.11 + ├── libjulia.so -> libjulia.so.1.3 + ├── libjulia.so.1 -> libjulia.so.1.3 + └── libjulia.so.1.3 +``` + +On Windows, we will just store everything in `bin` due to no convenient way of using `RPATH`. + +We create a new folder `lib` and copy the libraries into it (and remove the +sysimage, since we will create cusom sysimage anyway): + +``` +~/MyApp +❯ mkdir lib + +~/MyApp +❯ cp -r ~/julia/lib/ . + +~/MyApp +❯ rm lib/julia/sys.so +``` + +## Creating the binary and the bundle + +With some tweaks to the `rpath` entry so that the executable can find +`libjulia` the executable is created in the same way as in the previous tutorial. + +``` +~/MyApp +❯ gcc -DJULIAC_PROGRAM_LIBNAME=\"sys.so\" -o MyApp MyApp.c sys.so -O2 -I'/home/kc/julia/include/julia' -L'/home/kc/julia/lib' -fpie -Wl,-rpath,'$ORIGIN:$ORIGIN/../lib' -ljulia +``` + +We then finally move the executable and the sysimage to the `bin` folder: + +``` +~/MyApp +❯ mkdir bin + +~/MyApp +❯ cp MyApp sys.so bin/ +``` + +![](appexe.png) + +The final bundle of our relocatable app is then created by putting the `bin` +and `lib` folders into an archive: + + +``` +~/MyApp +❯ mkdir MyApp + +~/MyApp +❯ cp MyApp sys.so bin/ + +~/MyApp +❯ tar czvf MyApp.tar.gz MyApp +MyApp/ +MyApp/bin/ +MyApp/bin/MyApp +MyApp/bin/sys.so +MyApp/lib/ +MyApp/lib/julia/ +... +``` + +### macOS consideration + +On macOS we need to run `install_name_tool` to make it use the `rpath` entries +which is done by executing: + +``` +install_name_tool -change sys.so @rpath/sys.so MyApp` +``` + + +## Information about source code and build machine state stored in resulting app + +It should be noted that there is some state from the machine where the sysimage +and binary is built that can be observed and the original source code. Using +the [`strings`](https://linux.die.net/man/1/strings) application we can see what strings are embedded in +an executable or library. Running it and grepping for some relevant substrings +we can see that a bunch of absolute paths are stored inside the sysimage: + +``` +~/MyApp/MyApp/lib/julia +❯ strings sys.so | grep /home/kc +/home/kc/.julia/packages/Crayons/P4fls/src/downcasts.jl +/home/kc/.julia/packages/Crayons/P4fls/src/crayon.jl +/home/kc/.julia/packages/Crayons/P4fls/src/crayon_stack.jl +/home/kc/MyApp/MyApp.jl +/home/kc/.julia/packages/Crayons/P4fls/src/Crayons.jl +/home/kc/.julia/packages/Crayons/P4fls/src/crayon_wrapper.jl +/home/kc/.julia/packages/Crayons/P4fls/src/test_prints.jl +/home/kc/.julia/packages/Crayons/P4fls/src/macro.jl +``` + +In addition, when we print the stacktrace upon failure in the main function, +we also leak absolute paths of the build machine: + +``` +~/MyApp +❯ MyApp/bin/MyApp purple +ERROR: invalid color purple +Stacktrace: + [1] error(::String) at ./error.jl:33 + [2] real_main() at /home/kc/MyApp/MyApp.jl:20 + [3] julia_main() at /home/kc/MyApp/MyApp.jl:6 +``` + +This could be avoided by not printing stacktraces and perhaps even binary +patching out the paths in the sysimage (not covered in this blog post). + +The lowered code can also be read by loading the sysimage and using e.g. `@code_lowered` +on methods. + +## Relocatability of Julia packages + +The main problem with relocatability of Julia packages is that many packages +are encoding fundamentally non-relocatable information *into the source code*. +As an example, many packages tend to use a `build.jl` file (which runs when the +package is installed) that looks something like: + +```julia +lib_path = find_library("libfoo") +write("deps.jl", "const LIBFOO_PATH = $(repr(lib_path))") +``` + +The main package file then contains + +```julia +if !isfile("../build/deps.jl") + error("run Pkg.build(\"Package\") to re-build Package") +end +include("../build/deps.jl") + +function __init__() + libfoo = Libdl.dlopen(LIBFOO_PATH) +end +``` + +The problem here is that `deps.jl` contains an absolute path to the library and +this gets encoded into the source code of the package. If we would store the +package in the sysimage and try use it on another system, it would error when +initialized since the `LIBFOO_PATH` variable is not valid on the other system. +However, sometimes we need to bundle libraries and data files since the package +uses them. Fortunately, there is a plan for that which can be seen in the [blog +post about artifacts](https://julialang.org/blog/2019/11/artifacts). + +The idea is that with the new artifact system a file (`Artifacts.toml`), a +package can declaratively list external libraries and files that it needs. In +addition, the artifact system provides a way to find these files at runtime in +a deterministic way. It is then possible to make sure that all artifacts needed +for the package is bundled in the app and can also be found by the package +during runtime. + +The details are left out here since they become a bit technical but it should +give some incentive to switch to the artifact system. + diff --git a/docs/src/devdocs/sysimages_part_1.md b/docs/src/devdocs/sysimages_part_1.md new file mode 100644 index 00000000..40a2022f --- /dev/null +++ b/docs/src/devdocs/sysimages_part_1.md @@ -0,0 +1,473 @@ +# [Creating a sysimage](@id man-tutorial-sysimage) + +## Julia's compilation model and sysimages + +Julia is a JIT-compiled language. More specifically, functions are compiled +just before getting executed. A more suitable description of the Julia +compilation model might, therefore, be Just-Ahead-of-Time (JAOT) compilation. +The term JIT is sometimes used to describe the compilation model where code is +dynamically recompiled based on runtime performance data, which Julia does not +do. At the same time, Julia comes with a lot of built-in functionality +including several standard libraries. If all this built-in functionality would +need to be parsed, type inferred and compiled every time Julia started, the +startup-time would be longer than reasonable. Therefore, Julia bundles +something called a "sysimage" which is a [shared +library](https://en.wikipedia.org/wiki/Library_(computing)#Shared_libraries) +where (roughly) the state of a running Julia session has been stored +(serialized). When Julia starts, this sysimage gets loaded, which is a quite +quick process (50ms on the author's machine), and all the cached compiled code +can immediately be used, without requiring any compilation. + +## Custom sysimages + +There are cases where one wants to generate a custom sysimage for a similar +reason as to why Julia bundles one: to reduce time from Julia start until the +program is executing. The time from startup to execution is here denoted as +"latency" and we want to minimize the latency of our program. A drawback of +putting a package inside the sysimage is that it becomes "frozen" at the +particular version it was, when it got put into the sysimage. In addition, all +the dependencies of the package put into the sysimage will be frozen in the +same manner. In particular, it will no longer be updated like normal packages +when using the package manager. In some cases, other ways of reducing latency +might be preferable, for example, using [Revise.jl](/~https://github.com/timholy/Revise.jl) + +## Example workload + +To have something concrete to work with, let's assume we have a small script +that reads a CSV-file and computes some statistics on it. As an example, we +will use a sample CSV file containing Florida insurance data, which can be +downloaded [from +here](http://spatialkeydocs.s3.amazonaws.com/FL_insurance_sample.csv.zip). + +One way of loading this file into Julia is by using the `CSV.jl` package. We +can install `CSV.jl` using the Julia package manager `Pkg` as: + +```julia-repl +julia> import Pkg; Pkg.add("CSV") + Resolving package versions... + Updating `~/.julia/environments/v1.3/Project.toml` + [336ed68f] + CSV v0.5.13 + Updating `~/.julia/environments/v1.3/Manifest.toml` + [no changes] +``` + +When a package is loaded for the first time it gets "precompiled": + +```julia-repl +julia> @time using CSV +[ Info: Precompiling CSV [336ed68f-0bac-5ca0-87d4-7b16caf5d00b] + 13.321758 seconds (2.69 M allocations: 151.302 MiB, 0.05% gc time) +``` + +The term "precompiled" can be a bit misleading since there is no native +compiled code cached in the precompilation file. Julia is dynamically typed so +it is not obvious what types to compile the different methods for. + +Even with `CSV` "precompiled", there is a still some loading time, but it is +significantly lower: + +```julia-repl +julia> @time using CSV + 0.694224 seconds (1.90 M allocations: 114.210 MiB) +``` + +Let's load the sample CSV file: + + +```julia-repl +julia> @time CSV.read("FL_insurance_sample.csv"); +9.264898 seconds (37.17 M allocations: 2.278 GiB, 3.90% gc time)1 +``` + +That's is quite a long time to read a smallish CSV file. One way to check +the compilation overhead is by running the function again: + +```julia-repl +julia> @time CSV.read("FL_insurance_sample.csv"); + 0.083543 seconds (423 allocations: 34.695 KiB) +``` + +So clearly, the first call to the function is dominated by compilation time. +In many cases, this is not a problem in practice since often one wants to parse +multiple CSV files such that the overhead will become negligible or one keeps a +Julia session open for a longer time so that the compiled version of the +function is still in memory. + +However, since the end goal of this blog series is to create an executable that +can be distributed we want to try to avoid as much runtime compilation +(latency) as possible. + +## Creating a custom sysimage + +If we time the loading of a standard library, it is clear that it is "cached" +somehow since the time to load it is so short: + +```julia-repl +julia> @time using Dates + 0.000816 seconds (1.25 k allocations: 65.625 KiB) +``` + +Since `Dates` is a standard library it comes bundled in the system image. In +fact, `Dates` is already "loaded" when starting Julia. The effect of running +`using Dates` just makes the module available in the `Main` module namespace +which is what the REPL evaluates in. + +Delving into some internals, there is a dictionary in `Base` that keeps track +of all loaded modules: + +```julia-repl +julia> Base.loaded_modules +Dict{Base.PkgId,Module} with 33 entries: + SHA [ea8e919c-243c-51af-8825-aaa63cd721ce] => SHA + Profile [9abbd945-dff8-562f-b5e8-e1ebf5ef1b79] => Profile + Dates [ade2ca70-3891-5945-98fb-dc099432e06a] => Dates + Mmap [a63ad114-7e13-5084-954f-fe012c677804] => Mmap +... +``` + +and we can here see the `Dates` module is there, even after restarting Julia. +This means that `Dates` is in the sysimage itself and does not have to be loaded +from amywhere external. + +Creating and using a custom sysimage is done in three steps: + +1. Start Julia with the `--output-o=sys.o custom_sysimage.jl` where + `custom_sysimage.jl` is a file that creates the state that we want the + sysimage to contain and `sys.o` is the resulting [object + file](https://en.wikipedia.org/wiki/Object_file) that we will turn into a + sysimage. +2. Create a shared library from the object file by linking it with `libjulia`. + This is the actual sysimage. +3. Use the custom sysimage in Julia with the `-Jpath/to/sysimage` (or the + longer, more descriptive `--sysimage`) flag. + +### 1. Creating the object file + +For now, the goal is to put `CSV` in the sysimage (in the same way as the +standard library `Dates` is in it). We therefore initially simply create a file +called `custom_sysimage.jl` with the content. + +```julia +using CSV +``` + +in a `custom_sysimage.jl` file. Let's try using the flag `--output-o` (and +disabling using the startup file) and running the file: + +``` +julia --startup-file=no --output-o=sys.o -- custom_sysimage.jl +ERROR: could not open file boot.jl +``` + +That did not work well. It turns out that when using the `--output-o` option one +has to explicitly give a sysimage path ([due to this +line](/~https://github.com/JuliaLang/julia/blob/49fb7924498e9fe813444cc684a24002e75b2ac9/src/jloptions.c#L533)). Since we do not have a custom sysimage yet we +just want to give the path to the default sysimage which we can get the path to +via: + +```julia-repl +julia> unsafe_string(Base.JLOptions().image_file) +"/home/kc/julia/lib/julia/sys.so" +``` + +Let's try again, specifying the default sysimage path with the `-J` flag: + +``` +julia --startup-file=no --output-o sys.o -J"/home/kc/julia/lib/julia/sys.so" custom_sysimage.jl +signal (11): Segmentation fault +in expression starting at none:0 +uv_write2 at /workspace/srcdir/libuv/src/unix/stream.c:1397 +uv_write at /workspace/srcdir/libuv/src/unix/stream.c:1492 +jl_uv_write at /buildworker/worker/package_linux64/build/src/jl_uv.c:476 +uv_write_async at ./stream.jl:967 +uv_write at ./stream.jl:924 +``` + +Failure again! Another caveat when using `--output-o` is that modules +`__init__()` functions do not end up getting called, which is what normally +happens when a module is loaded. The reason for this is that often the state +that gets defined in `__init__` is not something that you want to serialize to +a file. In this particular case, some parts of the IO system have not been +initialized so Julia crashes while trying to print an error. The magic +incantation to make IO work properly is `Base.reinit_stdio()`. To figure out +the actual problem we modify the `custom_sysimage.jl` file to look like: + +```julia +Base.reinit_stdio() +using CSV +``` + + +and rerun the julia-command: + +``` +julia --startup-file=no --output-o sys.o -J"/home/kc/julia/lib/julia/sys.so" custom_sysimage.jl +ERROR: LoadError: ArgumentError: Package CSV not found in current path: +- Run `import Pkg; Pkg.add("CSV")` to install the CSV package. + +Stacktrace: + [1] require(::Module, ::Symbol) at ./loading.jl:887 + [2] include at ./boot.jl:328 [inlined] + [3] include_relative(::Module, ::String) at ./loading.jl:1105 + [4] include(::Module, ::String) at ./Base.jl:31 + [5] exec_options(::Base.JLOptions) at ./client.jl:295 + [6] _start() at ./client.jl:468 +in expression starting at /home/kc/custom_sysimage.jl:2 +``` + +Okay, now we can see the error. Julia can not find the `CSV` +package. Package-loading in Julia is based on the two arrays `LOAD_PATH` and +`DEPOT_PATH`. Adding `@show LOAD_PATH` and `@show DEPOT_PATH` to the +`custom_sysimage.jl` file and rerunning the command above prints: + +```julia +LOAD_PATH = String[] +DEPOT_PATH = String[] +``` + +Again, we have an initialization problem. Looking at [what Julia itself does +before including the standard libraries](/~https://github.com/JuliaLang/julia/blob/88c34fc51d962aaef973935942b2e073e2e2f398/base/sysimg.jl#L13-L14), we can see that +the functions initializing these variables are explicitly called. Let us do the +same by updating the `custom_sysimage.jl` file to: + +```julia +Base.init_depot_path() +Base.init_load_path() + +using CSV + +empty!(LOAD_PATH) +empty!(DEPOT_PATH) +``` + +and running + +``` +julia --startup-file=no --output-o sys.o -J"/home/kc/julia/lib/julia/sys.so" custom_sysimage.jl +``` + +This time, after some waiting (2 min on the authors quite beefy computer) we do +end up with a `sys.o` file. + +### 2. Creating the sysimage shared library from the object file + +The goal in this part is to take the object file, link it with `libjulia` to +finally produce a shared library which is our sysimage. For this, we need to +use a C-compiler e.g. `gcc`. We need to link with `libjulia` so we need to give +the compiler the path to where the julia library resides which can be gotten +by: + +```julia-repl +julia> abspath(Sys.BINDIR, Base.LIBDIR) +"/home/kc/julia/lib" +``` + +We tell `gcc` that we want a shared library with the `-shared` flag and to keep +all symbols into the library by passing the `--whole-archive` to the linker +(this is on Linux, see the later section for platform differences). The final +`gcc` invocation ends up as: + +``` +gcc -shared -o sys.so -Wl,--whole-archive sys.o -Wl,--no-whole-archive -L"/home/kc/julia/lib" -ljulia +``` + +which creates the sysimage `sys.so`. + +We can compare the size of the new sysimage versus the default one and see that the +new is a bit larger due to the extra packages it contains: + +```julia-repl +julia> stat("sys.so").size / (1024*1024) +162.16205596923828 + +julia> stat(unsafe_string(Base.JLOptions().image_file)).size / (1024*1024) +147.0646743774414 +``` + +#### Platform differences + +##### macOS + +On `macOS` the linker flag `-Wl,--whole-archive` is instead written as +`-Wl,-all_load` so the command would be + +``` +gcc -shared -o sys.dylib -Wl,-all_load sys.o -L"/home/kc/Applications/julia-1.3.0-rc4/lib" -ljulia +``` + +Note that the extension has been changed from `so` to `dylib` which is the +convention for shared libraries on macOS. + +##### Windows + +Getting a compiler toolchain on Windows that works well with Julia is a bit +trickier than on Linux or macOS. One quite simple way is to follow the same +process as needed to compile Julia on windows as outlined +[here](/~https://github.com/JuliaLang/julia/blob/master/doc/build/windows.md#cygwin-to-mingw-cross-compiling) and then use the `x86_64-w64-mingw32-gcc` +compiler in Cygwin instead of `gcc`. Alternatively, a mingw compiler can be +downloaded [from +here](https://sourceforge.net/projects/mingw-w64/files/mingw-w64/) The +`libjulia` is also in a different location on Windows. Instead of the `lib` +folder it is in the `bin` folder. Other than that, the same flags as for Linux +should work to produce the sysimage shared library. + +### 3. Running Julia with the new sysimage + +We start Julia with the `-Jsys.so` flag to load the new custom `sys.so` sysimage (or `sys.dylib`, `sys.dll` on macOS and Windows respecitively) +and indeed loading CSV is now very fast: + +```julia-repl +julia> @time using CSV + 0.000432 seconds (665 allocations: 32.656 KiB) +``` + +In fact, restarting Julia and looking at `Base.loaded_modules` we can see that, just like the standard libraries, CSV and +its dependencies are already loaded when Julia is started: + +```julia-repl +julia> Base.loaded_modules +Dict{Base.PkgId,Module} with 52 entries: + Parsers [69de0a69-1ddd-5017-9359-2bf0b02dc9f0] => Parsers +... + CSV [336ed68f-0bac-5ca0-87d4-7b16caf5d00b] => CSV +... +``` + +However, remember that a large part of the latency was not loading the package +but to compile the functions used by CSV the first time. Let's try it with the +custom sysimage: + + +```julia-repl +julia> @time using CSV + 0.001487 seconds (711 allocations: 35.203 KiB) + +julia> @time CSV.read("FL_insurance_sample.csv"); + 3.609626 seconds (16.34 M allocations: 795.619 MiB, 5.88% gc time) + +julia> @time CSV.read("FL_insurance_sample.csv"); + 0.026917 seconds (423 allocations: 34.695 KiB) +``` + +Reading the CSV file is significantly faster than before but still a lot slower +than the second time. As previously mentioned, the native code for the +functions in CSV is not compiled just by loading the package. This means that +even though CSV is in the sysimage the functions in CSV still need to be +compiled. The reason why the first call is faster at all is likely that +loading packages can invalidate other methods and they thus have to be +recompiled. With CSV in the sysimage, these invalidations have already been +resolved. + + +## Recording precompile statements + +We are now at the stage where we have CSV in the sysimage, but we still suffer +some latency because of compilation. +Note that Julia is a dynamically typed language, it is therefore not known statically +what types will be used in functions. Therefore, in order to be able to compile code +one needs to know what types functions should be compiled for. One way to do this is to run +some representative workload and record what types functions end up getting called with. +This is a little bit like [Profile Guide Optimization (PGO)](https://en.wikipedia.org/wiki/Profile-guided_optimization) +while it here being something more like Profile Guided Compilation.. + +There is indeed a way for Julia to record what functions are getting compiled. +We can save these and then when building the sysimage tell Julia to compile and store +the native code for these functions. + +We create a file called `generate_csv_precompile.jl` containing some "training +code" that we will use as a base to figure out what functions end up getting +compiled: + +```julia +using CSV +CSV.read("FL_insurance_sample.csv") +``` + +We then make julia run this code but we add the `--trace-compile` flag to +output "precompilation statements" to a file: + +``` +julia --startup-file=no --trace-compile=csv_precompile.jl generate_csv_precompile.jl +``` + +Looking at `csv_precompile.jl` we can see hundreds of functions that end up getting compiled. +For example, the line + +```julia +precompile(Tuple{typeof(CSV.getsource), String, Bool}) +``` + +instructs julia to compile the function `CSV.getsource` for the arguments of +type `String` and `Bool`. + +Note that some of the symbols in the list of precompile statements have a bit +of a weird syntax containing `Symbol(#...)`, e.g: + +```julia +precompile(Tuple{typeof(Base.map), getfield(CSV, Symbol("##4#5")), Base.SubString{String}}) +``` + +These are symbols that were not explicitly named in the source code but that +Julia automatically gave an internal name to refer to. These symbols are not +necessarily consistent between different Julia versions or even Julia built for +different operating systems. It is possible to make the precompile statements +more portable by filtering out any symbols starting with `#` but that naturally +leaves some latency on the table since these now have to be compiled during runtime. + +The way we make Julia cache the compilation of the functions in the list is +simply by executing the statement on each line when the sysimage is created. It +, unfortunately, isn't as simple as just adding an `include("csv_precompile")` +to our `custom_precompile.jl` file. Firstly, all the modules used in the +precompilation statements (like `DataFrames`) are not defined in the Main +namespace. Secondly, due to [some bugs in the way Julia export precompile +statements](/~https://github.com/JuliaLang/julia/issues/28808) running a +precompile statement can fail. The solution to these issues is to load all +modules in the sysimage by looping through `Base.loaded_modules` and to use a +`try-catch` for each precompile statement. In addition, we evaluate everything +in an anonymous module to not pollute the `Main` module which a bunch of +symbols. + +The end result is a `custom_sysimage.jl` file looking like: + +```julia +Base.init_depot_path() +Base.init_load_path() + +using CSV + +@eval Module() begin + for (pkgid, mod) in Base.loaded_modules + if !(pkgid.name in ("Main", "Core", "Base")) + eval(@__MODULE__, :(const $(Symbol(mod)) = $_mod)) + end + end + for statement in readlines("csv_precompile.jl") + try + Base.include_string(@__MODULE__, statement) + catch + # See julia issue #28808 + @info "failed to compile statement: $statement" + end + end +end # module + +empty!(LOAD_PATH) +empty!(DEPOT_PATH) +``` + +After repeating the process of creating the object file and using a compiler to +create the shared library sysimage, we are in a position to time again: + +```julia +julia> @time using CSV + 0.000408 seconds (665 allocations: 32.656 KiB) + +julia> @time CSV.read("FL_insurance_sample.csv"); + 0.031504 seconds (441 allocations: 37.383 KiB) + +julia> @time CSV.read("FL_insurance_sample.csv"); + 0.021355 seconds (423 allocations: 34.695 KiB) +``` + +And finally, our first time for parsing the CSV-file is close to the second time. + diff --git a/docs/src/examples/ohmyrepl.md b/docs/src/examples/ohmyrepl.md new file mode 100644 index 00000000..df989ffc --- /dev/null +++ b/docs/src/examples/ohmyrepl.md @@ -0,0 +1,55 @@ +# [Creating a sysimage with OhMyREPL](@id manual-omr) + +[OhMyREPL.jl](/~https://github.com/KristofferC/OhMyREPL.jl) is a package that +enhances the REPL with, for example, syntax highlighting. It does, however, +come with a bit of a startup time increase, so compiling a new system image +with OhMyREPL included is useful. Importing the OhMyREPL package is not the +only factor that contributes to the extra load time from using OhMyREPL In +addition, the time of compiling functions that OhMyREPL uses is also a factor. +Therefore, we also want to do "Profile Guided Compilation" (PGC), where we +record what functions gets compiled when using OhMyREPL, so they can be cached +into the system image. OhMyREPL is a bit different from most other packages in +that is used interactive. Normally to do PGC with PackageCompiler we pass a +script to to execute as the `precompile_exectution_file` which is used to +collect compilation data, but in this case, we will use Julia to manually +collect this data. + +First install `OhMyREPL` in the global environement using `import Pkg; +Pkg.add("OhMyREPL")`. Run `using OhMyREPL` and write something (like `1+1`). +It should be syntax highlighted, but you might have noticed that there was a bit +of a delay before the characters appeared. This is the extra latency from using +the package that we want to get rid off. + +![OhMyREPL installation](omr_install.png) + +The first goal is to have Julia emit the functions it compiles when running +OhMyREPL. To this end, start Julia with the +`--trace-compile=ohmyrepl_precompile.jl` flag. This will start a standard +Julia session but all functions that get compiled are output to the file +`ohmyrepl_precompile.jl`. In the Julia session, load OhMyREPL, use the REPL a bit +so that the functionality of OhMyREPL is exercised. Quit Julia and look into +the file `ohmyrepl_precompile`. It should be filled with lines like: + +``` +precompile(Tuple{typeof(OhMyREPL.Prompt.insert_keybindings), Any}) +precompile(Tuple{typeof(OhMyREPL.__init__)}) +``` + +These are functions that Julia compiled. We now just tell `create_sysimage` to +use these precompile statements when creating the system image: + +```julia +PackageCompiler.create_sysimage(:OhMyREPL; precompile_statements_file="ohmyrepl_precompile.jl", replace_default=true) +``` + +Restart julia and start typing something. If everything went well you should +see the type text become highlighted with a significantly smaller delay than +before creating the new system image + + +!!! note + If you want to go back to the default sysimage you can run + + ```julia + PackageCompiler.restore_default_sysimage() + ``` diff --git a/docs/src/examples/omr_install.png b/docs/src/examples/omr_install.png new file mode 100644 index 00000000..8cdc605d Binary files /dev/null and b/docs/src/examples/omr_install.png differ diff --git a/docs/src/examples/plots.md b/docs/src/examples/plots.md new file mode 100644 index 00000000..f2a96118 --- /dev/null +++ b/docs/src/examples/plots.md @@ -0,0 +1,59 @@ +# Creating a sysimage for fast plotting with Plots.jl + +A common complaint about Julia is that the "time to first plot" is a bit +longer than desired. In this example, we will create a sysimage that is made +to specifically improve this. + +To get a reference, we measure the time it takes to create the first plot with +the default sysimage: + +```julia-repl +julia> @time using Plots + 5.284989 seconds (5.22 M allocations: 308.954 MiB, 1.41% gc time) + +julia> @time (p = plot(rand(5), rand(5)); display(p)) + 13.769197 seconds (18.42 M allocations: 909.963 MiB, 1.75% gc time) +``` + +This is approximately 19 seconds from start of Julia to the first plot. + +We now create a precompilation file with exactly this workload in `precompile_plots.jl`: + + +```julia +using Plots +p = plot(rand(5), rand(5)) +display(p) +``` + +The custom sysimage is then created as: + +```julia +using PackageCompiler +create_sysimage(:Plots, sysimage_path="sys_plots.so", precompile_execution_file="precompile_plots.jl") +``` + +If we now start Julia with the flag `-Jsys_plots.so` and re-time our previous commands: + +```julia-repl +julia> @time using Plots + 0.000826 seconds (852 allocations: 42.125 KiB) + +julia> @time (p = plot(rand(5), rand(5)); display(p)) + 0.139642 seconds (468.42 k allocations: 12.176 MiB) +``` + +which is a sizeable speedup. + +Note that since we have more stuff in our sysimage, Julia is slightly slower to +start (0.04 seconds on this machine): + +``` +# Default sysimage +➜ time julia -e '' + 0.13s user 0.08s system 88% cpu 0.232 total + +# Custom sysimage +➜ time julia -Jsys_plots.so -e '' + 0.17s user 0.10s system 94% cpu 0.284 total +``` diff --git a/docs/src/index.md b/docs/src/index.md new file mode 100644 index 00000000..7664d5be --- /dev/null +++ b/docs/src/index.md @@ -0,0 +1,32 @@ +# PackageCompiler + +PackageCompiler is a Julia package with two main purposes: + +1. Creating custom sysimages for reduced latency when working locally with + packages that has a high startup time. + +2. Creating "apps" which are a bundle of files including an executable that can + be sent and run on other machines without Julia being installed on that machine. + +The manual contains some uses of Linux commands like `ls` (`dir` in Windows) +and `cat` but hopefully these commands are common enough that the points still +come across. + +## Installation instructions + +!!! note + + It is strongly recommended to use the official binaries that are downloaded from + https://julialang.org/downloads/. Distribution-provided Julia installations are + unlikely to work properly with this package. + +To use PackageCompiler a C-compiler needs to be available: + +### macOS, Linux + +Having a decently modern `gcc` or `clang` available should be enough to use PackageCompiler on Linux or macOS. +For macOS, using something like `homebrew` and for Linux the system package manager should work fine. + +### Windows + +A suitable compiler will be automatically installed the first time it is neeed. diff --git a/docs/src/refs.md b/docs/src/refs.md new file mode 100644 index 00000000..5c45c57c --- /dev/null +++ b/docs/src/refs.md @@ -0,0 +1,8 @@ +# References + +```@docs +PackageCompiler.create_sysimage +PackageCompiler.restore_default_sysimage +PackageCompiler.create_app +PackageCompiler.audit_app +``` diff --git a/docs/src/sysimages.md b/docs/src/sysimages.md new file mode 100644 index 00000000..b25d3d08 --- /dev/null +++ b/docs/src/sysimages.md @@ -0,0 +1,206 @@ +# Sysimages + +## What is a sysimage + +A sysimage is a file which, in a loose sense, contains a Julia session +serialized to a file. A "Julia session" includes things like loaded packages, +global variables, inferred and compiled code, etc. By starting Julia with a +sysimage, the stored Julia session is deserialized and loaded. The idea behind +the sysimage is that this deserialization is faster than having to reload +packages and recompile code from scratch. + +Julia ships with a sysimage that is used by default when Julia is started. That +sysimage contains the Julia compiler itself, the standard libraries and also +compiled code that has been put there to reduce the time required to do common +operations, like working in the REPL. + +Sometimes it is desirable to create a custom sysimage with custom precompiled +code. This is the case if one has some dependencies that take a significant +time to load or where the compilation time for the first call is uncomfortably +long. This section of the documentation is intended to document how to use +PackageCompiler to create such sysimages. + +### Drawbacks to custom sysimages + +It should be clearly stated that there are some drawbacks to using a custom +sysimage, thereby sidestepping the standard Julia package precompilation +system. The biggest drawback is that packages that are compiled into a +sysimage (including their dependencies!) are "locked" to the version they where +at when the sysimage was created. This means that no matter what package +version you have installed in your current project, the one in the sysimage +will take precedence. This can lead to bugs where you start with a project that +needs a specific version of a package, but you have another one compiled into +the sysimage. + +Putting packages in the sysimage is therefore only recommended if the load time +of those packages is a significant problem and when these packages +are not frequently updated. In addition, compiling "workflow packages" like +Revise.jl and OhMyREPL.jl and using that as a default sysimage might make sense. + +## Creating a sysimage using PackageCompiler + +PackageCompiler provides the function [`create_sysimage`](@ref) to create a +sysimage. It takes as the first argument a package or a list of packages that +should be embedded in the resulting sysimage. By default, the given packages are +loaded from the active project but a specific project can be specified by +giving a path with the `project` keyword. The location of the resulting +sysimage is given by the `sysimage_path` keyword. After the sysimage is +created, giving the command flag `-Jpath/to/sysimage` will start Julia with the +given sysimage. + +Below is an example of a new sysimage, from a separate project, being created +with the package Example.jl in it. Using `Base.loaded_modules` it can be seen +that the package is loaded without having to explicitly `import` it. + +``` +~ +❯mkdir NewSysImageEnv + +~ +❯ cd NewSysImageEnv + +~/NewSysImageEnv 29s +❯ julia -q + +julia> using PackageCompiler +[ Info: Precompiling PackageCompiler [dffaa6cc-da53-48e5-b007-4292dfcc27f1] + +(v1.3) pkg> activate . +Activating new environment at `~/NewSysImageEnv/Project.toml` + +(NewSysImageEnv) pkg> add Example + Updating registry at `~/.julia/registries/General` + Updating git-repo `/~https://github.com/JuliaRegistries/General.git` + Resolving package versions... + Updating `~/NewSysImageEnv/Project.toml` + [7876af07] + Example v0.5.3 + Updating `~/NewSysImageEnv/Manifest.toml` + [7876af07] + Example v0.5.3 + +julia> create_sysimage(:Example; sysimage_path="ExampleSysimage.so") +[ Info: PackageCompiler: creating system image object file, this might take a while... + +julia> exit() + +~/NewSysImageEnv +❯ ls +ExampleSysimage.so Manifest.toml Project.toml + +~/NewSysImageEnv +❯ julia -q -JExampleSysimage.so + +julia> Base.loaded_modules +Dict{Base.PkgId,Module} with 34 entries: +... + Example [7876af07-990d-54b4-ab0e-23690620f79a] => Example +... +``` + +Alternatively, instead of giving a path to where the new sysimage should appear, one +can choose to replace the default sysimage. +This is done by omitting the `sysimage_path` keyword and instead adding `replace_default=true`, for example: + +```julia +create_sysimage([:Debugger, :OhMyREPL]; replace_default=true) +``` + +If this is the first time `create_sysimage` is called with `replace_default`, a +backup of the default sysimage is created. The default sysimage can then be +restored with [`restore_default_sysimage()`](@ref). + +Note that sysimages are created "incrementally" in the sense that they add to +the sysimage of the process running PackageCompiler. If the default sysimage +has been replaced, the next `create_sysimage` call will create a new sysimage +based on the replaced sysimage. It is possible to create a sysimage +non-incrementally by passing the `incremental=false` keyword. This will create +a new system image from scratch. However, it will lose the special +precompilation that the Julia bundled sysimage provides which is what make the +REPL and package manager not require compilation after a Julia restart.. It is +therefore unlikely that `incremental=false` is of much use unless in special +cases for sysimage creation (for apps it is a different story though). + +### Precompilation + +The step where we included Example.jl in the sysimage meant that loading +Example is now pretty much instant (the package is already loaded when Julia +starts). However, functions inside Example.jl still need to be compiled when +executed for the first time. One way we can see this is by using the +`--trace-compile=stderr` flag which outputs a "precompile statement" every +time Julia compiles a function. Running the `hello` function inside Example.jl +we can see that it needs to be compiled (it shows the function +`Example.hello` was compiled for the input type `String`. + +``` +~/NewSysImageEnv +❯ julia -JExampleSysimage.so --trace-compile=stderr -e 'import Example; Example.hello("friend")' +precompile(Tuple{typeof(Example.hello), String}) +``` + +To remedy this, we can give a "precompile script" to `create_sysimage` which +causes functions executed in that script to be baked into the sysimage. As an +example, the script below simply calls the `hello` function in `Example`: + +``` +~/NewSysImageEnv +❯ cat precompile_example.jl +using Example +Example.hello("friend") +``` + +We now create a new system image called `ExampleSysimagePrecompile.so`, where +the `precompile_execution_file` keyword argument has been given, pointing to +the file just shown above: + +```julia-repl +~/NewSysImageEnv +❯ julia-q + +julia> using PackageCompiler + +(v1.3) pkg> activate . +Activating environment at `~/NewSysImageEnv/Project.toml` + +julia> PackageCompiler.create_sysimage(:Example; sysimage_path="ExampleSysimagePrecompile.so", + precompile_execution_file="precompile_example.jl") +[ Info: PackageCompiler: creating system image object file, this might take a while... + +julia> exit() +``` + +Using the just created system image, we can see that the `hello` function no longer needs to get compiled: + +``` +~/NewSysImageEnv +❯ julia -JExampleSysimagePrecompile.so --trace-compile=stderr -e 'import Example; Example.hello("friend")' + +~/NewSysImageEnv +❯ +``` + +#### Using a manually generated list of precompile statements + +Starting Julia with `--trace-compile=file.jl` will emit precompilation +statements to `file.jl` for the duration of the started Julia process. This +can be useful in cases where it is difficult to give a script that executes the +code (like with interactive use). A file with a list of such precompile +statements can be used when creating a sysimage by passing the keyword argument +`precompile_statements_file`. See the [OhMyREPL.jl example](@ref manual-omr) in the docs for more +details on how to use `--trace-compile` with PackageCompiler. + +It is also possible to use +[SnoopCompile.jl](https://timholy.github.io/SnoopCompile.jl/stable/snoopi/#auto-1) +to create files with precompilation statements. + + +#### Using a package's test suite to generate precompile statements + +It is also possible to use a package's test suite to generate a list of +precompile statements by including the content: + +```julia +import Example +include(joinpath(pkgdir(Example), "test", "runtests.jl")) +``` + +in the precompile file. Note that you need to have any test dependencies installed +in your current project. diff --git a/examples/MyApp/Artifacts.toml b/examples/MyApp/Artifacts.toml new file mode 100644 index 00000000..d33e2118 --- /dev/null +++ b/examples/MyApp/Artifacts.toml @@ -0,0 +1,46 @@ +[[fooifier]] +arch = "x86_64" +git-tree-sha1 = "98d93024ca384050c59d554415b75d61e467fd8c" +libc = "glibc" +os = "linux" + + [[fooifier.download]] + sha256 = "5208c63a9d07e592c78f541fc13caa8cd191b11e7e77b31d407237c2b13ec391" + url = "/~https://github.com/staticfloat/small_bin/raw/master/libfoo/libfoo.x86_64-linux-gnu.tar.gz" + +[[fooifier]] +arch = "i686" +git-tree-sha1 = "c3a9f27382862092e064bcf4aeb3cb7190578338" +libc = "glibc" +os = "linux" + + [[fooifier.download]] + sha256 = "97655b6a218d61284723b6923d7c96e6a256fa68b9419d723c588aa24404b102" + url = "/~https://github.com/staticfloat/small_bin/raw/master/libfoo/libfoo.i686-linux-gnu.tar.gz" + +[[fooifier]] +arch = "x86_64" +git-tree-sha1 = "f413ff2438a4e9e9dd69b23c35ca30de6af069cc" +os = "macos" + + [[fooifier.download]] + sha256 = "fcc268772d6f21d65b45fcf3854a3142679b78e53c7673dac26c95d6ccc89a24" + url = "/~https://github.com/staticfloat/small_bin/raw/master/libfoo/libfoo.x86_64-apple-darwin14.tar.gz" + +[[fooifier]] +arch = "x86_64" +git-tree-sha1 = "d61f806c76b57e54f343634c5219d00d4c81b077" # FIX! +os = "windows" + + [[fooifier.download]] + sha256 = "7f8939e9529835b83810d3ae7e2556f6e002d571f619894e54ece42ea5262b7f" + url = "/~https://github.com/staticfloat/small_bin/raw/master/libfoo/libfoo.x86_64-w64-mingw32.tar.gz" + +[[fooifier]] +arch = "aarch64" +git-tree-sha1 = "281cbe3dd65aa4bdb887bfb29651da500c81e242" +os = "linux" + + [[fooifier.download]] + sha256 = "36886ac25cf5678c01fe20630b413f9354b7a3721c6a2c2043162f7ebd147ff5" + url = "/~https://github.com/staticfloat/small_bin/raw/master/libfoo/libfoo.aarch64-linux-gnu.tar.gz" diff --git a/examples/MyApp/Manifest.toml b/examples/MyApp/Manifest.toml new file mode 100644 index 00000000..d590ae1d --- /dev/null +++ b/examples/MyApp/Manifest.toml @@ -0,0 +1,70 @@ +# This file is machine-generated - editing it directly is not advised + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[Example]] +git-tree-sha1 = "46e44e869b4d90b96bd8ed1fdcf32244fddfb6cc" +uuid = "7876af07-990d-54b4-ab0e-23690620f79a" +version = "0.5.3" + +[[HelloWorldC_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "8912dbb81d8c8bd7117426a9072068d58c38d1f4" +repo-rev = "master" +repo-url = "/~https://github.com/JuliaBinaryWrappers/HelloWorldC_jll.jl" +uuid = "dca1746e-5efc-54fc-8249-22745bc95a49" +version = "1.0.6+1" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[LibGit2]] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[Pkg]] +deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/examples/MyApp/Project.toml b/examples/MyApp/Project.toml new file mode 100644 index 00000000..246f67e9 --- /dev/null +++ b/examples/MyApp/Project.toml @@ -0,0 +1,9 @@ +name = "MyApp" +uuid = "f943f3d7-887a-4ed5-b0c0-a1d6899aa8f5" +authors = ["Kristoffer Carlsson "] +version = "0.1.0" + +[deps] +Example = "7876af07-990d-54b4-ab0e-23690620f79a" +HelloWorldC_jll = "dca1746e-5efc-54fc-8249-22745bc95a49" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" diff --git a/examples/MyApp/precompile_app.jl b/examples/MyApp/precompile_app.jl new file mode 100644 index 00000000..d70278d6 --- /dev/null +++ b/examples/MyApp/precompile_app.jl @@ -0,0 +1,4 @@ +using MyApp + +push!(ARGS, "arg") +MyApp.julia_main() diff --git a/examples/MyApp/src/MyApp.jl b/examples/MyApp/src/MyApp.jl new file mode 100644 index 00000000..cc3d5780 --- /dev/null +++ b/examples/MyApp/src/MyApp.jl @@ -0,0 +1,53 @@ +module MyApp + +using Example +using HelloWorldC_jll +using Pkg.Artifacts + +const fooifier = joinpath(ensure_artifact_installed("fooifier", joinpath(@__DIR__, "..", "Artifacts.toml")), + "bin", "fooifier" * (Sys.iswindows() ? ".exe" : "")) + +function julia_main() + try + real_main() + catch + Base.invokelatest(Base.display_error, Base.catch_stack()) + return 1 + end + return 0 +end + +function real_main() + @show ARGS + @show Base.PROGRAM_FILE + @show DEPOT_PATH + @show LOAD_PATH + @show pwd() + @show Base.active_project() + @show Threads.nthreads() + @show Sys.BINDIR + display(Base.loaded_modules) + + println("Running a jll package:") + HelloWorldC_jll.hello_world() do x + println("HelloWorld artifact at $(realpath(x))") + run(`$x`) + end + println() + + println("Running the artifact") + res = read(`$fooifier 5 10`, String) + println("The result of 2*5^2 - 10 == $res") + + println() + @show unsafe_string(Base.JLOptions().image_file) + @show Example.domath(5) + @show sin(0.0) + return +end + +if abspath(PROGRAM_FILE) == @__FILE__ + real_main() +end + +end # module diff --git a/examples/REQUIRE b/examples/REQUIRE deleted file mode 100644 index 9d79edf3..00000000 --- a/examples/REQUIRE +++ /dev/null @@ -1 +0,0 @@ -UnicodePlots diff --git a/examples/hello.jl b/examples/hello.jl deleted file mode 100644 index 6b6a7a2c..00000000 --- a/examples/hello.jl +++ /dev/null @@ -1,12 +0,0 @@ -module Hello - -using UnicodePlots - -Base.@ccallable function julia_main(ARGS::Vector{String})::Cint - println("hello, world") - @show sin(0.0) - println(lineplot(1:100, sin.(range(0, stop=2π, length=100)))) - return 0 -end - -end diff --git a/juliac.jl b/juliac.jl deleted file mode 100644 index 282b256a..00000000 --- a/juliac.jl +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env julia - -# ArgParse can't be a dependency here, -# so users need to install it. -using ArgParse, PackageCompiler - -Base.@ccallable function julia_main(args::Vector{String})::Cint - - s = ArgParseSettings("Static Julia Compiler", - version = "$(basename(@__FILE__)) version 0.7", - add_version = true) - - @add_arg_table s begin - "juliaprog" - arg_type = String - required = true - help = "Julia program to compile" - "cprog" - arg_type = String - help = "C program to compile (required only when building an executable, if not provided a minimal driver program is used)" - "--verbose", "-v" - action = :store_true - help = "increase verbosity" - "--quiet", "-q" - action = :store_true - help = "suppress non-error messages" - "--builddir", "-d" - arg_type = String - metavar = "" - help = "build directory" - "--outname", "-n" - arg_type = String - metavar = "" - help = "output files basename" - "--snoopfile", "-p" - arg_type = String - metavar = "" - help = "specify script calling functions to precompile" - "--clean", "-c" - action = :store_true - help = "remove build directory" - "--autodeps", "-a" - action = :store_true - help = "automatically build required dependencies" - "--object", "-o" - action = :store_true - help = "build object file" - "--shared", "-s" - action = :store_true - help = "build shared library" - "--init-shared", "-i" - action = :store_true - help = "add `init_jl_runtime` and `exit_jl_runtime` to shared library for runtime initialization" - "--executable", "-e" - action = :store_true - help = "build executable file" - "--rmtemp", "-t" - action = :store_true - help = "remove temporary build files" - "--copy-julialibs", "-j" - action = :store_true - help = "copy Julia libraries to build directory" - "--copy-file", "-f" - arg_type = String - action = :append_arg - dest_name = "copy-files" - metavar = "" - help = "copy file to build directory, can be repeated for multiple files" - "--release", "-r" - action = :store_true - help = "build in release mode, implies `-O3 -g0` unless otherwise specified" - "--Release", "-R" - action = :store_true - help = "perform a fully automated release build, equivalent to `-atjr`" - "--sysimage", "-J" - arg_type = String - metavar = "" - help = "start up with the given system image file" - "--home", "-H" - arg_type = String - metavar = "" - help = "set location of `julia` executable" - "--startup-file" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "load `~/.julia/config/startup.jl`" - "--handle-signals" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "enable or disable Julia's default signal handlers" - "--sysimage-native-code" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "use native code from system image if available" - "--compiled-modules" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "enable or disable incremental precompilation of modules" - "--depwarn" - arg_type = String - metavar = "{yes|no|error}" - range_tester = (x -> x ∈ ("yes", "no", "error")) - help = "enable or disable syntax and method deprecation warnings" - "--warn-overwrite" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "enable or disable method overwrite warnings" - "--compile" - arg_type = String - metavar = "{yes|no|all|min}" - range_tester = (x -> x ∈ ("yes", "no", "all", "min")) - help = "enable or disable JIT compiler, or request exhaustive compilation" - "--cpu-target", "-C" - arg_type = String - metavar = "" - help = "limit usage of CPU features up to (implies default `--sysimage-native-code=no`)" - "--optimize", "-O" - arg_type = Int - metavar = "{0,1,2,3}" - range_tester = (x -> x ∈ (0, 1, 2, 3)) - help = "set the optimization level" - "--debug", "-g" - arg_type = Int - metavar = "" - range_tester = (x -> x ∈ (0, 1, 2)) - help = "enable / set the level of debug info generation" - "--inline" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "control whether inlining is permitted" - "--check-bounds" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "emit bounds checks always or never" - "--math-mode" - arg_type = String - metavar = "{ieee,fast}" - range_tester = (x -> x ∈ ("ieee", "fast")) - help = "disallow or enable unsafe floating point optimizations" - "--cc" - arg_type = String - metavar = "" - help = "system C compiler" - "--cc-flag" - arg_type = String - action = :append_arg - dest_name = "cc-flags" - metavar = "" - help = "pass custom flag to the system C compiler when building a shared library or executable, can be repeated for multiple flags" - end - - s.epilog = """ - examples:\n - \ua0\ua0juliac.jl -vae hello.jl # verbose, build executable and deps\n - \ua0\ua0juliac.jl -vae hello.jl prog.c # embed into user defined C program\n - \ua0\ua0juliac.jl -qo hello.jl # quiet, build object file only\n - \ua0\ua0juliac.jl -vosej hello.jl # build all and copy Julia libs\n - \ua0\ua0juliac.jl -vRe hello.jl # fully automated release build - """ - - parsed_args = parse_args(args, s) - - parsed_args["copy-files"] == String[] && (parsed_args["copy-files"] = nothing) - - # TODO: in future it may be possible to broadcast dictionary indexing, see: https://discourse.julialang.org/t/accessing-multiple-values-of-a-dictionary/8648 - if getindex.(Ref(parsed_args), ["clean", "object", "shared", "executable", "rmtemp", "copy-julialibs", "copy-files"]) == [false, false, false, false, false, false, nothing] - parsed_args["quiet"] || println("nothing to do, exiting\ntry \"$(basename(@__FILE__)) -h\" for more information") - exit(0) - end - - juliaprog = pop!(parsed_args, "juliaprog") - filter!(kv -> kv.second !== nothing && kv.second !== false, parsed_args) - kw_args = [Symbol(replace(kv.first, "-" => "_")) => kv.second for kv in parsed_args] - - static_julia(juliaprog; kw_args...) - - return 0 -end - -if get(ENV, "COMPILE_STATIC", "false") == "false" - julia_main(ARGS) -end diff --git a/src/PackageCompiler.jl b/src/PackageCompiler.jl index 332fc17d..7e3e2433 100644 --- a/src/PackageCompiler.jl +++ b/src/PackageCompiler.jl @@ -1,166 +1,705 @@ module PackageCompiler -using Pkg, Serialization, Libdl, UUIDs -using Pkg: TOML, Operations, Types +using Base: active_project +using Libdl: Libdl +using Pkg: Pkg +using UUIDs: UUID +export create_sysimage, create_app, audit_app, restore_default_sysimage -include("compiler_flags.jl") -include("static_julia.jl") -include("api.jl") -include("snooping.jl") -include("system_image.jl") -include("pkg.jl") -include("incremental.jl") +include("juliaconfig.jl") +const NATIVE_CPU_TARGET = "native" +# See /~https://github.com/JuliaCI/julia-buildbot/blob/489ad6dee5f1e8f2ad341397dc15bb4fce436b26/master/inventory.py +function default_app_cpu_target() + if Sys.ARCH === :i686 + return "pentium4;sandybridge,-xsaveopt,clone_all" + elseif Sys.ARCH === :x86_64 + return "generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)" + elseif Sys.ARCH === :arm + return "armv7-a;armv7-a,neon;armv7-a,neon,vfp4" + elseif Sys.ARCH === :aarch64 + return "generic" # is this really the best here? + elseif Sys.ARCH === :powerpc64le + return "pwr8" + else + return "generic" + end +end -const sysimage_binaries = ("sys.$(Libdl.dlext)",) +current_process_sysimage_path() = unsafe_string(Base.JLOptions().image_file) -function copy_system_image(src, dest, ignore_missing = false) - for file in sysimage_binaries - # backup - srcfile = joinpath(src, file) - destfile = joinpath(dest, file) - if !isfile(srcfile) - ignore_missing && continue - error("No file: $srcfile") +all_stdlibs() = readdir(Sys.STDLIB) + +yesno(b::Bool) = b ? "yes" : "no" + +function bitflag() + if Sys.ARCH == :i686 + return `-m32` + elseif Sys.ARCH == :x86_64 + return `-m64` + else + return `` + end +end + +function march() + if Sys.ARCH === :i686 + return "-march=pentium4" + elseif Sys.ARCH === :x86_64 + return "-march=x86-64" + elseif Sys.ARCH === :arm + return "-march=armv7-a+simd" + elseif Sys.ARCH === :aarch64 + return "-march=armv8-a+crypto+simd" + elseif Sys.ARCH === :powerpc64le + return nothing + else + return nothing + end +end + +# Overwriting an open file is problematic in Windows +# so move it out of the way first +function move_default_sysimage_if_windows() + if Sys.iswindows() && isfile(default_sysimg_path()) + mv(default_sysimg_path(), tempname()) + end +end + +function windows_compiler_artifact_path(f, compiler) + if Sys.iswindows() + withenv("PATH" => string(ENV["PATH"], ";", dirname(compiler))) do + f() end - if isfile(destfile) - if isfile(destfile * ".backup") - rm(destfile * ".backup", force = true) + else + f() + end +end + +function get_compiler() + cc = get(ENV, "JULIA_CC", nothing) + if cc !== nothing + return cc + end + if Sys.iswindows() + return joinpath(Pkg.Artifacts.artifact"x86_64-w64-mingw32", "mingw64", "bin", "gcc.exe") + end + if Sys.which("gcc") !== nothing + return "gcc" + elseif Sys.which("clang") !== nothing + return "clang" + end + error("could not find a compiler, looked for `gcc` and `clang`") +end + +function get_julia_cmd() + julia_path = joinpath(Sys.BINDIR, Base.julia_exename()) + cmd = `$julia_path --color=yes --startup-file=no` +end + +function rewrite_sysimg_jl_only_needed_stdlibs(stdlibs::Vector{String}) + sysimg_source_path = Base.find_source_file("sysimg.jl") + sysimg_content = read(sysimg_source_path, String) + # replaces the hardcoded list of stdlibs in sysimg.jl with + # the stdlibs that is given as argument + return replace(sysimg_content, + r"stdlibs = \[(.*?)\]"s => string("stdlibs = [", join(":" .* stdlibs, ",\n"), "]")) +end + +function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String) + tmp = mktempdir() + sysimg_source_path = Base.find_source_file("sysimg.jl") + base_dir = dirname(sysimg_source_path) + tmp_corecompiler_ji = joinpath(tmp, "corecompiler.ji") + tmp_sys_ji = joinpath(tmp, "sys.ji") + compiler_source_path = joinpath(base_dir, "compiler", "compiler.jl") + + @info "PackageCompiler: creating base system image (incremental=false)..." + cd(base_dir) do + # Create corecompiler.ji + cmd = `$(get_julia_cmd()) --cpu-target $cpu_target --output-ji $tmp_corecompiler_ji + -g0 -O0 $compiler_source_path` + @debug "running $cmd" + read(cmd) + + # Use that to create sys.ji + new_sysimage_content = rewrite_sysimg_jl_only_needed_stdlibs(stdlibs) + new_sysimage_source_path = joinpath(base_dir, "sysimage_packagecompiler_x.jl") + write(new_sysimage_source_path, new_sysimage_content) + try + cmd = `$(get_julia_cmd()) --cpu-target $cpu_target + --sysimage=$tmp_corecompiler_ji + -g1 -O0 --output-ji=$tmp_sys_ji $new_sysimage_source_path` + @debug "running $cmd" + read(cmd) + finally + rm(new_sysimage_source_path; force=true) + end + end + + return tmp_sys_ji +end + +function run_precompilation_script(project::String, sysimg::String, precompile_file::Union{String, Nothing}) + tracefile = tempname() + if precompile_file == nothing + arg = `-e ''` + else + arg = `$precompile_file` + end + touch(tracefile) + cmd = `$(get_julia_cmd()) --sysimage=$(sysimg) --project=$project + --compile=all --trace-compile=$tracefile $arg` + @debug "run_precompilation_script: running $cmd" + read(cmd) + return tracefile +end + +# Load packages in a normal julia process to make them precompile "normally" +function do_ensurecompiled(project, packages, sysimage) + use = join("using " .* packages, '\n') + cmd = `$(get_julia_cmd()) --sysimage=$sysimage --project=$project -e $use` + @debug "running $cmd" + read(cmd, String) + return nothing +end + +function create_sysimg_object_file(object_file::String, packages::Vector{String}; + project::String, + base_sysimage::String, + precompile_execution_file::Vector{String}, + precompile_statements_file::Vector{String}, + cpu_target::String, + script::Union{Nothing, String}, + isapp::Bool) + + # Handle precompilation + precompile_statements = "" + @debug "running precompilation execution script..." + tracefiles = String[] + for file in (isempty(precompile_execution_file) ? (nothing,) : precompile_execution_file) + tracefile = run_precompilation_script(project, base_sysimage, file) + precompile_statements *= " append!(precompile_statements, readlines($(repr(tracefile))))\n" + end + for file in precompile_statements_file + precompile_statements *= + " append!(precompile_statements, readlines($(repr(file))))\n" + end + + precompile_code = """ + # This @eval prevents symbols from being put into Main + @eval Module() begin + PrecompileStagingArea = Module() + for (_pkgid, _mod) in Base.loaded_modules + if !(_pkgid.name in ("Main", "Core", "Base")) + eval(PrecompileStagingArea, :(const \$(Symbol(_mod)) = \$_mod)) + end + end + precompile_statements = String[] + $precompile_statements + for statement in sort(precompile_statements) + # println(statement) + try + Base.include_string(PrecompileStagingArea, statement) + catch + # See julia issue #28808 + @debug "failed to execute \$statement" + end + end + end # module + """ + + # include all packages into the sysimg + julia_code = """ + Base.reinit_stdio() + @eval Sys BINDIR = ccall(:jl_get_julia_bindir, Any, ())::String + Base.init_load_path() + Base.init_depot_path() + """ + + # Ensure packages to be put into sysimage are precompiled by loading them in a + # separate process first. + if !isempty(packages) + do_ensurecompiled(project, packages, base_sysimage) + end + + for pkg in packages + julia_code *= """ + using $pkg + """ + end + + julia_code *= precompile_code + + if script !== nothing + julia_code *= """ + include($(repr(abspath(script)))) + """ + end + + if isapp + # If it is an app, there is only one packages + @assert length(packages) == 1 + packages[1] + app_start_code = """ + Base.@ccallable function julia_main()::Cint + try + $(packages[1]).julia_main() + catch + Core.print("julia_main() threw an unhandled exception") + return 1 end - mv(destfile, destfile * ".backup", force = true) end - @info "Copying system image: $srcfile to $destfile" - cp(srcfile, destfile, force = true) + """ + julia_code *= app_start_code + end + + julia_code *= """ + empty!(LOAD_PATH) + empty!(DEPOT_PATH) + """ + + # finally, make julia output the resulting object file + @debug "creating object file at $object_file" + @info "PackageCompiler: creating system image object file, this might take a while..." + + cmd = `$(get_julia_cmd()) --cpu-target=$cpu_target + --sysimage=$base_sysimage --project=$project --output-o=$(object_file) -e $julia_code` + @debug "running $cmd" + run(cmd) +end + +default_sysimg_path() = abspath(Sys.BINDIR, "..", "lib", "julia", "sys." * Libdl.dlext) +default_sysimg_name() = basename(default_sysimg_path()) +backup_default_sysimg_path() = default_sysimg_path() * ".backup" +backup_default_sysimg_name() = basename(backup_default_sysimg_path()) + +# TODO: Also check UUIDs for stdlibs, not only names +gather_stdlibs_project(project::String) = gather_stdlibs_project(create_pkg_context(project)) +function gather_stdlibs_project(ctx) + @assert ctx.env.manifest !== nothing + stdlibs = all_stdlibs() + stdlibs_project = String[] + for (uuid, pkg) in ctx.env.manifest + if pkg.name in stdlibs + push!(stdlibs_project, pkg.name) + end end + return stdlibs_project end -julia_cpu_target(x) = error("CPU target needs to be a string or `nothing`") -julia_cpu_target(x::String) = x # TODO: match against available targets -function julia_cpu_target(::Nothing) - replace(Base.julia_cmd().exec[2], "-C" => "") +function check_packages_in_project(ctx, packages) + packages_in_project = collect(keys(ctx.env.project.deps)) + if ctx.env.pkg !== nothing + push!(packages_in_project, ctx.env.pkg.name) + end + packages_not_in_project = setdiff(string.(packages), packages_in_project) + if !isempty(packages_not_in_project) + error("package(s) $(join(packages_not_in_project, ", ")) not in project") + end end """ -Reverts a forced compilation of the system image. -This will restore any previously backed up system image files, or -build a new, clean system image. + create_sysimage(packages::Union{Symbol, Vector{Symbol}}; kwargs...) + +Create a system image that includes the package(s) in `packages`. An attempt +to automatically find a compiler will be done but can also be given explicitly +by setting the environment variable `JULIA_CC` to a path to a compiler + +### Keyword arguments: + +- `sysimage_path::Union{String,Nothing}`: The path to where + the resulting sysimage should be saved. If set to `nothing` the keyword argument + `replace_defalt` needs to be set to `true`. + +- `project::String`: The project that should be active when the sysimage is created, + defaults to the current active project. + +- `precompile_execution_file::Union{String, Vector{String}}`: A file or list of + files that contain code which precompilation statements should be recorded from. + +- `precompile_statements_file::Union{String, Vector{String}}`: A file or list of + files that contains precompilation statements that should be included in the sysimage. + +- `incremental::Bool`: If `true`, build the new sysimage on top of the sysimage + of the current process otherwise build a new sysimage from scratch. Defaults to `true`. + +- `filter_stdlibs::Bool`: If `true`, only include stdlibs that are in the project file. + Defaults to `false`, only set to `true` if you know the potential pitfalls. + +- `replace_default::Bool`: If `true`, replaces the default system image which is automatically + used when Julia starts. To replace with the one Julia ships with, use [`restore_default_sysimage()`](@ref) + +### Advanced keyword arguments + +- `cpu_target::String`: The value to use for `JULIA_CPU_TARGET` when building the system image. + +- `script::String`: Path to a file that gets executed in the `--output-o` process. """ -function revert(debug = false) - syspath = default_sysimg_path(debug) - sysimg_backup = dirname(get_backup!(debug)) - copy_system_image(sysimg_backup, syspath) -end +function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; + sysimage_path::Union{String,Nothing}=nothing, + project::String=dirname(active_project()), + precompile_execution_file::Union{String, Vector{String}}=String[], + precompile_statements_file::Union{String, Vector{String}}=String[], + incremental::Bool=true, + filter_stdlibs=false, + replace_default::Bool=false, + cpu_target::String=NATIVE_CPU_TARGET, + script::Union{Nothing, String}=nothing, + base_sysimage::Union{Nothing, String}=nothing, + isapp::Bool=false) + precompile_statements_file = abspath.(precompile_statements_file) + if replace_default==true + if sysimage_path !== nothing + error("cannot specify `sysimage_path` when `replace_default` is `true`") + end + end + if sysimage_path === nothing + if replace_default == false + error("`sysimage_path` cannot be `nothing` if `replace_default` is `false`") + end + # We will replace the default sysimage so just put it somewhere for now + tmp = mktempdir() + sysimage_path = joinpath(tmp, string("sys.", Libdl.dlext)) + end + if filter_stdlibs && incremental + error("must use `incremental=false` to use `filter_stdlibs=true`") + end + + # Functions lower down handles `packages` and precompilation file as arrays so convert here + packages = string.(vcat(packages)) # Package names are often used as string inside Julia + precompile_execution_file = vcat(precompile_execution_file) + precompile_statements_file = vcat(precompile_statements_file) + + # Instantiate the project + ctx = create_pkg_context(project) + @debug "instantiating project at $(repr(project))" + Pkg.instantiate(ctx) -function get_root_dir(path) - path, name = splitdir(path) - if isempty(name) - return splitdir(path)[2] + check_packages_in_project(ctx, packages) + + if !incremental + if base_sysimage !== nothing + error("cannot specify `base_sysimage` when `incremental=false`") + end + if filter_stdlibs + stdlibs = gather_stdlibs_project(ctx) + else + stdlibs= all_stdlibs() + end + base_sysimage = create_fresh_base_sysimage(stdlibs; cpu_target=cpu_target) else - name + if base_sysimage == nothing + base_sysimage = current_process_sysimage_path() + end + end + + # Create the sysimage + object_file = tempname() * ".o" + create_sysimg_object_file(object_file, packages; + project=project, + base_sysimage=base_sysimage, + precompile_execution_file=precompile_execution_file, + precompile_statements_file=precompile_statements_file, + cpu_target=cpu_target, + script=script, + isapp=isapp) + create_sysimg_from_object_file(object_file, sysimage_path) + + # Maybe replace default sysimage + if replace_default + if !isfile(backup_default_sysimg_path()) + @debug "making a backup of default sysimg" + cp(default_sysimg_path(), backup_default_sysimg_path()) + end + move_default_sysimage_if_windows() + mv(sysimage_path, default_sysimg_path(); force=true) + @info "PackageCompiler: default sysimg replaced, restart Julia for the new sysimg to be in effect" end + rm(object_file; force=true) + return nothing end -function sysimg_folder(files...) - base_path = normpath(abspath(joinpath(@__DIR__, "..", "sysimg"))) - isdir(base_path) || mkpath(base_path) - normpath(abspath(joinpath(base_path, files...))) +function create_sysimg_from_object_file(input_object::String, sysimage_path::String) + julia_libdir = dirname(Libdl.dlpath("libjulia")) + + # Prevent compiler from stripping all symbols from the shared lib. + # TODO: On clang on windows this is called something else + if Sys.isapple() + o_file = `-Wl,-all_load $input_object` + else + o_file = `-Wl,--whole-archive $input_object -Wl,--no-whole-archive` + end + extra = Sys.iswindows() ? `-Wl,--export-all-symbols` : `` + compiler = get_compiler() + m = something(march(), ``) + cmd = `$compiler $(bitflag()) $m -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra` + @debug "running $cmd" + windows_compiler_artifact_path(compiler) do + run(cmd) + end + return nothing end -function sysimgbackup_folder(files...) - backup = sysimg_folder("backup") - isdir(backup) || mkpath(backup) - sysimg_folder("backup", files...) +""" + restore_default_sysimage() + +Restores the default system image to the one that Julia shipped with. +Useful after running [`create_sysimage`](@ref) with `replace_default=true`. +""" +function restore_default_sysimage() + if !isfile(backup_default_sysimg_path()) + error("did not find a backup sysimg") + end + move_default_sysimage_if_windows() + mv(backup_default_sysimg_path(), default_sysimg_path(); force=true) + @info "PackageCompiler: default sysimg restored, restart Julia for the new sysimg to be in effect" + return nothing end -function package_folder(package...) - packages = normpath(abspath(joinpath(@__DIR__, "..", "packages"))) - isdir(packages) || mkpath(packages) - normpath(abspath(joinpath(packages, package...))) +const REQUIRES = "Requires" => UUID("ae029012-a4dd-5104-9daa-d747884805df") + +function create_pkg_context(project) + project_toml_path = Pkg.Types.projectfile_path(project; strict=true) + if project_toml_path === nothing + error("could not find project at $(repr(project))") + end + return Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path)) end """ - compile_package(packages...; kw_args...) + audit_app(app_dir::String) + +Check for possible problems with regards to relocatability for +the project at `app_dir`. -with packages being either a string naming a package, or a tuple `(package_name, precompile_file)`. -If no precompile file is given, it will use the packages `runtests.jl`, which is a good canditate -for figuring out what functions to compile! +!!! warning + This cannot guarantee that the project is free of relocatability problems, + it can only detect some known bad cases and warn about those. """ -function compile_package(packages...; kw_args...) - args = map(packages) do package - # If no explicit path to a seperate precompile file, use runtests - isa(package, String) && return (package, "test/runtests.jl") - isa(package, Tuple{String, String}) && return package - error("Unrecognized package. Use `packagename::String`, or `(packagename::String, rel_path_to_testfile::String)`. Found: `$package`") +audit_app(app_dir::String) = audit_app(create_pkg_context(app_dir)) +function audit_app(ctx::Pkg.Types.Context) + # Check for Requires.jl usage + if REQUIRES in ctx.env.project.deps + @warn "Project has a dependency on Requires.jl, code in `@require` will not be run" + end + for (uuid, pkg) in ctx.env.manifest + if REQUIRES in pkg.deps + @warn "$(pkg.name) has a dependency on Requires.jl, code in `@require` will not be run" + end end - compile_package(args...; kw_args...) + + # Check for build script usage + if isfile(joinpath(dirname(ctx.env.project_file), "deps", "build.jl")) + @warn "Project has a build script, this might indicate that it is not relocatable" + end + pkgs = Pkg.Types.PackageSpec[] + Pkg.Operations.load_all_deps!(ctx, pkgs) + for pkg in pkgs + pkg_source = Pkg.Operations.source_path(pkg) + pkg_source === nothing && continue + if isfile(joinpath(pkg_source, "deps", "build.jl")) + @warn "Package $(pkg.name) has a build script, this might indicate that it is not relocatable" + end + end + return end """ - compile_package( - packages::Tuple{String, String}...; - force = false, reuse = false, debug = false, cpu_target = nothing, - additional_packages = Symbol[] - ) - -Compile a list of packages. Each package comes as a tuple of `(package_name, precompile_file)` -where the precompile file should contain all function calls, that should get compiled into the system image. -Usually the `runtests.jl` file is a good candidate, since it should run all important functions of a package. -You can pass `additional_packages` a vector of symbols with package names, to help AOT compiling -uninstalled, recursive dependencies of `packages`. Look at `compile_incremental` to -use a toml instead. + create_app(app_source::String, compiled_app::String) + +Compile an app with the source in `app_source` to the folder `compiled_app`. +The folder `app_source` needs to contain a package where the package include a +function with the signature + +``` +julia_main()::Cint + # Perhaps do something based on ARGS + ... +end +``` + +The executable will be placed in a folder called `bin` in `compiled_app` and +when the executabl run the `julia_main` function is called. + +An attempt to automatically find a compiler will be done but can also be given +explicitly by setting the envirnment variable `JULIA_CC` to a path to a +compiler. + +### Keyword arguments: + +- `precompile_execution_file::Union{String, Vector{String}}`: A file or list of + files that contain code which precompilation statements should be recorded from. + +- `precompile_statements_file::Union{String, Vector{String}}`: A file or list of + files that contains precompilation statements that should be included in the sysimage + for the app. + +- `incremental::Bool`: If `true`, build the new sysimage on top of the sysimage + of the current process otherwise build a new sysimage from scratch. Defaults to `false`. + +- `filter_stdlibs::Bool`: If `true`, only include stdlibs that are in the project file. + Defaults to `false`, only set to `true` if you know the potential pitfalls. + +- `audit::Bool`: Warn about eventual relocatability problems with the app, defaults + to `true`. + +- `force::Bool`: Remove the folder `compiled_app` if it exists before creating the app. + +- `cpu_target::String`: The value to use for `JULIA_CPU_TARGET` when building the system image. """ -function compile_package( - packages::Tuple{String, String}...; - force = false, reuse = false, debug = false, - cpu_target = nothing, verbose = false - ) - userimg = sysimg_folder("precompile.jl") - if !reuse - # TODO that's a pretty weak way to check that it's not a path... - ispackage = all(x-> !occursin(Base.Filesystem.path_separator, x), first.(packages)) - isruntests = all(x-> x == "test/runtests.jl", last.(packages)) - if ispackage && isruntests - snoop_packages(Symbol.(first.(packages))...; file = userimg) - else - ispackage || @warn "Giving path to package deprecated. Use Package name!" - isruntests || @warn "Giving a snoopfile is deprecated. Use runtests from package!" +function create_app(package_dir::String, + app_dir::String; + precompile_execution_file::Union{String, Vector{String}}=String[], + precompile_statements_file::Union{String, Vector{String}}=String[], + incremental=false, + filter_stdlibs=false, + audit=true, + force=false, + cpu_target::String=default_app_cpu_target()) + precompile_statements_file = abspath.(precompile_statements_file) + package_dir = abspath(package_dir) + ctx = create_pkg_context(package_dir) + if isempty(ctx.env.manifest) + @warn "it is not recommended to create an app without a preexisting manifest" + end + if ctx.env.pkg === nothing + error("expected package to have a `name`-entry") + end + app_name = ctx.env.pkg.name + sysimg_file = app_name * "." * Libdl.dlext + if isdir(app_dir) + if !force + error("directory $(repr(app_dir)) already exists, use `force=true` to overwrite (will completely", + " remove the directory") end + rm(app_dir; force=true, recursive=true) end - !isfile(userimg) && reuse && error("Nothing to reuse. Please run `compile_package(reuse = true)`") - image_path = sysimg_folder() - build_sysimg(image_path, userimg, cpu_target=cpu_target, verbose = verbose) - imgfile = joinpath(image_path, "sys.$(Libdl.dlext)") - syspath = joinpath(default_sysimg_path(debug), "sys.$(Libdl.dlext)") - if force - try - backup = syspath * ".packagecompiler_backup" - isfile(backup) || mv(syspath, backup) - cp(imgfile, syspath, force=true) - @info """ - Replaced system image successfully. Next start of julia will load the newly compiled system image. - If you encounter any errors with the new julia image, try `PackageCompiler.revert([debug = false])`. - """ - catch e - @warn "An error occured while replacing sysimg files:" error = e - @info "Recovering old system image from backup" - # if any file is missing in default system image, revert! - if !isfile(syspath) - @info "$syspath missing. Reverting!" - revert(debug) - end + + audit && audit_app(ctx) + + mkpath(app_dir) + + bundle_julia_libraries(app_dir) + bundle_artifacts(ctx, app_dir) + + # TODO: Create in a temp dir and then move it into place? + binpath = joinpath(app_dir, "bin") + mkpath(binpath) + cd(binpath) do + if !incremental + tmp = mktempdir() + # Use workaround at /~https://github.com/JuliaLang/julia/issues/34064#issuecomment-563950633 + # by first creating a normal "empty" sysimage and then use that to finally create the one + # with the @ccallable function + tmp_base_sysimage = joinpath(tmp, "tmp_sys.so") + create_sysimage(Symbol[]; sysimage_path=tmp_base_sysimage, project=package_dir, + incremental=false, filter_stdlibs=filter_stdlibs, + cpu_target=cpu_target) + + create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=package_dir, + incremental=true, + precompile_execution_file=precompile_execution_file, + precompile_statements_file=precompile_statements_file, + cpu_target=cpu_target, + base_sysimage=tmp_base_sysimage, + isapp=true) + else + create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=package_dir, + incremental=incremental, filter_stdlibs=filter_stdlibs, + precompile_execution_file=precompile_execution_file, + precompile_statements_file=precompile_statements_file, + cpu_target=cpu_target, + isapp=true) + end + create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=app_name) + if Sys.isapple() + cmd = `install_name_tool -change $sysimg_file @rpath/$sysimg_file $app_name` + @debug "running $cmd" + run(cmd) end + end + return +end + +function create_executable_from_sysimg(;sysimage_path::String, + executable_path::String) + flags = join((cflags(), ldflags(), ldlibs()), " ") + flags = Base.shell_split(flags) + wrapper = joinpath(@__DIR__, "embedding_wrapper.c") + if Sys.iswindows() + rpath = `` + elseif Sys.isapple() + rpath = `-Wl,-rpath,'@executable_path' -Wl,-rpath,'@executable_path/../lib'` else - @info """ - Not replacing system image. - You can start julia with $(`julia -J $imgfile`) at a posix shell to load the compiled files. - """ + rpath = `-Wl,-rpath,\$ORIGIN:\$ORIGIN/../lib` + end + compiler = get_compiler() + m = something(march(), ``) + cmd = `$compiler -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) $(bitflag()) $m -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $rpath $flags` + @debug "running $cmd" + run(cmd) + windows_compiler_artifact_path(compiler) do + run(cmd) end - imgfile + return nothing end +function bundle_julia_libraries(app_dir) + app_libdir = joinpath(app_dir, Sys.isunix() ? "lib" : "bin") + cp(julia_libdir(), app_libdir; force=true) + # We do not want to bundle the sysimg (nor the backup): + rm(joinpath(app_libdir, "julia", default_sysimg_name()); force=true) + rm(joinpath(app_libdir, "julia", backup_default_sysimg_name()); force=true) + return +end + +function bundle_artifacts(ctx, app_dir) + @debug "bundling artifacts..." + + pkgs = Pkg.Types.PackageSpec[] + Pkg.Operations.load_all_deps!(ctx, pkgs) + # Also want artifacts for the project itself + @assert ctx.env.pkg !== nothing + # This is kinda ugly... + ctx.env.pkg.path = dirname(ctx.env.project_file) + push!(pkgs, ctx.env.pkg) + + # Collect all artifacts needed for the project + artifact_paths = String[] + for pkg in pkgs + pkg_source_path = Pkg.Operations.source_path(pkg) + pkg_source_path === nothing && continue + # Check to see if this package has an (Julia)Artifacts.toml + for f in Pkg.Artifacts.artifact_names + artifacts_toml_path = joinpath(pkg_source_path, f) + if isfile(artifacts_toml_path) + @debug "bundling artifacts for $(pkg.name)" + artifact_dict = Pkg.Artifacts.load_artifacts_toml(artifacts_toml_path) + for name in keys(artifact_dict) + meta = Pkg.Artifacts.artifact_meta(name, artifacts_toml_path) + meta == nothing && continue + @debug " \"$name\"" + push!(artifact_paths, Pkg.Artifacts.ensure_artifact_installed(name, artifacts_toml_path)) + end + break + end + end + end -export compile_package, revert, force_native_image!, executable_ext, build_executable, build_shared_lib, static_julia, compile_incremental + # Copy the artifacts needed to the app directory + artifact_app_path = joinpath(app_dir, "artifacts") + if !isempty(artifact_paths) + mkpath(artifact_app_path) + end + for artifact_path in artifact_paths + artifact_name = basename(artifact_path) + # force=true? + cp(artifact_path, joinpath(artifact_app_path, artifact_name)) + end + return +end end # module diff --git a/src/api.jl b/src/api.jl deleted file mode 100644 index 6aae6289..00000000 --- a/src/api.jl +++ /dev/null @@ -1,125 +0,0 @@ -""" - build_sysimg(sysimg_path=default_sysimg_path(), userimg_path=nothing; cpu_target="native", force=false) - -Rebuild the system image. Store it in `sysimg_path`, which defaults to a file named `sys.ji` -that sits in the same folder as `libjulia.{so,dylib}`, except on Windows where it defaults -to `Sys.BINDIR/../lib/julia/sys.ji`. Use the cpu instruction set given by `cpu_target`. -Valid CPU targets are the same as for the `-C` option to `julia`, or the `-march` option to -`gcc`. Defaults to `native`, which means to use all CPU instructions available on the -current processor. Include the user image file given by `userimg_path`, which should contain -directives such as `using MyPackage` to include that package in the new system image. New -system image will not replace an older image unless `force` is set to true. -""" -function build_sysimg( - sysimg_path = default_sysimg_path(), userimg_path = nothing; - verbose = false, quiet = false, release = false, - home = nothing, startup_file = nothing, handle_signals = nothing, - sysimage_native_code = nothing, compiled_modules = nothing, - depwarn = nothing, warn_overwrite = nothing, - compile = nothing, cpu_target = nothing, optimize = nothing, debug = nothing, - inline = nothing, check_bounds = nothing, math_mode = nothing, - cc = nothing, cc_flags = nothing - ) - # build vanilla backup system image - clean_sysimg = get_backup!(occursin("debug", basename(Base.julia_cmd().exec[1])), cpu_target) - static_julia( - userimg_path, verbose = verbose, quiet = quiet, builddir = sysimg_path, outname = "sys", - autodeps = true, shared = true, release = release, - sysimage = clean_sysimg, home = home, startup_file = startup_file, handle_signals = handle_signals, - sysimage_native_code = sysimage_native_code, compiled_modules = compiled_modules, - depwarn = depwarn, warn_overwrite = warn_overwrite, - compile = compile, cpu_target = cpu_target, optimize = optimize, debug = debug, - inline = inline, check_bounds = check_bounds, math_mode = math_mode, - cc = cc, cc_flags = cc_flags - ) -end - -""" - build_shared_lib( - julia_program, output_name = nothing; - snoopfile = nothing, builddir = nothing, verbose = false, quiet = false, - init_shared = false, copy_julialibs = true, copy_files = nothing, release = false, Release = false, - sysimage = nothing, home = nothing, startup_file = nothing, handle_signals = nothing, - sysimage_native_code = nothing, compiled_modules = nothing, - depwarn = nothing, warn_overwrite = nothing, - compile = nothing, cpu_target = nothing, optimize = nothing, debug = nothing, - inline = nothing, check_bounds = nothing, math_mode = nothing, - cc = nothing, cc_flags = nothing - ) - `julia_program` needs to be a Julia script containing a `julia_main` function, e.g. like `examples/hello.jl` - `snoopfile` is optional and can be a Julia script which calls functions that you want to make sure to have precompiled - `builddir` is where the compiled artifacts will end up -""" -function build_shared_lib( - julia_program, output_name = nothing; - snoopfile = nothing, builddir = nothing, verbose = false, quiet = false, - init_shared = false, copy_julialibs = true, copy_files = nothing, release = false, Release = false, - sysimage = nothing, home = nothing, startup_file = nothing, handle_signals = nothing, - sysimage_native_code = nothing, compiled_modules = nothing, - depwarn = nothing, warn_overwrite = nothing, - compile = nothing, cpu_target = nothing, optimize = nothing, debug = nothing, - inline = nothing, check_bounds = nothing, math_mode = nothing, - cc = nothing, cc_flags = nothing - ) - static_julia( - julia_program, verbose = verbose, quiet = quiet, - builddir = builddir, outname = output_name, snoopfile = snoopfile, autodeps = true, shared = true, - init_shared = init_shared, copy_julialibs = copy_julialibs, copy_files = copy_files, release = release, Release = release, - sysimage = sysimage, home = home, startup_file = startup_file, handle_signals = handle_signals, - sysimage_native_code = sysimage_native_code, compiled_modules = compiled_modules, - depwarn = depwarn, warn_overwrite = warn_overwrite, - compile = compile, cpu_target = cpu_target, optimize = optimize, debug = debug, - inline = inline, check_bounds = check_bounds, math_mode = math_mode, - cc = cc, cc_flags = cc_flags - ) -end - -""" - build_executable( - julia_program, output_name = nothing, c_program = nothing; - snoopfile = nothing, builddir = nothing, verbose = false, quiet = false, - copy_julialibs = true, copy_files = nothing, release = false, Release = false, - sysimage = nothing, home = nothing, startup_file = nothing, handle_signals = nothing, - sysimage_native_code = nothing, compiled_modules = nothing, - depwarn = nothing, warn_overwrite = nothing, - compile = nothing, cpu_target = nothing, optimize = nothing, debug = nothing, - inline = nothing, check_bounds = nothing, math_mode = nothing, - cc = nothing, cc_flags = nothing - ) - `julia_program` needs to be a Julia script containing a `julia_main` function, e.g. like `examples/hello.jl` - `snoopfile` is optional and can be a Julia script which calls functions that you want to make sure to have precompiled - `builddir` is where the compiled artifacts will end up -""" -function build_executable( - julia_program, output_name = nothing, c_program = nothing; - snoopfile = nothing, builddir = nothing, verbose = false, quiet = false, - copy_julialibs = true, copy_files = nothing, release = false, Release = false, - sysimage = nothing, home = nothing, startup_file = nothing, handle_signals = nothing, - sysimage_native_code = nothing, compiled_modules = nothing, - depwarn = nothing, warn_overwrite = nothing, - compile = nothing, cpu_target = nothing, optimize = nothing, debug = nothing, - inline = nothing, check_bounds = nothing, math_mode = nothing, - cc = nothing, cc_flags = nothing - ) - static_julia( - julia_program, cprog = c_program, verbose = verbose, quiet = quiet, - builddir = builddir, outname = output_name, snoopfile = snoopfile, autodeps = true, executable = true, - copy_julialibs = copy_julialibs, copy_files = copy_files, release = release, Release = release, - sysimage = sysimage, home = home, startup_file = startup_file, handle_signals = handle_signals, - sysimage_native_code = sysimage_native_code, compiled_modules = compiled_modules, - depwarn = depwarn, warn_overwrite = warn_overwrite, - compile = compile, cpu_target = cpu_target, optimize = optimize, debug = debug, - inline = inline, check_bounds = check_bounds, math_mode = math_mode, - cc = cc, cc_flags = cc_flags - ) -end - -""" - force_native_image!() -Builds a clean system image, similar to a fresh Julia install. -Can also be used to build a native system image for a downloaded cross compiled julia binary. -""" -function force_native_image!(debug = false) # debug is ignored right now - sysimg = get_backup!(debug, "native") - copy_system_image(dirname(sysimg), default_sysimg_path(debug)) -end diff --git a/src/compiler_flags.jl b/src/compiler_flags.jl deleted file mode 100644 index 65d2b0bb..00000000 --- a/src/compiler_flags.jl +++ /dev/null @@ -1,265 +0,0 @@ -# This code is derived from `julia-config.jl` (part of Julia) and should be kept aligned with it. - -threadingOn() = ccall(:jl_threading_enabled, Cint, ()) != 0 - -function shell_escape(str) - str = replace(str, "'" => "'\''") - return "'$str'" -end - -function libDir() - return if ccall(:jl_is_debugbuild, Cint, ()) != 0 - dirname(abspath(Libdl.dlpath("libjulia-debug"))) - else - dirname(abspath(Libdl.dlpath("libjulia"))) - end -end - -private_libDir() = abspath(Sys.BINDIR, Base.PRIVATE_LIBDIR) - -function includeDir() - return abspath(Sys.BINDIR, Base.INCLUDEDIR, "julia") -end - -function ldflags() - fl = "-L$(shell_escape(libDir()))" - if Sys.iswindows() - fl = fl * " -Wl,--stack,8388608" - elseif Sys.islinux() - fl = fl * " -Wl,--export-dynamic" - end - return fl -end - -function ldlibs() - libname = if ccall(:jl_is_debugbuild, Cint, ()) != 0 - "julia-debug" - else - "julia" - end - if Sys.isunix() - return "-Wl,-rpath,$(shell_escape(libDir())) -Wl,-rpath,$(shell_escape(private_libDir())) -l$libname" - else - return "-l$libname -lopenlibm" - end -end - -function cflags() - flags = IOBuffer() - print(flags, "-std=gnu99") - include = shell_escape(includeDir()) - print(flags, " -I", include) - if threadingOn() - print(flags, " -DJULIA_ENABLE_THREADING=1") - end - if Sys.isunix() - print(flags, " -fPIC") - end - return String(take!(flags)) -end - -function allflags() - return "$(cflags()) $(ldflags()) $(ldlibs())" -end - - - - -const short_flag_to_jloptions = Dict( - "C" => :cpu_target, - "J" => :image_file, - "O" => :opt_level, - "g" => :debug_level -) - -const flag_to_jloptions = Dict( - "check-bounds" => :check_bounds, - "code-coverage" => :code_coverage, - "compile" => :compile_enabled, - "compiled-modules" => :use_compiled_modules, - "cpu-target" => :cpu_target, - "depwarn" => :depwarn, - "handle-signals" => :handle_signals, - "history-file" => :historyfile, - "machine-file" => :machine_file, - "math-mode" => :fast_math, - "optimize" => :opt_level, - "output-bc" => :outputbc, - "output-ji" => :outputji, - "output-jitbc" => :outputjitbc, - "output-o" => :outputo, - "output-unoptbc" => :outputunoptbc, - "project" => :project, - "startup-file" => :startupfile, - "sysimage" => :image_file, - "sysimage-native-code" => :use_sysimage_native_code, - "trace-compile" => :trace_compile, - "track-allocation" => :malloc_log, - "warn-overwrite" => :warn_overwrite, - "inline" => :can_inline -) - -const jl_options_to_flag = Dict( - :can_inline => "inline", - :handle_signals => "handle-signals", - :opt_level => "optimize", - :depwarn => "depwarn", - :malloc_log => "track-allocation", - :outputo => "output-o", - :startupfile => "startup-file", - :compile_enabled => "compile", - :trace_compile => "trace-compile", - :check_bounds => "check-bounds", - :outputji => "output-ji", - :use_sysimage_native_code => "sysimage-native-code", - :outputunoptbc => "output-unoptbc", - :historyfile => "history-file", - :outputbc => "output-bc", - :warn_overwrite => "warn-overwrite", - :machine_file => "machine-file", - :code_coverage => "code-coverage", - :image_file => "sysimage", - :cpu_target => "cpu-target", - :outputjitbc => "output-jitbc", - :project => "project", - :fast_math => "math-mode", - :use_compiled_modules => "compiled-modules", - :debug_level => "g" -) - -const flags_with_cmdval = Set([ - :handle_signals, - :use_sysimage_native_code, - :depwarn, - :can_inline, - :historyfile, - :startupfile, - :use_compiled_modules, - :warn_overwrite, - :check_bounds, - :fast_math, - :compile_enabled, - :malloc_log, - :code_coverage -]) - -function to_cmd_val(key::Symbol, val) - # undocumented auto!? well we can only skip it I guess - if key in (:depwarn, :warn_overwrite) - val == 0 && return "no" - val == 1 && return "yes" - val == 2 && return "error" - end - if key in (:code_coverage, :malloc_log) - val == 0 && return "none" - val == 1 && return "user" - val == 2 && return "all" - end - if key == :compile_enabled - val == 0 && return "no" - val == 1 && return "yes" - val == 2 && return "all" - end - if key == :fast_math - val == 0 && return "ieee" - val == 1 && return "fast" - end - val in (0, -1) && return "" - val == 1 && return "yes" - val == 2 && return "no" -end - - -function jl_option_value(opts, key) - value = getfield(opts, key) - if value isa Ptr{UInt8} - return value == C_NULL ? "" : unsafe_string(value) - end - if key in flags_with_cmdval - return to_cmd_val(key, value) - end - return value -end - -is_short_flag(flag) = haskey(short_flag_to_jloptions, flag) - -function jl_option_key(flag::Symbol) - # if symbol is used, we can also check the fields directly. - # TODO should we also do this for strings? - flag in fieldnames(Base.JLOptions) && return flag - jl_option_key(string(flag)) -end - -function jl_option_key(flag::String) - haskey(short_flag_to_jloptions, flag) && return short_flag_to_jloptions[flag] - haskey(flag_to_jloptions, flag) && return flag_to_jloptions[flag] - m_flag = replace(flag, "_" => "-") - haskey(flag_to_jloptions, m_flag) && return flag_to_jloptions[m_flag] - flags = replace.([keys(flag_to_jloptions)..., keys(short_flag_to_jloptions)...], ("-" => "_",)) - error("Flag $flag not a valid Julia Options. Valid flags are:\n$(join(flags, " "))") -end - -""" - extract_flag(flag, jl_cmd = Base.julia_cmd()) - -Extracts the value for `flag` from `jl_cmd`. -""" -function jl_flag_value(flag, jl_options = Base.JLOptions()) - jl_option_value(jl_options, jl_option_key(flag)) -end - -current_systemimage() = jl_flag_value("J") - -""" - run_julia( - code::String; - g = nothing, O = nothing, output_o = nothing, J = nothing, - startup_file = "no" - ) - -Run `code` in a julia command. -You can overwrite any julia command line flag by setting it to a value. -If the flag has the value `nothing`, the value of the flag of the current julia process is used. -""" -function run_julia(code::String; kw...) - run(julia_code_cmd(code; kw...)) -end - -function jl_command(flag, value) - (value === nothing || isempty(value)) && return "" - if is_short_flag(flag) - string("-", flag, value) - else - string("--", flag, "=", value) - end -end - - -function push_command!(cmd, flag, value) - command = jl_command(flag, value) - isempty(command) || push!(cmd.exec, command) -end - -function julia_code_cmd( - code::String, jl_options = Base.JLOptions(); - kw... - ) - jl_cmd = Base.julia_cmd() - cmd = `$(jl_cmd.exec[1])` # extract julia exe - # Add the commands from the keyword arguments - for (k, v) in kw - jl_key = jl_option_key(k) - flag = jl_options_to_flag[jl_key] - push_command!(cmd, flag, v) - end - # add remaining commands from JLOptions - for key in setdiff(keys(jl_options_to_flag), keys(kw)) - flag = jl_options_to_flag[key] - push_command!(cmd, flag, jl_option_value(jl_options, key)) - end - # for better debug, let's not make a tmp file which would get lost! - file = sysimg_folder("run_julia_code.jl") - open(io-> println(io, code), file, "w") - push!(cmd.exec, file) - cmd -end diff --git a/examples/program.c b/src/embedding_wrapper.c similarity index 66% rename from examples/program.c rename to src/embedding_wrapper.c index 67a6d0a1..6c447d2c 100644 --- a/examples/program.c +++ b/src/embedding_wrapper.c @@ -3,29 +3,41 @@ // Standard headers #include #include +#include // Julia headers (for initialization and gc commands) #include "uv.h" #include "julia.h" -#ifdef JULIA_DEFINE_FAST_TLS // only available in Julia v0.7 and above JULIA_DEFINE_FAST_TLS() -#endif + +// TODO: Windows wmain handling as in repl.c // Declare C prototype of a function defined in Julia -extern int julia_main(jl_array_t*); +int julia_main(jl_array_t*); // main function (windows UTF16 -> UTF8 argument conversion code copied from julia's ui/repl.c) int main(int argc, char *argv[]) { - int retcode; - int i; uv_setup_args(argc, argv); // no-op on Windows // initialization libsupport_init(); - // jl_options.compile_enabled = JL_OPTIONS_COMPILE_OFF; + // Get the current exe path so we can compute a relative depot path + char *free_path = (char*)malloc(PATH_MAX); + size_t path_size = PATH_MAX; + if (!free_path) + jl_errorf("fatal error: failed to allocate memory: %s", strerror(errno)); + if (uv_exepath(free_path, &path_size)) { + jl_error("fatal error: unexpected error while retrieving exepath"); + } + + char buf[PATH_MAX]; + snprintf(buf, sizeof(buf), "JULIA_DEPOT_PATH=%s/../../", free_path); + putenv(buf); + putenv("JULIA_LOAD_PATH=@"); + // JULIAC_PROGRAM_LIBNAME defined on command-line for compilation jl_options.image_file = JULIAC_PROGRAM_LIBNAME; julia_init(JL_IMAGE_JULIA_HOME); @@ -40,13 +52,13 @@ int main(int argc, char *argv[]) // Set Base.ARGS to `String[ unsafe_string(argv[i]) for i = 1:argc ]` jl_array_t *ARGS = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("ARGS")); jl_array_grow_end(ARGS, argc - 1); - for (i = 1; i < argc; i++) { + for (int i = 1; i < argc; i++) { jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); jl_arrayset(ARGS, s, i - 1); } // call the work function, and get back a value - retcode = julia_main(ARGS); + int retcode = julia_main(ARGS); // Cleanup and gracefully exit jl_atexit_hook(retcode); diff --git a/src/incremental.jl b/src/incremental.jl deleted file mode 100644 index d87e29b8..00000000 --- a/src/incremental.jl +++ /dev/null @@ -1,178 +0,0 @@ -""" -Init basic C libraries -""" -function InitBase() - """ - Base.__init__() - Sys.__init__() #fix /~https://github.com/JuliaLang/julia/issues/30479 - """ -end - -""" -# Initialize REPL module for Docs -""" -function InitREPL() - """ - using REPL - Base.REPL_MODULE_REF[] = REPL - """ -end -function Include(path) - """ - Mod = @eval module \$(gensym("anon_module")) end - # Include into anonymous module to not polute namespace - Mod.include($(repr(path))) - """ -end - -""" -Exit hooks can get serialized and therefore end up in weird behaviour -When incrementally compiling -""" -function ExitHooksStart() - """ - atexit_hook_copy = copy(Base.atexit_hooks) # make backup - # clean state so that any package we use can carelessly call atexit - empty!(Base.atexit_hooks) - """ -end - -function ExitHooksEnd() - """ - Base._atexit() # run all exit hooks we registered during precompile - empty!(Base.atexit_hooks) # don't serialize the exit hooks we run + added - # atexit_hook_copy should be empty, but who knows what base will do in the future - append!(Base.atexit_hooks, atexit_hook_copy) - """ -end - -function PackageCallbacksStart() - """ - package_callbacks_copy = copy(Base.package_callbacks) - empty!(Base.package_callbacks) - """ -end - -function PackageCallbacksEnd() - """ - empty!(Base.package_callbacks) - append!(Base.package_callbacks, package_callbacks_copy) - """ -end - -function REPLHooksStart() - """ - repl_hooks_copy = copy(Base.repl_hooks) - empty!(Base.repl_hooks) - """ -end - -function REPLHooksEnd() - """ - empty!(Base.repl_hooks) - append!(Base.repl_hooks, repl_hooks_copy) - """ -end - -function DisableLibraryThreadingHooksStart() - """ - if isdefined(Base, :disable_library_threading_hooks) - disable_library_threading_hooks_copy = copy(Base.disable_library_threading_hooks) - empty!(Base.disable_library_threading_hooks) - end - """ -end - -function DisableLibraryThreadingHooksEnd() - """ - if isdefined(Base, :disable_library_threading_hooks) - empty!(Base.disable_library_threading_hooks) - append!(Base.disable_library_threading_hooks, disable_library_threading_hooks_copy) - end - """ -end - -""" -The command to pass to julia --output-o, that runs the julia code in `path` during compilation. -""" -function PrecompileCommand(path) - ExitHooksStart() * - PackageCallbacksStart() * - REPLHooksStart() * - DisableLibraryThreadingHooksStart() * - InitBase() * - InitREPL() * - Include(path) * - DisableLibraryThreadingHooksEnd() * - REPLHooksEnd() * - PackageCallbacksEnd() * - ExitHooksEnd() -end - - - -""" - compile_incremental( - toml_path::String, snoopfile::String; - force = false, precompile_file = nothing, verbose = true, - debug = false, cc_flags = nothing - ) - - Extract all calls from `snoopfile` and ahead of time compiles them - incrementally into the current system image. - `force = true` will replace the old system image with the new one. - The argument `toml_path` should contain a project file of the packages that `snoopfile` explicitly uses. - Implicitly used packages & modules don't need to be contained! - - To compile just a single package, see the simpler version `compile_incremental(package::Symbol)`: -""" -function compile_incremental( - toml_path::Union{String, Nothing}, precompiles::String; - force = false, verbose = true, - debug = false, cc_flags = nothing - ) - systemp = sysimg_folder("sys.a") - sysout = sysimg_folder("sys.$(Libdl.dlext)") - code = PrecompileCommand(precompiles) - run_julia( - code, O = 3, output_o = systemp, g = 1, - track_allocation = "none", startup_file = "no", code_coverage = "none" - ) - build_shared(sysout, systemp, false, sysimg_folder(), verbose, "3", debug, system_compiler, cc_flags) - curr_syso = current_systemimage() - force && cp(sysout, curr_syso, force = true) - return sysout, curr_syso -end - -""" - compile_incremental( - packages::Symbol...; - force = false, reuse = false, verbose = true, - debug = false, cc_flags = nothing, - blacklist::Vector = [] - ) - - Incrementally compile `package` into the current system image. - `force = true` will replace the old system image with the new one. - This process requires a script that julia will run in order to determine - which functions to compile. A package may define a script called `snoopfile.jl` - for this purpose. If this file cannot be found the package's test script - `Package/test/runtests.jl` will be used. `compile_incremental` will search - for `snoopfile.jl` in the package's root directory and in the folders - `Package/src` and `Package/snoop`. For a more explicit version of compile_incremental, - see: `compile_incremental(toml_path::String, snoopfile::String)` - - Not all packages can currently be compiled into the system image. By default, - `compile_incremental(:Package) will also compile all of Package's dependencies. - It can still be desirable to compile packages with dependencies that cannot be - compiled. For this reason `compile_incremental` offers - the ability for the user to pass a list of blacklisted packages - that will be ignored during the compilation process. These are passed as a - vector of package names (defined as either strings or symbols) using the - `blacklist keyword argument` -""" -function compile_incremental(pkg::Symbol, packages::Symbol...; - blacklist::Vector=[], kw...) - toml, precompile = snoop_packages(pkg, packages...; blacklist=blacklist) - compile_incremental(toml, precompile; kw...) -end diff --git a/src/juliaconfig.jl b/src/juliaconfig.jl new file mode 100644 index 00000000..dda5bc95 --- /dev/null +++ b/src/juliaconfig.jl @@ -0,0 +1,63 @@ +# adopted from /~https://github.com/JuliaLang/julia/blob/release-0.6/contrib/julia-config.jl + +function shell_escape(str) + str = replace(str, "'" => "'\''") + return "'$str'" +end + +function julia_libdir() + return if ccall(:jl_is_debugbuild, Cint, ()) != 0 + dirname(abspath(Libdl.dlpath("libjulia-debug"))) + else + dirname(abspath(Libdl.dlpath("libjulia"))) + end +end + +function julia_private_libdir() + @static if Sys.iswindows() + return julia_libdir() + else + return abspath(Sys.BINDIR, Base.PRIVATE_LIBDIR) + end +end + +julia_includedir() = abspath(Sys.BINDIR, Base.INCLUDEDIR, "julia") + +function ldflags() + fl = "-L$(shell_escape(julia_libdir()))" + if Sys.iswindows() + fl = fl * " -Wl,--stack,8388608" + fl = fl * " -Wl,--export-all-symbols" + elseif Sys.islinux() + fl = fl * " -Wl,--export-dynamic" + end + return fl +end + +# TODO +function ldlibs(relative_path=nothing) + libname = if ccall(:jl_is_debugbuild, Cint, ()) != 0 + "julia-debug" + else + "julia" + end + if Sys.islinux() + return "-Wl,-rpath-link,$(shell_escape(julia_libdir())) -Wl,-rpath-link,$(shell_escape(julia_private_libdir())) -l$libname" + elseif Sys.iswindows() + return "-l$libname -lopenlibm" + else + return "-l$libname" + end +end + +function cflags() + flags = IOBuffer() + print(flags, "-std=gnu99") + include = shell_escape(julia_includedir()) + print(flags, " -I", include) + if Sys.isunix() + print(flags, " -fPIC") + end + return String(take!(flags)) +end + diff --git a/src/pkg.jl b/src/pkg.jl deleted file mode 100644 index 8f5f59fb..00000000 --- a/src/pkg.jl +++ /dev/null @@ -1,178 +0,0 @@ -#= -genfile & create_project_from_require have been taken from the PR -/~https://github.com/JuliaLang/PkgDev.jl/pull/144 -which was created by /~https://github.com/KristofferC - -THIS IS JUST A TEMPORARY SOLUTION FOR PACKAGES WITHOUT A TOML AND WILL GET MOVED OUT! -=# - -function packages_from_require(reqfile::String) - ctx = Pkg.Types.Context() - pkgs = Types.PackageSpec[] - compatibility = Pair{String, String}[] - for r in Pkg.Pkg2.Reqs.read(reqfile) - r isa Pkg.Pkg2.Reqs.Requirement || continue - r.package == "julia" && continue - push!(pkgs, Types.PackageSpec(r.package)) - intervals = r.versions.intervals - if length(intervals) != 1 - @warn "Project.toml creator cannot handle multiple requirements for $(r.package), ignoring" - else - l = intervals[1].lower - h = intervals[1].upper - if l != v"0.0.0-" - # no upper bound - if h == typemax(VersionNumber) - push!(compatibility, r.package => string(">=", VersionNumber(l.major, l.minor, l.patch))) - else # assume semver - push!(compatibility, r.package => string(">=", VersionNumber(l.major, l.minor, l.patch), ", ", - "<", VersionNumber(h.major, h.minor, h.patch))) - end - end - end - end - Operations.registry_resolve!(ctx.env, pkgs) - Operations.ensure_resolved(ctx.env, pkgs) - pkgs -end -function create_project_from_require(pkgname::String, path::String, toml_path::String) - ctx = Pkg.Types.Context() - # Package data - path = abspath(path) - mainpkg = Types.PackageSpec(pkgname) - Pkg.Operations.registry_resolve!(ctx.env, [mainpkg]) - if !Operations.has_uuid(mainpkg) - uuid = UUIDs.uuid1() - @info "Unregistered package $pkgname, giving it a new UUID: $uuid" - mainpkg.version = v"0.1.0" - else - uuid = mainpkg.uuid - @info "Registered package $pkgname, using already given UUID: $(mainpkg.uuid)" - Pkg.Operations.set_maximum_version_registry!(ctx.env, mainpkg) - v = mainpkg.version - # Remove the build - mainpkg.version = VersionNumber(v.major, v.minor, v.patch) - end - # Dependency data - dep_pkgs = Types.PackageSpec[] - test_pkgs = Types.PackageSpec[] - compatibility = Pair{String, String}[] - - reqfiles = [joinpath(path, "REQUIRE"), joinpath(path, "test", "REQUIRE")] - for (reqfile, pkgs) in zip(reqfiles, [dep_pkgs, test_pkgs]) - if isfile(reqfile) - append!(pkgs, packages_from_require(reqfile)) - end - end - - stdlib_deps = Pkg.Operations.find_stdlib_deps(ctx, path) - for (stdlib_uuid, stdlib) in stdlib_deps - pkg = Types.PackageSpec(stdlib, stdlib_uuid) - if stdlib == "Test" - push!(test_pkgs, pkg) - else - push!(dep_pkgs, pkg) - end - end - - # Write project - - project = Dict( - "name" => pkgname, - "uuid" => string(uuid), - "version" => string(mainpkg.version), - "deps" => Dict(pkg.name => string(pkg.uuid) for pkg in dep_pkgs) - ) - - if !isempty(compatibility) - project["compat"] = - Dict(name => ver for (name, ver) in compatibility) - end - - if !isempty(test_pkgs) - project["extras"] = Dict(pkg.name => string(pkg.uuid) for pkg in test_pkgs) - project["targets"] = Dict("test" => [pkg.name for pkg in test_pkgs]) - end - - open(toml_path, "w") do io - Pkg.TOML.print(io, project, sorted=true, by=key -> (Types.project_key_order(key), key)) - end -end - -function package_toml(package::Symbol) - pstr = string(package) - # could use eval using here?! Not sure what is actually better - pkg_module = Base.require(Module(), package) - pkg_root = normpath(joinpath(dirname(pathof(pkg_module)), "..")) - toml = joinpath(pkg_root, "Project.toml") - # Check for snoopfile and fall back to runtests.jl - # if it can't be found - snoopfile = get_snoopfile(pkg_root) - # We will create a new toml, based that will include all test dependencies etc - # We're also using the precompile toml as a temp toml for packages not having a toml - precompile_toml = package_folder(pstr, "Project.toml") - isdir(dirname(precompile_toml)) || mkpath(dirname(precompile_toml)) - test_deps = Dict() - if !isfile(toml) - create_project_from_require(pstr, pkg_root, precompile_toml) - else - testreq = joinpath(pkg_root, "test", "REQUIRE") - if isfile(testreq) - pkgs = packages_from_require(testreq) - test_deps = Dict(pkg.name => string(pkg.uuid) for pkg in pkgs) - end - cp(toml, precompile_toml, force = true) - chmod(precompile_toml, 0o644) - end - # remove any old manifest - if isfile(package_folder(pstr, "Manifest.toml")) - rm(package_folder(pstr, "Manifest.toml")) - end - # add ourselves as dependencies and ensure we have a manifest - run_julia(""" - using Pkg - Pkg.instantiate() - pkg"add PackageCompiler Pkg" - """, project = precompile_toml, startup_file = "no") - - toml = TOML.parsefile(precompile_toml) - - deps = merge(get(toml, "deps", Dict()), test_deps) - # Add the package itself - deps[toml["name"]] = toml["uuid"] - # Add the packages we need - test_deps = get(toml, "extras", Dict()) - compile_toml = Dict() - compile_toml["name"] = string(package, "Precompile") - compile_toml["deps"] = merge(test_deps, deps) - if haskey(toml, "compat") - compile_toml["compat"] = toml["compat"] - end - write_toml(precompile_toml, compile_toml) - precompile_toml, snoopfile -end - -function write_toml(path, dict) - open(path, "w") do io - TOML.print( - io, dict, - sorted = true, by = key-> (Types.project_key_order(key), key) - ) - end -end - -function get_snoopfile(pkg_root) - snoopfileroot = joinpath(pkg_root,"snoopfile.jl") - snoopfilesnoopdir = joinpath(pkg_root, "snoop", "snoopfile.jl") - snoopfilesrc = joinpath(pkg_root, "src", "snoopfile.jl") - if isfile(snoopfileroot) - snoopfile = snoopfileroot - elseif isfile(snoopfilesnoopdir) - snoopfile = snoopfilesnoopdir - elseif isfile(snoopfilesrc) - snoopfile = snoopfilesrc - else - snoopfile = joinpath(pkg_root, "test", "runtests.jl") - end - return snoopfile -end diff --git a/src/snooping.jl b/src/snooping.jl deleted file mode 100644 index f53573d1..00000000 --- a/src/snooping.jl +++ /dev/null @@ -1,149 +0,0 @@ - - -function snoop(package, tomlpath, snoopfile, outputfile, reuse = false, blacklist = []) - - command = """ - using Pkg, PackageCompiler - """ - - if tomlpath !== nothing - command *= """ - empty!(Base.LOAD_PATH) - # Take LOAD_PATH from parent process - append!(Base.LOAD_PATH, $(repr(Base.LOAD_PATH))) - Pkg.activate($(repr(tomlpath))) - Pkg.resolve() - Pkg.instantiate() - """ - end - - command *= """ - # let's wrap the snoop file in a try catch... - # This way we still do some snooping even if there is an error in the tests! - try - include($(repr(snoopfile))) - catch e - @warn("Snoop file errored. Precompile statements were recorded untill error!", exception = e) - end - """ - - # let's use a file in the PackageCompiler dir, - # so it doesn't get lost if later steps fail - tmp_file = package_folder("precompile_tmp.jl") - if !reuse - run_julia(command, compile = "all", O = 0, g = 1, trace_compile = tmp_file, startup_file = "no") - end - used_packages = Set{String}() # e.g. from test/REQUIRE - if package !== nothing - push!(used_packages, string(package)) - end - usings = "" - if tomlpath !== nothing - # add toml packages, in case extract_used_packages misses a package - deps = get(TOML.parsefile(tomlpath), "deps", Dict{String, Any}()) - union!(used_packages, string.(keys(deps))) - end - if !isempty(used_packages) - packages = join(setdiff(used_packages,string.(blacklist)), ", ") - usings *= """ - using $packages - for Mod in [$packages] - isdefined(Mod, :__init__) && Mod.__init__() - end - """ - end - - line_idx = 0; missed = 0 - open(outputfile, "w") do io - println(io, """ - # We need to use all used packages in the precompile file for maximum - # usage of the precompile statements. - # Since this can be any recursive dependency of the package we AOT compile, - # we decided to just use them without installing them. An added - # benefit is, that we can call __init__ this way more easily, since - # incremental sysimage compilation won't call __init__ on `using` - # /~https://github.com/JuliaLang/julia/issues/22910 - $usings - # bring recursive dependencies of used packages and standard libraries into namespace - for Mod in Base.loaded_modules_array() - if !Core.isdefined(@__MODULE__, nameof(Mod)) - Core.eval(@__MODULE__, Expr(:const, Expr(:(=), nameof(Mod), Mod))) - end - end - """) - for line in eachline(tmp_file) - line_idx += 1 - # replace function instances, which turn up as typeof(func)(). - # TODO why would they be represented in a way that doesn't actually work? - line = replace(line, r"typeof\(([\u00A0-\uFFFF\w_!´\.]*@?[\u00A0-\uFFFF\w_!´]+)\)\(\)" => s"\1") - # Is this ridicilous? Yes, it is! But we need a unique symbol to replace `_`, - # which otherwise ends up as an uncatchable syntax error - line = replace(line, r"\b_\b" => "🐃") - try - expr = Meta.parse(line, raise = true) - if expr.head != :incomplete - # after all this, we still need to wrap into try catch, - # since some anonymous symbols won't be found... - println(io, "try;", line, "; catch e; @debug \"couldn't precompile statement $line_idx\" exception = e; end") - else - missed += 1 - @debug "Incomplete line in precompile file: $line" - end - catch e - missed += 1 - @debug "Parse error in precompile file: $line" exception=e - end - end - end - @info "used $(line_idx - missed) out of $line_idx precompile statements" - outputfile -end - - -function snoop_packages(packages::Symbol...; blacklist = [], file = package_folder("incremental_precompile.jl")) - finaltoml = Dict{Any, Any}( - "deps" => Dict(), - "compat" => Dict(), - ) - toml_path = package_folder("Project.toml") - manifest_dict = Dict{String, Vector{Dict{String, Any}}}() - open(file, "w") do compile_io - println(compile_io, "# Precompile file for $(join(packages, " "))") - # make sure we have all packages from toml installed - println(compile_io, """ - using Pkg - empty!(Base.LOAD_PATH) - # Take LOAD_PATH from parent process - append!(Base.LOAD_PATH, $(repr(Base.LOAD_PATH))) - Pkg.activate($(repr(toml_path))) - Pkg.instantiate() - """) - for package in packages - precompiles = package_folder(string(package), "incremental_precompile.jl") - toml, snoopfile = package_toml(package) - snoop(package, toml, snoopfile, precompiles, false, blacklist) - pkg_toml = TOML.parsefile(toml) - manifest = joinpath(dirname(toml), "Manifest.toml") - if isfile(manifest) # not all get a manifest (only if pkg operations are executed I suppose) - pkg_manifest = TOML.parsefile(manifest) - for (name, pkgs) in pkg_manifest - pkg_vec = get!(()-> Dict{String, Any}[], manifest_dict, name) - append!(pkg_vec, pkgs); unique!(pkg_vec) - end - end - merge!(finaltoml["deps"], get(pkg_toml, "deps", Dict())) - merge!(finaltoml["compat"], get(pkg_toml, "compat", Dict())) - println(compile_io) - write(compile_io, read(precompiles)) - end - end - finaltoml["name"] = "PackagesPrecompile" - write_toml(toml_path, finaltoml) - manifest_path = package_folder("Manifest.toml") - # make sure we don't reuse old manifests - isfile(manifest_path) && rm(manifest_path) - if !isempty(manifest_dict) - write_toml(manifest_path, manifest_dict) - end - return toml_path, file -end diff --git a/src/static_julia.jl b/src/static_julia.jl deleted file mode 100644 index 3a7bb14d..00000000 --- a/src/static_julia.jl +++ /dev/null @@ -1,373 +0,0 @@ -const depsfile = normpath(@__DIR__, "..", "deps", "deps.jl") - -if isfile(depsfile) - include(depsfile) - gccworks = try - success(`$gcc -v`) - catch - false - end - if !gccworks - error("GCC wasn't found. Please make sure that gcc is on the path and run Pkg.build(\"PackageCompiler\")") - end -else - error("Package wasn't built correctly. Please run Pkg.build(\"PackageCompiler\")") -end - -system_compiler = gcc -executable_ext = Sys.iswindows() ? ".exe" : "" - -if Sys.iswindows() - function run_PATH(cmd) - bindir = dirname(cmd.exec[1]) - run(setenv(cmd, ["PATH=" * bindir * ";" * ENV["PATH"]])) - end -else - const run_PATH = run -end - - -""" - static_julia(juliaprog::String; kw_args...) - -compiles the Julia file at path `juliaprog` with keyword arguments: - - cprog C program to compile (required only when building an executable, if not provided a minimal driver program is used) - verbose increase verbosity - quiet suppress non-error messages - builddir build directory - outname output files basename - snoopfile specify script calling functions to precompile - clean remove build directory - autodeps automatically build required dependencies - object build object file - shared build shared library - init_shared add `init_jl_runtime` and `exit_jl_runtime` to shared library for runtime initialization - executable build executable file - rmtemp remove temporary build files - copy_julialibs copy Julia libraries to build directory - copy_files copy user-specified files to build directory (either `nothing` or a string array) - release build in release mode, implies `-O3 -g0` unless otherwise specified - Release perform a fully automated release build, equivalent to `-atjr` - sysimage start up with the given system image file - home set location of `julia` executable - startup_file {yes|no} load `~/.julia/config/startup.jl` - handle_signals {yes|no} enable or disable Julia's default signal handlers - sysimage_native_code {yes|no} use native code from system image if available - compiled_modules {yes|no} enable or disable incremental precompilation of modules - depwarn {yes|no|error} enable or disable syntax and method deprecation warnings - warn_overwrite {yes|no} enable or disable method overwrite warnings - compile {yes|no|all|min} enable or disable JIT compiler, or request exhaustive compilation - cpu_target limit usage of CPU features up to (implies default `--sysimage_native_code=no`) - optimize {0,1,2,3} set the optimization level - debug enable / set the level of debug info generation - inline {yes|no} control whether inlining is permitted - check_bounds {yes|no} emit bounds checks always or never - math_mode {ieee,fast} disallow or enable unsafe floating point optimizations - cc system C compiler - cc_flags pass custom flags to the system C compiler when building a shared library or executable (either `nothing` or a string array) -""" -function static_julia( - juliaprog; - cprog = nothing, verbose = false, quiet = false, builddir = nothing, outname = nothing, snoopfile = nothing, - clean = false, autodeps = false, object = false, shared = false, init_shared = false, executable = false, rmtemp = false, - copy_julialibs = false, copy_files = nothing, release = false, Release = false, - sysimage = nothing, home = nothing, startup_file = nothing, handle_signals = nothing, - sysimage_native_code = nothing, compiled_modules = nothing, - depwarn = nothing, warn_overwrite = nothing, - compile = nothing, cpu_target = nothing, optimize = nothing, debug = nothing, - inline = nothing, check_bounds = nothing, math_mode = nothing, - cc = nothing, cc_flags = nothing - ) - - cprog == nothing && (cprog = normpath(@__DIR__, "..", "examples", "program.c")) - builddir == nothing && (builddir = "builddir") - outname == nothing && (outname = splitext(basename(juliaprog))[1]) - cc == nothing && (cc = system_compiler) - - verbose && quiet && (quiet = false) - - if Release - autodeps = rmtemp = copy_julialibs = release = true - end - - if autodeps - executable && (shared = true) - shared && (object = true) - end - - if release - optimize == nothing && (optimize = "3") - debug == nothing && (debug = "0") - end - - if juliaprog != nothing - juliaprog = abspath(juliaprog) - isfile(juliaprog) || error("Cannot find file: \"$juliaprog\"") - quiet || println("Julia program file:\n \"$juliaprog\"") - end - - if executable - cprog = abspath(cprog) - isfile(cprog) || error("Cannot find file: \"$cprog\"") - quiet || println("C program file:\n \"$cprog\"") - end - - builddir = abspath(builddir) - quiet || println("Build directory:\n \"$builddir\"") - - if [clean, object, shared, executable, rmtemp, copy_julialibs, copy_files] == [false, false, false, false, false, false, nothing] - quiet || println("Nothing to do") - return - end - - if clean - if isdir(builddir) - verbose && println("Remove build directory") - rm(builddir, recursive = true) - else - verbose && println("Build directory does not exist") - end - end - - if [object, shared, executable, rmtemp, copy_julialibs, copy_files] == [false, false, false, false, false, nothing] - quiet || println("Clean completed") - return - end - - if !isdir(builddir) - verbose && println("Make build directory") - mkpath(builddir) - end - - o_file = outname * ".a" - s_file = outname * ".$(Libdl.dlext)" - e_file = outname * executable_ext - - if object - if snoopfile != nothing - snoopfile = abspath(snoopfile) - verbose && println("Executing snoopfile: \"$snoopfile\"") - precompfile = joinpath(builddir, "precompiled.jl") - snoop(nothing, nothing, snoopfile, precompfile) - jlmain = joinpath(builddir, "julia_main.jl") - open(jlmain, "w") do io - # this file will get included via @eval Module() include(...) - # but the juliaprog should be included in the main module - juliaprog != nothing && println(io, "Base.include(Main, $(repr(relpath(juliaprog, builddir))))") - println(io, "Base.include(@__MODULE__, $(repr(relpath(precompfile, builddir))))") - end - juliaprog = jlmain - end - build_object( - juliaprog, o_file, builddir, verbose, - sysimage, home, startup_file, handle_signals, sysimage_native_code, compiled_modules, - depwarn, warn_overwrite, compile, cpu_target, optimize, debug, inline, check_bounds, math_mode - ) - end - - shared && build_shared(s_file, o_file, init_shared, builddir, verbose, optimize, debug, cc, cc_flags) - - executable && build_exec(e_file, cprog, s_file, builddir, verbose, optimize, debug, cc, cc_flags) - - rmtemp && remove_temp_files(builddir, verbose) - - copy_julialibs && copy_julia_libs(builddir, verbose) - - copy_files != nothing && copy_files_array(copy_files, builddir, verbose, "Copy user-specified files to build directory:") - - quiet || println("All done") -end - -function julia_flags(optimize, debug, cc_flags) - allflags = Base.shell_split(PackageCompiler.allflags()) - bitness_flag = Sys.ARCH == :arm ? "-mbe32" : Sys.ARCH == :aarch64 ? `` : Int == Int32 ? "-m32" : "-m64" - allflags = `$allflags $bitness_flag` - optimize == nothing || (allflags = `$allflags -O$optimize`) - debug == 2 && (allflags = `$allflags -g`) - cc_flags == nothing || isempty(cc_flags) || (allflags = `$allflags $cc_flags`) - allflags -end - -function build_julia_cmd( - sysimage, home, startup_file, handle_signals, sysimage_native_code, compiled_modules, - depwarn, warn_overwrite, compile, cpu_target, optimize, debug, inline, check_bounds, math_mode - ) - # TODO: `sysimage_native_code` and `compiled_modules` may be removed in future, see: /~https://github.com/JuliaLang/PackageCompiler.jl/issues/47 - sysimage_native_code == nothing && cpu_target != nothing && (sysimage_native_code = "no") - compiled_modules == nothing && (compiled_modules = "no") - # TODO: `startup_file` may be removed in future with `julia-compile`, see: /~https://github.com/JuliaLang/julia/issues/15864 - startup_file == nothing && (startup_file = "no") - julia_cmd = `$(Base.julia_cmd())` - function get_flag_idx(julia_cmd, flag_str) - findfirst(f->(length(f) > length(flag_str) && - f[1:length(flag_str)]==flag_str), julia_cmd.exec) - end - function set_flag(julia_cmd, flag_str, val) - flag_idx = get_flag_idx(julia_cmd, flag_str) - if flag_idx != nothing - julia_cmd.exec[flag_idx] = "$flag_str$val" - else - push!(julia_cmd.exec, "$flag_str$val") - end - end - sysimage == nothing || set_flag(julia_cmd, "-J", sysimage) - home == nothing || set_flag(julia_cmd, "-H=", home) - startup_file == nothing || set_flag(julia_cmd, "--startup-file=", startup_file) - handle_signals == nothing || set_flag(julia_cmd, "--handle-signals=", handle_signals) - sysimage_native_code == nothing || set_flag(julia_cmd, "--sysimage-native-code=", sysimage_native_code) - compiled_modules == nothing || set_flag(julia_cmd, "--compiled-modules=", compiled_modules) - depwarn == nothing || set_flag(julia_cmd, "--depwarn=", depwarn) - warn_overwrite == nothing || set_flag(julia_cmd, "--warn-overwrite=", warn_overwrite) - compile == nothing || set_flag(julia_cmd, "--compile=", compile) - cpu_target == nothing || set_flag(julia_cmd, "-C", cpu_target) - optimize == nothing || set_flag(julia_cmd, "-O", optimize) - debug == nothing || set_flag(julia_cmd, "-g", debug) - inline == nothing || set_flag(julia_cmd, "--inline=", inline) - check_bounds == nothing || set_flag(julia_cmd, "--check-bounds=", check_bounds) - math_mode == nothing || set_flag(julia_cmd, "--math-mode=", math_mode) - # Disable incompatible flags - set_flag(julia_cmd, "--code-coverage=", "none") - - julia_cmd -end - -function build_object( - juliaprog, o_file, builddir, verbose, - sysimage, home, startup_file, handle_signals, sysimage_native_code, compiled_modules, - depwarn, warn_overwrite, compile, cpu_target, optimize, debug, inline, check_bounds, math_mode - ) - # TODO really refactor this :D - build_object( - juliaprog, o_file, builddir, verbose; - sysimage = sysimage, startup_file = startup_file, - handle_signals = handle_signals, sysimage_native_code = sysimage_native_code, - compiled_modules = compiled_modules, - depwarn = depwarn, warn_overwrite = warn_overwrite, - compile = compile, cpu_target = cpu_target, optimize = optimize, - debug_level = debug, inline = inline, check_bounds = check_bounds, math_mode = math_mode - ) -end - -function build_object( - juliaprog, o_file, builddir, verbose; julia_flags... - ) - Sys.iswindows() && (juliaprog != nothing) && (juliaprog = replace(juliaprog, "\\" => "\\\\")) - command = ExitHooksStart() * InitBase() * InitREPL() - if juliaprog != nothing - command *= Include(juliaprog) - end - command *= ExitHooksEnd() - verbose && println("Build static library $(repr(o_file)):\n $command") - cd(builddir) do - run_julia( - command; julia_flags..., - output_o = o_file, track_allocation = "none", code_coverage = "none", startup_file = "no", - ) - end -end - -function build_shared(s_file, o_file, init_shared, builddir, verbose, optimize, debug, cc, cc_flags) - if init_shared - i_file = joinpath(builddir, "lib_init.c") - open(i_file, "w") do io - print(io, """ - // Julia headers (for initialization and gc commands) - #include "uv.h" - #include "julia.h" - void init_jl_runtime() // alternate name for jl_init_with_image, with hardcoded library name - { - // JULIAC_PROGRAM_LIBNAME defined on command-line for compilation - const char rel_libname[] = JULIAC_PROGRAM_LIBNAME; - jl_init_with_image(NULL, rel_libname); - } - void exit_jl_runtime(int retcode) // alternate name for jl_atexit_hook - { - jl_atexit_hook(retcode); - } - """ - ) - end - i_file = `$i_file` - else - i_file = `` - end - # Prevent compiler from stripping all symbols from the shared lib. - if Sys.isapple() - o_file = `-Wl,-all_load $o_file` - else - o_file = `-Wl,--whole-archive $o_file -Wl,--no-whole-archive` - end - command = `$cc -shared -DJULIAC_PROGRAM_LIBNAME=\"$s_file\" -o $s_file $o_file $i_file $(julia_flags(optimize, debug, cc_flags))` - if Sys.isapple() - command = `$command -Wl,-install_name,@rpath/$s_file` - elseif Sys.iswindows() - command = `$command -Wl,--export-all-symbols` - end - verbose && println("Build shared library $(repr(s_file)):\n $command") - cd(builddir) do - run_PATH(command) - end -end - -function build_exec(e_file, cprog, s_file, builddir, verbose, optimize, debug, cc, cc_flags) - command = `$cc -DJULIAC_PROGRAM_LIBNAME=\"$s_file\" -o $e_file $cprog $s_file $(julia_flags(optimize, debug, cc_flags))` - if Sys.iswindows() - # functionality doesn't readily exist on this platform - elseif Sys.isapple() - command = `$command -Wl,-rpath,@executable_path` - else - command = `$command -Wl,-rpath,\$ORIGIN` - end - if Int == Int32 - # TODO: this was added because of an error with julia on win32 that suggested this line, it seems to work but I'm not sure if it's correct - command = `$command -march=pentium4` - end - verbose && println("Build executable $(repr(e_file)):\n $command") - cd(builddir) do - run_PATH(command) - end -end - -function remove_temp_files(builddir, verbose) - verbose && println("Remove temporary files:") - remove = false - for tmp in filter(x -> endswith(x, ".a") || startswith(x, "cache_ji_v"), readdir(builddir)) - verbose && println(" $tmp") - rm(joinpath(builddir, tmp), recursive = true) - remove = true - end - verbose && !remove && println(" none") -end - -function copy_files_array(files_array, builddir, verbose, message) - verbose && println(message) - copy = false - for src in files_array - isfile(src) || error("Cannot find file: \"$src\"") - dst = joinpath(builddir, basename(src)) - if filesize(src) != filesize(dst) || ctime(src) > ctime(dst) || mtime(src) > mtime(dst) - verbose && println(" $(basename(src))") - cp(src, dst, force = true, follow_symlinks = false) - copy = true - end - end - verbose && !copy && println(" none") -end - -function copy_julia_libs(builddir, verbose) - # TODO: these flags should probably be emitted also by `julia-config.jl` and `compiler_flags.jl` - shlibdir = Sys.iswindows() ? Sys.BINDIR : joinpath(Sys.BINDIR, Base.LIBDIR) - private_shlibdir = joinpath(Sys.BINDIR, Base.PRIVATE_LIBDIR) - libfiles = String[] - dlext = "." * Libdl.dlext - for dir in (shlibdir, private_shlibdir) - if Sys.iswindows() || Sys.isapple() - append!(libfiles, joinpath.(dir, filter(x -> endswith(x, dlext) && !startswith(x, "sys"), readdir(dir)))) - else - append!(libfiles, joinpath.(dir, filter(x -> occursin(r"^lib.+\.so(?:\.\d+)*$", x), readdir(dir)))) - end - end - filter!(v -> !occursin(r"debug", v), libfiles) - copy_files_array(libfiles, builddir, verbose, "Copy Julia libraries to build directory:") -end diff --git a/src/system_image.jl b/src/system_image.jl deleted file mode 100644 index 530507cf..00000000 --- a/src/system_image.jl +++ /dev/null @@ -1,66 +0,0 @@ -# This code is derived from `build_sysimg.jl` (part of Julia) and should be kept aligned with it. - -function default_sysimg_path(debug = false) - ext = debug ? "sys-debug" : "sys" - if Sys.isunix() - dirname(Libdl.dlpath(ext)) - else - normpath(Sys.BINDIR, "..", "lib", "julia") - end -end - -function compile_system_image(sysimg_path, cpu_target = nothing; debug = false) - # Enter base and setup some useful paths - base_dir = dirname(Base.find_source_file("sysimg.jl")) - cd(base_dir) do - # This can probably get merged with build_object. - # At some point, I will need to understand build_object a bit better before doing that move, though! - julia_cmd = Base.julia_cmd() - julia = julia_cmd.exec[1] - cpu_target = if cpu_target === nothing - replace(julia_cmd.exec[2], "-C" => "") - else - cpu_target - end - cc = system_compiler - # Ensure we have write-permissions to wherever we're trying to write to - try - touch("$sysimg_path.ji") - catch - error("Unable to modify $sysimg_path.ji, ensure that parent directory exists and is writable") - end - compiler_path = joinpath(dirname(sysimg_path), "basecompiler") - compiler = "compiler/compiler.jl" - - # Start by building inference.{ji,o} - inference_path = joinpath(dirname(sysimg_path), "inference") - command = `$julia -C $cpu_target --output-ji $compiler_path.ji --output-o $compiler_path.o $compiler` - @info "Building `inference.o`:\n$command" - run(command) - - # Bootstrap off of that to create sys.{ji,o} - command = `$julia -C $cpu_target --output-ji $sysimg_path.ji --output-o $sysimg_path.o -J $compiler_path.ji --startup-file=no sysimg.jl` - @info "Building `sys.o`:\n$command" - run(command) - - build_shared( - "$sysimg_path.$(Libdl.dlext)", "$sysimg_path.o", false, - ".", true, nothing, debug ? 2 : nothing, cc, nothing - ) - end -end - -""" -Returns the system image file stored in the backup folder. -If there is no backup, this function will automatically generate a system image -in the backup folder. -""" -function get_backup!(debug, cpu_target = nothing) - target = julia_cpu_target(cpu_target) - sysimg_backup = sysimgbackup_folder(target) - isdir(sysimg_backup) || mkpath(sysimg_backup) - if !all(x-> isfile(joinpath(sysimg_backup, x)), sysimage_binaries) # we have a backup - compile_system_image(joinpath(sysimg_backup, "sys"), target; debug = debug) - end - return joinpath(sysimg_backup, "sys.$(Libdl.dlext)") -end diff --git a/test/REQUIRE b/test/REQUIRE deleted file mode 100644 index d85c9081..00000000 --- a/test/REQUIRE +++ /dev/null @@ -1,6 +0,0 @@ -ColorTypes -FixedPointNumbers -UnicodePlots -JSON -DataStructures -OffsetArrays diff --git a/test/TestPackage/Project.toml b/test/TestPackage/Project.toml deleted file mode 100644 index c5484ce4..00000000 --- a/test/TestPackage/Project.toml +++ /dev/null @@ -1,4 +0,0 @@ -name = "TestPackage" -uuid = "d4c01e28-d041-5f16-991d-b5745e8a2c25" -authors = ["SimonDanisch "] -version = "0.1.0" diff --git a/test/TestPackage/src/TestPackage.jl b/test/TestPackage/src/TestPackage.jl deleted file mode 100644 index 537eab5a..00000000 --- a/test/TestPackage/src/TestPackage.jl +++ /dev/null @@ -1,5 +0,0 @@ -module TestPackage - -greet() = print("Hello World!") - -end # module diff --git a/test/TestPackage/test/runtests.jl b/test/TestPackage/test/runtests.jl deleted file mode 100644 index 447bcf8c..00000000 --- a/test/TestPackage/test/runtests.jl +++ /dev/null @@ -1,3 +0,0 @@ -using TestPackage - -TestPackage.greet() diff --git a/test/TestPackage2/Project.toml b/test/TestPackage2/Project.toml deleted file mode 100644 index a71f41d0..00000000 --- a/test/TestPackage2/Project.toml +++ /dev/null @@ -1,4 +0,0 @@ -name = "TestPackage2" -uuid = "886979a0-3551-11e9-32b6-2991215e940c" -authors = ["Patrick Belliveau "] -version = "0.1.0" diff --git a/test/TestPackage2/snoop/snoopfile.jl b/test/TestPackage2/snoop/snoopfile.jl deleted file mode 100644 index 1fd89165..00000000 --- a/test/TestPackage2/snoop/snoopfile.jl +++ /dev/null @@ -1,3 +0,0 @@ -using TestPackage2 - -TestPackage2.greet() diff --git a/test/TestPackage2/src/TestPackage2.jl b/test/TestPackage2/src/TestPackage2.jl deleted file mode 100644 index 00841e76..00000000 --- a/test/TestPackage2/src/TestPackage2.jl +++ /dev/null @@ -1,5 +0,0 @@ -module TestPackage2 - -greet() = print("Hello World!") - -end # module diff --git a/test/myapp b/test/myapp new file mode 100755 index 00000000..b9d9bd2a Binary files /dev/null and b/test/myapp differ diff --git a/test/precompile_execution.jl b/test/precompile_execution.jl new file mode 100644 index 00000000..786fde11 --- /dev/null +++ b/test/precompile_execution.jl @@ -0,0 +1,3 @@ +using Example + +Example.domath(5) diff --git a/test/precompile_statements.jl b/test/precompile_statements.jl new file mode 100644 index 00000000..1e598ca8 --- /dev/null +++ b/test/precompile_statements.jl @@ -0,0 +1 @@ +precompile(Tuple{typeof(Base.peek), Base.IOStream}) diff --git a/test/precompile_statements2.jl b/test/precompile_statements2.jl new file mode 100644 index 00000000..1e598ca8 --- /dev/null +++ b/test/precompile_statements2.jl @@ -0,0 +1 @@ +precompile(Tuple{typeof(Base.peek), Base.IOStream}) diff --git a/test/runtests.jl b/test/runtests.jl index a0cc69ff..7fc2a1d2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,161 +1,74 @@ -using PackageCompiler, Test +using PackageCompiler: PackageCompiler, create_sysimage, create_app +using Test +using Libdl -@testset "compilage_package" begin - @testset "FixedPointNumbers" begin - sysimage = PackageCompiler.compile_package("FixedPointNumbers", verbose = true) - test_code = """ - using FixedPointNumbers; N0f8(0.5); println("no segfaults, yay") - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = sysimage) - @test read(cmd, String) == "no segfaults, yay\n" - end - @testset "FixedPointNumbers ColorTypes" begin - sysimage = PackageCompiler.compile_package("FixedPointNumbers", "ColorTypes", verbose = true) - test_code = """ - using FixedPointNumbers, ColorTypes; N0f8(0.5); RGB(0.0, 0.0, 0.0); println("no segfaults, yay") - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = sysimage) - @test read(cmd, String) == "no segfaults, yay\n" - end -end +ENV["JULIA_DEBUG"] = "PackageCompiler" +# Make a new depot +new_depot = mktempdir() +mkpath(joinpath(new_depot, "registries")) +cp(joinpath(DEPOT_PATH[1], "registries", "General"), joinpath(new_depot, "registries", "General")) +ENV["JULIA_DEPOT_PATH"] = new_depot +Base.init_depot_path() -@testset "compile_incremental" begin - @testset "unregistered package with runtests.jl" begin - path = joinpath(@__DIR__, "TestPackage") - push!(Base.LOAD_PATH, path) - syso, syso_old = PackageCompiler.compile_incremental(:TestPackage) - test_code = """ - push!(Base.LOAD_PATH, $(repr(path))) - using TestPackage; TestPackage.greet() - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) - @test read(cmd, String) == "Hello World!" - pop!(Base.LOAD_PATH) - end - @testset "unregistered package with snoopfile.jl" begin - path = joinpath(@__DIR__, "TestPackage2") - push!(Base.LOAD_PATH, path) - syso, syso_old = PackageCompiler.compile_incremental(:TestPackage2) - test_code = """ - push!(Base.LOAD_PATH, $(repr(path))) - using TestPackage2; TestPackage2.greet() - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) - @test read(cmd, String) == "Hello World!" - pop!(Base.LOAD_PATH) - end - @testset "FixedPointNumbers" begin - # This is the new compile_package - syso, syso_old = PackageCompiler.compile_incremental(:FixedPointNumbers) - test_code = """ - using FixedPointNumbers; N0f8(0.5); println("no segfaults, yay") - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) - @test read(cmd, String) == "no segfaults, yay\n" - end - @testset "FixedPointNumbers ColorTypes" begin - syso, syso_old = PackageCompiler.compile_incremental(:FixedPointNumbers, :ColorTypes) - test_code = """ - using FixedPointNumbers, ColorTypes; N0f8(0.5); RGB(0.0, 0.0, 0.0); println("no segfaults, yay") - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) - @test read(cmd, String) == "no segfaults, yay\n" - end - @testset "JSON with Distributed blacklisted" begin - # This is the new compile_package - syso, syso_old = PackageCompiler.compile_incremental(:JSON, blacklist=[:Distributed]) - test_code = """ - using JSON - s = \"{\\"a_number\\" : 5.0, \\"an_array\\" : [\\"string\\", 9]}\" - j = JSON.parse(s) - println(\"no segfaults, yay\") - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) - @test read(cmd, String) == "no segfaults, yay\n" - end +is_slow_ci = haskey(ENV, "CI") && Sys.ARCH == :aarch64 + +if haskey(ENV, "CI") + @show Sys.ARCH end -julia = Base.julia_cmd().exec[1] +@testset "PackageCompiler.jl" begin + tmp = mktempdir() + sysimage_path = joinpath(tmp, "sys." * Libdl.dlext) + script = tempname() + write(script, "script_func() = println(\"I am a script\")") + create_sysimage(:Example; sysimage_path=sysimage_path, + precompile_execution_file="precompile_execution.jl", + precompile_statements_file=["precompile_statements.jl", + "precompile_statements2.jl"], + script=script) + # Check we can load sysimage and that Example is available in Main + str = read(`$(Base.julia_cmd()) -J $(sysimage_path) -e 'println(Example.hello("foo")); script_func()'`, String) + @test occursin("Hello, foo", str) + @test occursin("I am a script", str) -@testset "build_executable" begin - jlfile = joinpath(@__DIR__, "..", "examples", "hello.jl") - basedir = mktempdir() - relativebuilddir = "build" - cd(basedir) do - mkdir(relativebuilddir) - snoopfile = "snoop.jl" - open(snoopfile, "w") do io - write(io, open(read, jlfile)) - println(io) - println(io, "using .Hello; Hello.julia_main(String[])") + # Test creating an app + app_source_dir = joinpath(@__DIR__, "..", "examples/MyApp/") + # TODO: Also test something that actually gives audit warnings + @test_logs PackageCompiler.audit_app(app_source_dir) + app_compiled_dir = joinpath(tmp, "MyAppCompiled") + for incremental in (is_slow_ci ? (false,) : (true, false)) + if incremental == false + filter_stdlibs = (is_slow_ci ? (true, ) : (true, false)) + else + filter_stdlibs = (false,) end - build_executable( - jlfile, snoopfile = snoopfile, builddir = relativebuilddir, verbose = true - ) - end - builddir = joinpath(basedir, relativebuilddir) - @test isfile(joinpath(builddir, "hello.$(PackageCompiler.Libdl.dlext)")) - @test isfile(joinpath(builddir, "hello$executable_ext")) - @test startswith(read(`$(joinpath(builddir, "hello$executable_ext"))`, String), "hello, world") - for i = 1:100 - # Windows seems to have problems with removing files - it can error - # making this test fail. - try rm(basedir, recursive = true) catch end - sleep(1/100) - end -end + for filter in filter_stdlibs + tmp_app_source_dir = joinpath(tmp, "MyApp") + cp(app_source_dir, tmp_app_source_dir) + create_app(tmp_app_source_dir, app_compiled_dir; incremental=incremental, force=true, filter_stdlibs=filter, + precompile_execution_file=joinpath(app_source_dir, "precompile_app.jl")) + rm(tmp_app_source_dir; recursive=true) + # Get rid of some local state + rm(joinpath(new_depot, "packages"); recursive=true) + rm(joinpath(new_depot, "compiled"); recursive=true) + app_path = abspath(app_compiled_dir, "bin", "MyApp" * (Sys.iswindows() ? ".exe" : "")) + app_output = read(`$app_path`, String) -@testset "program.c" begin - @testset "args" begin - basedir = mktempdir(); - argsjlfile = mktemp(basedir)[1]; - write(argsjlfile, raw""" - Base.@ccallable function julia_main(argv::Vector{String})::Cint - println("@__FILE__: $(@__FILE__)") - println("PROGRAM_FILE: $(PROGRAM_FILE)") - println("argv: $(argv)") - # Sometimes code accesses ARGS directly, as a global - println("ARGS: $ARGS") - println("Base.ARGS: $(Base.ARGS)") - println("Core.ARGS: $(Core.ARGS)") - return 0 + # Check stdlib filtering + if filter == true + @test !(occursin("LinearAlgebra", app_output)) + else + @test occursin("LinearAlgebra", app_output) end - """) - builddir = joinpath(basedir, "builddir") - outname = "args" - build_executable( - argsjlfile, outname; builddir = builddir - ) - # Check that the output from the program is as expected: - exe = joinpath(builddir, outname*executable_ext) - output = read(`$exe a b c`, String) - println(output) - @test output == - "@__FILE__: $argsjlfile\n" * - "PROGRAM_FILE: $exe\n" * - "argv: [\"a\", \"b\", \"c\"]\n" * - "ARGS: [\"a\", \"b\", \"c\"]\n" * - "Base.ARGS: [\"a\", \"b\", \"c\"]\n" * - "Core.ARGS: Any[\"$(Sys.iswindows() ? replace(exe, "\\" => "\\\\") : exe)\", \"a\", \"b\", \"c\"]\n" - end -end - -@testset "juliac" begin - mktempdir() do builddir - juliac = joinpath(@__DIR__, "..", "juliac.jl") - jlfile = joinpath(@__DIR__, "..", "examples", "hello.jl") - cfile = joinpath(@__DIR__, "..", "examples", "program.c") - @test success(`$julia $juliac -vaej $jlfile $cfile --builddir $builddir`) - @test isfile(joinpath(builddir, "hello.$(PackageCompiler.Libdl.dlext)")) - @test isfile(joinpath(builddir, "hello$executable_ext")) - @test success(`$(joinpath(builddir, "hello$executable_ext"))`) - @testset "--cc-flag" begin - # Try passing `--help` to $cc. This should work for any system compiler. - # Then grep the output for "-g", which should be present on any system. - @test occursin("-g", read(`$julia $juliac -se --cc-flag="--help" $jlfile $cfile --builddir $builddir`, String)) - # Just as a control, make sure that without passing '--help', we don't see "-g" - @test !occursin("-g", read(`$julia $juliac -se $jlfile $cfile --builddir $builddir`, String)) + # Check dependency run + @test occursin("Example.domath", app_output) + # Check jll package runs + @test occursin("Hello, World!", app_output) + # Check artifact runs + @test occursin("The result of 2*5^2 - 10 == 40.000000", app_output) + # Check artifact gets run from the correct place + @test occursin("HelloWorld artifact at $(realpath(app_compiled_dir))", app_output) end end end diff --git a/test/shipping.jl b/test/shipping.jl deleted file mode 100644 index ea925363..00000000 --- a/test/shipping.jl +++ /dev/null @@ -1,49 +0,0 @@ -using PackageCompiler, Pkg - -dir(f...) = joinpath(@__DIR__, f...) -cd(@__DIR__) -build_executable( - dir("makietest.jl"), - "makie", - dir("..", "examples", "program.c"); - snoopfile = dir("makiesnoop.jl"), - builddir = dir("build"), - verbose = true, quiet = false, - cpu_target = "x86-64", optimize = "3" -) -PackageCompiler.build_object( - dir("build", "julia_main.jl"), dir("build"), dir("build", "makie.o"), true, - nothing, nothing, "x86-64", "3", nothing, nothing, nothing, - nothing, nothing -) - - -packages = [ - "Quaternions", - "GLVisualize", - "StaticArrays", - "GeometryTypes", - "Reactive", - "GLAbstraction", - "GLWindow", - "AbstractNumbers", - "Contour", - "FileIO", - "Images", - "UnicodeFun", - "ColorBrewer", - "Interact", # for displaying signals of Image - a bit unfortunate - "Hiccup", - "Media", - "Juno", - "ModernGL", - "GLFW", - "Fontconfig", - "FreeType", - "FreeTypeAbstraction", - "ImageMagick", -] - -for elem in packages - cp(normpath(Base.find_package(elem), "..", ".."), dir("build", elem)) -end