Precompute all possible slider states

Project.toml

@@ -9,17 +9,22 @@ Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 BetterFileWatching = "c9fd44ac-77b5-486c-9482-9798bd063cc6"
 Configurations = "5218b696-f38b-4ac9-8b61-a12ec717816d"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 FromFile = "ff7dd447-1dcb-4ce3-b8ac-22a812192de7"
 Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2"
 GitHubActions = "6b79fd1a-b13a-48ab-b6b0-aaee1fee41df"
 Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
+Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
+OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Pluto = "c3e4b0f8-55cb-11ea-2926-15256bba5781"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 TerminalLoggers = "5d786b92-1e48-4d6f-9151-6b4477ca9bed"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
@@ -29,18 +34,21 @@ AbstractPlutoDingetjes = "1.1"
 BetterFileWatching = "^0.1.2"
 Configurations = "0.16, 0.17"
 FromFile = "0.1"
+Distributions = "0.25"
 Git = "1"
 GitHubActions = "0.1"
 Glob = "1"
 HTTP = "^1.0.2"
+OrderedCollections = "1"
 JSON = "0.21"
 Pluto = "0.19.18"
 TerminalLoggers = "0.1"
 julia = "1.6"
 
 [extras]
-Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Deno_jll = "04572ae6-984a-583e-9378-9577a1c2574d"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 
 [targets]
-test = ["Test", "Random"]
+test = ["Deno_jll", "Test", "Random"]

src/Actions.jl

@@ -1,12 +1,15 @@
-import Pluto: Pluto, without_pluto_file_extension, generate_html, @asynclog
+import Pluto:
+    Pluto, without_pluto_file_extension, generate_html, @asynclog, withtoken, Firebasey
 using Base64
 using FromFile
+import HTTP.URIs
 
 @from "./MoreAnalysis.jl" import bound_variable_connections_graph
 @from "./Export.jl" import try_get_exact_pluto_version, try_fromcache, try_tocache
 @from "./Types.jl" import NotebookSession, RunningNotebook, FinishedNotebook, RunResult
 @from "./Configuration.jl" import PlutoDeploySettings, is_glob_match
 @from "./FileHelpers.jl" import find_notebook_files_recursive
+@from "./precomputed/index.jl" import generate_precomputed_staterequests
 @from "./PlutoHash.jl" import plutohash
 
 
@@ -64,8 +67,12 @@ function process(
     end
 
     keep_running =
-        settings.SliderServer.enabled && !is_glob_match(path, settings.SliderServer.exclude)
-    skip_cache = keep_running || is_glob_match(path, settings.Export.ignore_cache)
+        (settings.SliderServer.enabled || settings.Precompute.enabled) &&
+        !is_glob_match(path, settings.SliderServer.exclude)
+    skip_cache =
+        keep_running ||
+        is_glob_match(path, settings.Export.ignore_cache) ||
+        path ∈ settings.Export.ignore_cache
 
     cached_state = skip_cache ? nothing : try_fromcache(settings.Export.cache_dir, new_hash)
 
@@ -115,9 +122,25 @@ function process(
         )
     end
 
+    new_session = NotebookSession(;
+        path=s.path,
+        current_hash=new_hash,
+        desired_hash=s.desired_hash,
+        run=run,
+    )
+    if settings.Precompute.enabled
+        generate_precomputed_staterequests(
+            new_session;
+            settings,
+            pluto_session=server_session,
+            output_dir,
+        )
+        # TODO shutdown
+    end
+
     @info "### ✓ $(progress) Ready" s.path new_hash
 
-    NotebookSession(; path, current_hash=new_hash, desired_hash=s.desired_hash, run)
+    new_session
 end
 
 ###
@@ -205,7 +228,11 @@ function generate_static_export(
 
     slider_server_running_somewhere =
         settings.Export.slider_server_url !== nothing ||
-        (settings.SliderServer.serve_static_export_folder && settings.SliderServer.enabled)
+        (
+            settings.SliderServer.serve_static_export_folder &&
+            settings.SliderServer.enabled
+        ) ||
+        settings.Precompute.enabled
 
     notebookfile_js = if settings.Export.offer_binder || slider_server_running_somewhere
         if settings.Export.baked_notebookfile
@@ -300,4 +327,7 @@ function remove_static_export(path; settings, output_dir)
        !settings.Export.baked_notebookfile
         tryrm(export_jl_path)
     end
-end
+
+
+
+end

src/Configuration.jl

@@ -1,7 +1,8 @@
 using Configurations
 import TOML
 import Pluto
-export SliderServerSettings, ExportSettings, PlutoDeploySettings, get_configuration
+export SliderServerSettings,
+    ExportSettings, PrecomputeSettings, PlutoDeploySettings, get_configuration
 using TerminalLoggers: TerminalLogger
 using Logging: global_logger
 using FromFile
@@ -24,6 +25,17 @@ import Glob
     simulated_lag::Real = 0
 end
 
+
+@extract_docs @option struct PrecomputeSettings
+    "Precompute slider server requests?"
+    enabled::Bool = false
+    "List of notebook files to skip precomputation. Provide paths relative to `start_dir`."
+    exclude::Vector{String} = String[]
+    "Whether or not to partially precompute notebooks. If `true`, notebooks will only be precomputed if **all** their sliders can be precomputed"
+    only_fully::Bool = false
+    max_filesize_per_group::Integer = 1_000_000
+end
+
 @extract_docs @option struct ExportSettings
     "Generate static HTML files? This setting can only be `false` if you are also running a slider server."
     enabled::Bool = true
@@ -57,6 +69,7 @@ end
 @option struct PlutoDeploySettings
     SliderServer::SliderServerSettings = SliderServerSettings()
     Export::ExportSettings = ExportSettings()
+    Precompute::PrecomputeSettings = PrecomputeSettings()
     Pluto::Pluto.Configuration.Options = Pluto.Configuration.Options()
 end
 

src/HTTPRouter.jl

@@ -12,6 +12,7 @@ using HTTP
 using Sockets
 import JSON
 
+@from "./run_bonds.jl" import run_bonds_get_patches
 @from "./IndexJSON.jl" import generate_index_json
 @from "./IndexHTML.jl" import temp_index, generate_basic_index_html
 @from "./Types.jl" import NotebookSession, RunningNotebook
@@ -35,7 +36,7 @@ function make_router(
 )
     router = HTTP.Router()
 
-    function get_sesh(request::HTTP.Request)
+    function get_sesh(request::HTTP.Request)::Union{Nothing,NotebookSession}
         uri = HTTP.URI(request.target)
 
         parts = HTTP.URIs.splitpath(uri.path)
@@ -62,7 +63,7 @@ function make_router(
         end
     end
 
-    function get_bonds(request::HTTP.Request)
+    function get_bonds(request::HTTP.Request)::Dict{Symbol,Any}
         request_body = if request.method == "POST"
             IOBuffer(HTTP.payload(request))
         elseif request.method == "GET"
@@ -104,67 +105,17 @@ function make_router(
                 lag > 0 && sleep(lag)
             end
 
-            topological_order, new_state = withtoken(sesh.run.token) do
-                try
-                    notebook.bonds = bonds
+            ##
+            result = run_bonds_get_patches(server_session, sesh.run, bonds)
+            ##
 
-                    names::Vector{Symbol} = Symbol.(keys(bonds))
-
-                    topological_order = Pluto.set_bond_values_reactive(
-                        session=server_session,
-                        notebook=notebook,
-                        bound_sym_names=names,
-                        is_first_values=[false for _n in names], # because requests should be stateless. We might want to do something special for the (actual) initial request (containing every initial bond value) in the future.
-                        run_async=false,
-                    )::Pluto.TopologicalOrder
-
-                    new_state = Pluto.notebook_to_js(notebook)
-
-                    topological_order, new_state
-                catch e
-                    @error "Failed to set bond values" exception = (e, catch_backtrace())
-                    nothing, nothing
-                end
-            end
-            topological_order === nothing && return (
+            if result === nothing
                 HTTP.Response(500, "Failed to set bond values") |>
                 with_cors! |>
                 with_not_cacheable!
-            )
-
-            ids_of_cells_that_ran = [c.cell_id for c in topological_order.runnable]
-
-            @debug "Finished running!" length(ids_of_cells_that_ran)
-
-            # We only want to send state updates about...
-            function only_relevant(state)
-                new = copy(state)
-                # ... the cells that just ran and ...
-                new["cell_results"] = filter(state["cell_results"]) do (id, cell_state)
-                    id ∈ ids_of_cells_that_ran
-                end
-                # ... nothing about bond values, because we don't want to synchronize among clients. and...
-                delete!(new, "bonds")
-                # ... we ignore changes to the status tree caused by a running bonds.
-                delete!(new, "status_tree")
-                new
             end
 
-            patches = Firebasey.diff(
-                only_relevant(sesh.run.original_state),
-                only_relevant(new_state),
-            )
-            patches_as_dicts::Array{Dict} = Firebasey._convert(Array{Dict}, patches)
-
-            HTTP.Response(
-                200,
-                Pluto.pack(
-                    Dict{String,Any}(
-                        "patches" => patches_as_dicts,
-                        "ids_of_cells_that_ran" => ids_of_cells_that_ran,
-                    ),
-                ),
-            ) |>
+            HTTP.Response(200, Pluto.pack(result)) |>
             with_cacheable! |>
             with_cors! |>
             with_msgpack!

src/PlutoSliderServer.jl

@@ -17,6 +17,7 @@ using FromFile
 @from "./ReloadFolder.jl" import update_sessions!, select
 @from "./HTTPRouter.jl" import make_router, ReferrerMiddleware
 @from "./gitpull.jl" import fetch_pull
+@from "./precomputed/debug.jl" import start_debugging
 
 @from "./PlutoHash.jl" import plutohash, base64urlencode, base64urldecode
 export plutohash, base64urlencode, base64urldecode
@@ -528,4 +529,5 @@ function kind_of_debounced(f)
 end
 
 
+
 end

src/precomputed/debug.jl

@@ -0,0 +1,62 @@
+
+# this file is included in src/PlutoSliderServer.jl
+# yolo
+
+import Pluto: Pluto, @asynclog, tamepath, is_pluto_notebook
+import Configurations
+using FromFile
+
+@from "./index.jl" import variable_groups, generate_precomputed_staterequests_report
+@from "../Types.jl" import RunningNotebook
+@from "../Configuration.jl" import PlutoDeploySettings
+@from "../MoreAnalysis.jl" import bound_variable_connections_graph
+
+
+function start_debugging(notebook_path::String; kwargs...)
+    notebook_path = tamepath(notebook_path)
+    @assert is_pluto_notebook(notebook_path)
+
+    settings = Configurations.from_kwargs(PlutoDeploySettings; kwargs...)
+
+
+    @info "Running notebook..."
+
+    pluto_session = Pluto.ServerSession(; options=settings.Pluto)
+    notebook = Pluto.SessionActions.open(pluto_session, notebook_path; run_async=false)
+
+    @info "Notebook ready! Starting server..."
+
+    pluto_session.options.server.show_file_system = false
+    t = @asynclog Pluto.run(pluto_session)
+    sleep(1)
+
+    repeat = true
+    while repeat
+        connections = bound_variable_connections_graph(notebook)
+
+        run = RunningNotebook(;
+            path=notebook_path,
+            notebook=notebook,
+            bond_connections=connections,
+            original_state=Pluto.notebook_to_js(notebook),
+        )
+
+        groups = variable_groups(connections; pluto_session, notebook=run.notebook)
+
+        report =
+            generate_precomputed_staterequests_report(groups, run; settings, pluto_session)
+
+        for _ = 1:first(displaysize(stdout))
+            println(stdout)
+        end
+        show(stdout, MIME"text/plain"(), report)
+        println(stdout)
+        println(stdout)
+
+
+        repeat = Base.prompt("Run again? (y/n)"; default="y") == "y"
+    end
+
+    wait(t)
+
+end