From ba8a012b547317dc14831c18d8b66fc4ba128521 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 25 Oct 2021 15:33:14 +0200 Subject: [PATCH 01/29] hello --- Project.toml | 1 + src/Actions.jl | 123 ++++++++++++++++++++++++++++++++++++++- src/HTTPRouter.jl | 62 +++++--------------- src/PlutoSliderServer.jl | 9 ++- test/staterequest.jl | 8 ++- 5 files changed, 148 insertions(+), 55 deletions(-) diff --git a/Project.toml b/Project.toml index 4f72298..24028cf 100644 --- a/Project.toml +++ b/Project.toml @@ -11,6 +11,7 @@ FromFile = "ff7dd447-1dcb-4ce3-b8ac-22a812192de7" GitHubActions = "6b79fd1a-b13a-48ab-b6b0-aaee1fee41df" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Pluto = "c3e4b0f8-55cb-11ea-2926-15256bba5781" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" diff --git a/src/Actions.jl b/src/Actions.jl index a837c45..160b30c 100644 --- a/src/Actions.jl +++ b/src/Actions.jl @@ -1,7 +1,9 @@ module Actions - import Pluto: Pluto, without_pluto_file_extension + import Pluto: Pluto, ServerSession, Firebasey, Token, withtoken, pluto_file_extensions, without_pluto_file_extension using Base64 using SHA + using OrderedCollections + import HTTP using FromFile @from "./MoreAnalysis.jl" import MoreAnalysis @from "./Export.jl" using Export @@ -136,4 +138,123 @@ module Actions @info "Written to $(export_html_path)" end + + + function run_bonds_get_patch_info(server_session, notebook_session::NotebookSession, bonds::AbstractDict{Symbol,<:Any})::Union{AbstractDict{String,Any},Nothing} + sesh = notebook_session + + notebook = sesh.notebook + + topological_order, new_state = withtoken(sesh.token) do + try + notebook.bonds = bonds + + names::Vector{Symbol} = Symbol.(keys(bonds)) + + topological_order = Pluto.set_bond_values_reactive( + session=server_session, + notebook=notebook, + bound_sym_names=names, + run_async=false, + )::Pluto.TopologicalOrder + + new_state = Pluto.notebook_to_js(notebook) + + topological_order, new_state + catch e + @error "Failed to set bond values" exception=(e, catch_backtrace()) + nothing, nothing + end + end + if topological_order === nothing + return nothing + end + + ids_of_cells_that_ran = [c.cell_id for c in topological_order.runnable] + + @debug "Finished running!" length(ids_of_cells_that_ran) + + # We only want to send state updates about... + function only_relevant(state) + new = copy(state) + # ... the cells that just ran and ... + new["cell_results"] = filter(state["cell_results"]) do (id, cell_state) + id ∈ ids_of_cells_that_ran + end + # ... nothing about bond values, because we don't want to synchronize among clients. + new["bonds"] = Dict{String,Dict{String,Any}}() + new + end + + patches = Firebasey.diff(only_relevant(sesh.original_state), only_relevant(new_state)) + patches_as_dicts::Array{Dict} = patches + + Dict{String,Any}( + "patches" => patches_as_dicts, + "ids_of_cells_that_ran" => ids_of_cells_that_ran, + ) + end + + function generate_static_staterequests(path, settings::PlutoDeploySettings, pluto_session::Pluto.ServerSession, notebook_session::NotebookSession, output_dir=".") + sesh = notebook_session + connections = sesh.bond_connections + + mkpath( + joinpath( + output_dir, + "bondconnections", + ) + ) + + mkpath(joinpath( + output_dir, + "staterequest", + HTTP.URIs.escapeuri(sesh.current_hash), + )) + + write_path = joinpath( + output_dir, + "bondconnections", + HTTP.URIs.escapeuri(sesh.current_hash) + ) + + write(write_path, Pluto.pack(sesh.bond_connections)) + + @info "Written bond connections to " write_path + + for variable_group in Set(values(connections)) + + names = sort(variable_group) + + possible_values = [Pluto.possible_bond_values(pluto_session::Pluto.ServerSession, sesh.notebook::Pluto.Notebook, n::Symbol) for n in names] + + for combination in Iterators.product(possible_values...) + bonds = OrderedDict{Symbol,Any}( + n => OrderedDict{String,Any}("value" => v, "is_first_value" => true) + for (n,v) in zip(names, combination) + ) + + result = run_bonds_get_patch_info(pluto_session, sesh, bonds) + + if result !== nothing + write_path = joinpath( + output_dir, + "staterequest", + HTTP.URIs.escapeuri(sesh.current_hash), + Pluto.pack(bonds) |> + base64encode |> + HTTP.URIs.escapeuri + ) + + write(write_path, Pluto.pack(result)) + + @info "Written state request to " write_path + + end + end + + end + + + end end \ No newline at end of file diff --git a/src/HTTPRouter.jl b/src/HTTPRouter.jl index de2c348..c1e9760 100644 --- a/src/HTTPRouter.jl +++ b/src/HTTPRouter.jl @@ -2,10 +2,13 @@ ### # HTTP ROUTER +using FromFile +@from "./Actions.jl" import Actions: run_bonds_get_patch_info + function make_router(settings::PlutoDeploySettings, server_session::ServerSession, notebook_sessions::AbstractVector{<:NotebookSession}; static_dir::Union{String,Nothing}=nothing) router = HTTP.Router() - function get_sesh(request::HTTP.Request) + function get_sesh(request::HTTP.Request)::Union{Nothing,NotebookSession} uri = HTTP.URI(request.target) parts = HTTP.URIs.splitpath(uri.path) @@ -32,7 +35,7 @@ function make_router(settings::PlutoDeploySettings, server_session::ServerSessio end end - function get_bonds(request::HTTP.Request) + function get_bonds(request::HTTP.Request)::Dict{Symbol,Any} request_body = if request.method == "POST" IOBuffer(HTTP.payload(request)) elseif request.method == "GET" @@ -70,53 +73,16 @@ function make_router(settings::PlutoDeploySettings, server_session::ServerSessio let lag = settings.SliderServer.simulated_lag lag > 0 && sleep(lag) end - - topological_order, new_state = withtoken(sesh.token) do - try - notebook.bonds = bonds - - names::Vector{Symbol} = Symbol.(keys(bonds)) - - topological_order = Pluto.set_bond_values_reactive( - session=server_session, - notebook=notebook, - bound_sym_names=names, - run_async=false, - )::Pluto.TopologicalOrder - - new_state = Pluto.notebook_to_js(notebook) - - topological_order, new_state - catch e - @error "Failed to set bond values" exception=(e, catch_backtrace()) - nothing, nothing - end - end - topological_order === nothing && return (HTTP.Response(500, "Failed to set bond values") |> with_cors! |> with_not_cachable!) - - ids_of_cells_that_ran = [c.cell_id for c in topological_order.runnable] - - @debug "Finished running!" length(ids_of_cells_that_ran) - - # We only want to send state updates about... - function only_relevant(state) - new = copy(state) - # ... the cells that just ran and ... - new["cell_results"] = filter(state["cell_results"]) do (id, cell_state) - id ∈ ids_of_cells_that_ran - end - # ... nothing about bond values, because we don't want to synchronize among clients. - new["bonds"] = Dict{String,Dict{String,Any}}() - new + + ## + result = run_bonds_get_patch_info(server_session, sesh, bonds) + ## + + if result === nothing + HTTP.Response(500, "Failed to set bond values") |> with_cors! |> with_not_cachable! + else + HTTP.Response(200, Pluto.pack(result)) |> with_cachable! |> with_cors! |> with_msgpack! end - - patches = Firebasey.diff(only_relevant(sesh.original_state), only_relevant(new_state)) - patches_as_dicts::Array{Dict} = patches - - HTTP.Response(200, Pluto.pack(Dict{String,Any}( - "patches" => patches_as_dicts, - "ids_of_cells_that_ran" => ids_of_cells_that_ran, - ))) |> with_cachable! |> with_cors! |> with_msgpack! elseif sesh isa QueuedNotebookSession HTTP.Response(503, "Still loading the notebooks... check back later!") |> with_cors! |> with_not_cachable! else diff --git a/src/PlutoSliderServer.jl b/src/PlutoSliderServer.jl index 399f5d2..0c511aa 100644 --- a/src/PlutoSliderServer.jl +++ b/src/PlutoSliderServer.jl @@ -5,7 +5,7 @@ using FromFile @from "./MoreAnalysis.jl" import MoreAnalysis @from "./FileHelpers.jl" import FileHelpers: find_notebook_files_recursive, list_files_recursive @from "./Export.jl" using Export -@from "./Actions.jl" import Actions: add_to_session!, generate_static_export, myhash, showall +@from "./Actions.jl" import Actions: add_to_session!, generate_static_export, generate_static_staterequests, myhash, showall @from "./Types.jl" using Types import Pluto @@ -96,7 +96,7 @@ If `static_export` is `true`, then additional `Export_` keywords can be given, s function run_directory( start_dir::String="."; notebook_paths::Vector{String}=find_notebook_files_recursive(start_dir), - static_export::Bool=false, run_server::Bool=true, + static_export::Bool=false, static_export_state::Bool=false, run_server::Bool=true, on_ready::Function=((args...)->()), config_toml_path::Union{String,Nothing}=joinpath(Base.active_project() |> dirname, "PlutoDeployment.toml"), kwargs... @@ -212,7 +212,7 @@ function run_directory( local session, jl_contents, original_state # That's because you can't continue in a loop try - session, jl_contents, original_state = add_to_session!(notebook_sessions, server_session, path, settings, run_server, start_dir) + session, jl_contents, original_state = add_to_session!(notebook_sessions, server_session, path, settings, run_server || static_export_state, start_dir) catch e rethrow(e) continue @@ -220,6 +220,9 @@ function run_directory( if static_export generate_static_export(path, settings, original_state, output_dir, jl_contents) + if static_export_state + generate_static_staterequests(path, settings, server_session, session, output_dir) + end end @info "[$(i)/$(length(to_run))] Ready $(path)" session.current_hash diff --git a/test/staterequest.jl b/test/staterequest.jl index e66e94d..b522e89 100644 --- a/test/staterequest.jl +++ b/test/staterequest.jl @@ -49,11 +49,13 @@ using Base64 @testset "Bond connections - $(name)" for (i, name) in enumerate(notebook_paths) s = notebook_sessions[i] - response = HTTP.get("http://localhost:$(port)/bondconnections/$(HTTP.URIs.escapeuri(s.current_hash))/") + for ending in ["", "/"] + response = HTTP.get("http://localhost:$(port)/bondconnections/$(HTTP.URIs.escapeuri(s.current_hash))" * ending) - result = Pluto.unpack(response.body) + result = Pluto.unpack(response.body) - @test result == Dict(String(k) => String.(v) for (k,v) in s.bond_connections) + @test result == Dict(String(k) => String.(v) for (k,v) in s.bond_connections) + end end From 53f72d6b5f0acbaedb347f34bb2d0a4285ce9090 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 29 Nov 2021 19:45:02 +0100 Subject: [PATCH 02/29] Update Configuration.jl --- src/Configuration.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Configuration.jl b/src/Configuration.jl index 865d7be..d1d5999 100644 --- a/src/Configuration.jl +++ b/src/Configuration.jl @@ -26,6 +26,7 @@ end @extract_docs @option struct ExportSettings "Generate static HTML files? This setting can only be `false` if you are also running a slider server." enabled::Bool = true + static_export_state::Bool = false "Folder to write generated HTML files to (will create directories to preserve the input folder structure). The behaviour of the default value depends on whether you are running the slider server, or just exporting. If running the slider server, we use a temporary directory; otherwise, we use `start_dir` (i.e. we generate each HTML file in the same folder as the notebook file)." output_dir::Union{Nothing,String} = nothing "List of notebook files to skip. Provide paths relative to `start_dir`." From 31dcfbab84df71a97fa2042073cab74209710259 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 29 Nov 2021 19:55:34 +0100 Subject: [PATCH 03/29] merge part 2 --- src/HTTPRouter.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/HTTPRouter.jl b/src/HTTPRouter.jl index 1269b12..1b0cdd5 100644 --- a/src/HTTPRouter.jl +++ b/src/HTTPRouter.jl @@ -16,7 +16,7 @@ using Sockets using Logging: global_logger using GitHubActions: GitHubActionsLogger -@from "./Actions.jl" import Actions: run_bonds_get_patch_info +@from "./Actions.jl" import run_bonds_get_patch_info @from "./Export.jl" import generate_index_html @from "./Types.jl" import NotebookSession, RunningNotebook @from "./Configuration.jl" import PlutoDeploySettings, get_configuration @@ -50,7 +50,7 @@ function make_router( This means that the notebook file used by the web client does not precisely match any of the notebook files running in this server. If this is an automated setup, then this could happen inotebooketween deployments. - + If this is a manual setup, then running the .jl notebook file might have caused a small change (e.g. the version number or a whitespace change). Copy notebooks to a temporary directory before running them using the bind server. =# @info "Request hash not found. See errror hint in my source code." notebook_hash nothing From 4f98d91a72007a52161501773a2bf08e9ee68a24 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 29 Nov 2021 20:06:57 +0100 Subject: [PATCH 04/29] merge part 3 --- src/Actions.jl | 189 +++++++++++++++++++++++++------------------------ 1 file changed, 95 insertions(+), 94 deletions(-) diff --git a/src/Actions.jl b/src/Actions.jl index 83e4f43..20eaac8 100644 --- a/src/Actions.jl +++ b/src/Actions.jl @@ -305,125 +305,126 @@ function remove_static_export(path; settings, output_dir) end - function run_bonds_get_patch_info( - server_session, - notebook_session::NotebookSession, - bonds::AbstractDict{Symbol,<:Any}, - )::Union{AbstractDict{String,Any},Nothing} - sesh = notebook_session - - notebook = sesh.notebook - - topological_order, new_state = withtoken(sesh.run.token) do - try - notebook.bonds = bonds - - names::Vector{Symbol} = Symbol.(keys(bonds)) - - topological_order = Pluto.set_bond_values_reactive( - session=server_session, - notebook=notebook, - bound_sym_names=names, - run_async=false, - )::Pluto.TopologicalOrder - - new_state = Pluto.notebook_to_js(notebook) - - topological_order, new_state - catch e - @error "Failed to set bond values" exception = (e, catch_backtrace()) - nothing, nothing - end - end - if topological_order === nothing - return nothing - end - ids_of_cells_that_ran = [c.cell_id for c in topological_order.runnable] +end + +function run_bonds_get_patch_info( + server_session, + notebook_session::NotebookSession, + bonds::AbstractDict{Symbol,<:Any}, +)::Union{AbstractDict{String,Any},Nothing} + sesh = notebook_session - @debug "Finished running!" length(ids_of_cells_that_ran) + notebook = sesh.notebook - # We only want to send state updates about... - function only_relevant(state) - new = copy(state) - # ... the cells that just ran and ... - new["cell_results"] = filter(state["cell_results"]) do (id, cell_state) - id ∈ ids_of_cells_that_ran - end - # ... nothing about bond values, because we don't want to synchronize among clients. - new["bonds"] = Dict{String,Dict{String,Any}}() - new + topological_order, new_state = withtoken(sesh.run.token) do + try + notebook.bonds = bonds + + names::Vector{Symbol} = Symbol.(keys(bonds)) + + topological_order = Pluto.set_bond_values_reactive( + session=server_session, + notebook=notebook, + bound_sym_names=names, + run_async=false, + )::Pluto.TopologicalOrder + + new_state = Pluto.notebook_to_js(notebook) + + topological_order, new_state + catch e + @error "Failed to set bond values" exception = (e, catch_backtrace()) + nothing, nothing end + end + if topological_order === nothing + return nothing + end - patches = - Firebasey.diff(only_relevant(sesh.run.original_state), only_relevant(new_state)) - patches_as_dicts::Array{Dict} = patches + ids_of_cells_that_ran = [c.cell_id for c in topological_order.runnable] - Dict{String,Any}( - "patches" => patches_as_dicts, - "ids_of_cells_that_ran" => ids_of_cells_that_ran, - ) + @debug "Finished running!" length(ids_of_cells_that_ran) + + # We only want to send state updates about... + function only_relevant(state) + new = copy(state) + # ... the cells that just ran and ... + new["cell_results"] = filter(state["cell_results"]) do (id, cell_state) + id ∈ ids_of_cells_that_ran + end + # ... nothing about bond values, because we don't want to synchronize among clients. + new["bonds"] = Dict{String,Dict{String,Any}}() + new end - function generate_static_staterequests( - path, - settings::PlutoDeploySettings, - pluto_session::Pluto.ServerSession, - notebook_session::NotebookSession, - output_dir=".", + patches = + Firebasey.diff(only_relevant(sesh.run.original_state), only_relevant(new_state)) + patches_as_dicts::Array{Dict} = patches + + Dict{String,Any}( + "patches" => patches_as_dicts, + "ids_of_cells_that_ran" => ids_of_cells_that_ran, ) - sesh = notebook_session - connections = sesh.bond_connections +end - mkpath(joinpath(output_dir, "bondconnections")) +function generate_static_staterequests( + path, + settings::PlutoDeploySettings, + pluto_session::Pluto.ServerSession, + notebook_session::NotebookSession, + output_dir=".", +) + sesh = notebook_session + connections = sesh.bond_connections - mkpath(joinpath(output_dir, "staterequest", HTTP.URIs.escapeuri(sesh.current_hash))) + mkpath(joinpath(output_dir, "bondconnections")) - write_path = - joinpath(output_dir, "bondconnections", HTTP.URIs.escapeuri(sesh.current_hash)) + mkpath(joinpath(output_dir, "staterequest", HTTP.URIs.escapeuri(sesh.current_hash))) - write(write_path, Pluto.pack(sesh.bond_connections)) + write_path = + joinpath(output_dir, "bondconnections", HTTP.URIs.escapeuri(sesh.current_hash)) - @info "Written bond connections to " write_path + write(write_path, Pluto.pack(sesh.bond_connections)) - for variable_group in Set(values(connections)) + @info "Written bond connections to " write_path - names = sort(variable_group) + for variable_group in Set(values(connections)) - possible_values = [ - Pluto.possible_bond_values( - pluto_session::Pluto.ServerSession, - sesh.notebook::Pluto.Notebook, - n::Symbol, - ) for n in names - ] + names = sort(variable_group) - for combination in Iterators.product(possible_values...) - bonds = OrderedDict{Symbol,Any}( - n => - OrderedDict{String,Any}("value" => v, "is_first_value" => true) - for (n, v) in zip(names, combination) - ) + possible_values = [ + Pluto.possible_bond_values( + pluto_session::Pluto.ServerSession, + sesh.notebook::Pluto.Notebook, + n::Symbol, + ) for n in names + ] + + for combination in Iterators.product(possible_values...) + bonds = OrderedDict{Symbol,Any}( + n => OrderedDict{String,Any}("value" => v, "is_first_value" => true) for + (n, v) in zip(names, combination) + ) - result = run_bonds_get_patch_info(pluto_session, sesh, bonds) + result = run_bonds_get_patch_info(pluto_session, sesh, bonds) - if result !== nothing - write_path = joinpath( - output_dir, - "staterequest", - HTTP.URIs.escapeuri(sesh.current_hash), - Pluto.pack(bonds) |> base64encode |> HTTP.URIs.escapeuri, - ) + if result !== nothing + write_path = joinpath( + output_dir, + "staterequest", + HTTP.URIs.escapeuri(sesh.current_hash), + Pluto.pack(bonds) |> base64encode |> HTTP.URIs.escapeuri, + ) - write(write_path, Pluto.pack(result)) + write(write_path, Pluto.pack(result)) - @info "Written state request to " write_path + @info "Written state request to " write_path - end end - end - end + + end \ No newline at end of file From df6881b1c1730eb896f71d77a5f545a2adcab32e Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 29 Nov 2021 20:07:00 +0100 Subject: [PATCH 05/29] Update Project.toml --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index a5394f7..0ea5085 100644 --- a/Project.toml +++ b/Project.toml @@ -29,6 +29,7 @@ FromFile = "0.1" Git = "1" GitHubActions = "0.1" HTTP = "^0.9.3" +OrderedCollections = "1" Pluto = "0.15, 0.16, 0.17" TerminalLoggers = "0.1" julia = "1.6" From 10043431affaf8153ea3986edfabedd2a8268a45 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 30 Nov 2021 00:32:32 +0100 Subject: [PATCH 06/29] Update Actions.jl --- src/Actions.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Actions.jl b/src/Actions.jl index 20eaac8..2b803d5 100644 --- a/src/Actions.jl +++ b/src/Actions.jl @@ -315,7 +315,7 @@ function run_bonds_get_patch_info( )::Union{AbstractDict{String,Any},Nothing} sesh = notebook_session - notebook = sesh.notebook + notebook = sesh.run.notebook topological_order, new_state = withtoken(sesh.run.token) do try @@ -396,7 +396,7 @@ function generate_static_staterequests( possible_values = [ Pluto.possible_bond_values( pluto_session::Pluto.ServerSession, - sesh.notebook::Pluto.Notebook, + sesh.run.notebook::Pluto.Notebook, n::Symbol, ) for n in names ] From 28908d305307fc91448772e898d8689951b5b0fa Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 30 Nov 2021 02:01:29 +0100 Subject: [PATCH 07/29] tests --- Project.toml | 3 +- src/Actions.jl | 68 +++++++----- test/basic3.jl | 208 ++++++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + test/staterequest static.jl | 150 ++++++++++++++++++++++++++ 5 files changed, 405 insertions(+), 25 deletions(-) create mode 100644 test/basic3.jl create mode 100644 test/staterequest static.jl diff --git a/Project.toml b/Project.toml index 0ea5085..b82e05b 100644 --- a/Project.toml +++ b/Project.toml @@ -35,7 +35,8 @@ TerminalLoggers = "0.1" julia = "1.6" [extras] +Deno_jll = "04572ae6-984a-583e-9378-9577a1c2574d" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test"] +test = ["Deno_jll", "Test"] diff --git a/src/Actions.jl b/src/Actions.jl index 2b803d5..25c0fa0 100644 --- a/src/Actions.jl +++ b/src/Actions.jl @@ -1,8 +1,10 @@ -import Pluto: Pluto, without_pluto_file_extension, generate_html, @asynclog +import Pluto: + Pluto, without_pluto_file_extension, generate_html, @asynclog, withtoken, Firebasey using Base64 using SHA using OrderedCollections using FromFile +import HTTP.URIs @from "./MoreAnalysis.jl" import bound_variable_connections_graph @from "./Export.jl" import try_get_exact_pluto_version, try_fromcache, try_tocache @@ -67,7 +69,7 @@ function process( @warn "Notebook file does not have desired hash. This probably means that the file changed too quickly. Continuing and hoping for the best!" s.path new_hash s.desired_hash end - keep_running = settings.SliderServer.enabled + keep_running = settings.SliderServer.enabled || settings.Export.static_export_state skip_cache = keep_running || path ∈ settings.Export.ignore_cache cached_state = skip_cache ? nothing : try_fromcache(settings.Export.cache_dir, new_hash) @@ -113,18 +115,25 @@ function process( start_dir, output_dir, ) - if settings.Export.static_export_state - generate_static_staterequests(path, settings, server_session, session, output_dir) - end - - @info "### ✓ $(progress) Ready" s.path new_hash - - NotebookSession(; + new_session = NotebookSession(; path=s.path, current_hash=new_hash, desired_hash=s.desired_hash, run=run, ) + if settings.Export.static_export_state + generate_static_staterequests( + new_session; + settings, + pluto_session=server_session, + output_dir, + ) + # TODO shutdown + end + + @info "### ✓ $(progress) Ready" s.path new_hash + + new_session end ### @@ -369,23 +378,25 @@ function run_bonds_get_patch_info( end function generate_static_staterequests( - path, + notebook_session::NotebookSession; settings::PlutoDeploySettings, pluto_session::Pluto.ServerSession, - notebook_session::NotebookSession, output_dir=".", ) sesh = notebook_session - connections = sesh.bond_connections + run = sesh.run + connections = run.bond_connections + current_hash = sesh.current_hash + + @assert run isa RunningNotebook mkpath(joinpath(output_dir, "bondconnections")) - mkpath(joinpath(output_dir, "staterequest", HTTP.URIs.escapeuri(sesh.current_hash))) + mkpath(joinpath(output_dir, "staterequest", URIs.escapeuri(current_hash))) - write_path = - joinpath(output_dir, "bondconnections", HTTP.URIs.escapeuri(sesh.current_hash)) + write_path = joinpath(output_dir, "bondconnections", URIs.escapeuri(current_hash)) - write(write_path, Pluto.pack(sesh.bond_connections)) + write(write_path, Pluto.pack(run.bond_connections)) @info "Written bond connections to " write_path @@ -394,11 +405,19 @@ function generate_static_staterequests( names = sort(variable_group) possible_values = [ - Pluto.possible_bond_values( - pluto_session::Pluto.ServerSession, - sesh.run.notebook::Pluto.Notebook, - n::Symbol, - ) for n in names + let + result = Pluto.possible_bond_values( + pluto_session::Pluto.ServerSession, + run.notebook::Pluto.Notebook, + n::Symbol, + ) + if result isa Symbol + @error "Failed to get possible values for $(n)" result + [] + else + result + end + end for n in names ] for combination in Iterators.product(possible_values...) @@ -413,13 +432,14 @@ function generate_static_staterequests( write_path = joinpath( output_dir, "staterequest", - HTTP.URIs.escapeuri(sesh.current_hash), - Pluto.pack(bonds) |> base64encode |> HTTP.URIs.escapeuri, + URIs.escapeuri(current_hash), + Pluto.pack(bonds) |> base64encode |> URIs.escapeuri, ) write(write_path, Pluto.pack(result)) - @info "Written state request to " write_path + @info "Written state request to " write_path values = + (; (zip(names, combination))...) end end diff --git a/test/basic3.jl b/test/basic3.jl new file mode 100644 index 0000000..29dfade --- /dev/null +++ b/test/basic3.jl @@ -0,0 +1,208 @@ +### A Pluto.jl notebook ### +# v0.17.2 + +using Markdown +using InteractiveUtils + +# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error). +macro bind(def, element) + quote + local iv = try Base.loaded_modules[Base.PkgId(Base.UUID("6e696c72-6542-2067-7265-42206c756150"), "AbstractPlutoDingetjes")].Bonds.initial_value catch; b -> missing; end + local el = $(esc(element)) + global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : iv(el) + el + end +end + +# ╔═╡ 22dc8ce8-392e-4202-a549-f0fd46152322 +import AbstractPlutoDingetjes.Bonds + +# ╔═╡ 9b342ef4-fff9-46d0-b316-cc383fe71a59 +begin + struct CoolSlider + end + function Base.show(io::IO, ::MIME"text/html", ::CoolSlider) + write(io, "") + end + Bonds.initial_value(::CoolSlider) = 1 + Bonds.possible_values(::CoolSlider) = 1:10 +end + +# ╔═╡ 635e5ebc-6567-11eb-1d9d-f98bfca7ec27 +@bind x CoolSlider() + +# ╔═╡ ad853ac9-a8a0-44ef-8d41-c8cea165ad57 +@bind y CoolSlider() + +# ╔═╡ 26025270-9b5e-4841-b295-0c47437bc7db +x + y + +# ╔═╡ 1352da54-e567-4f59-a3da-19ed3f4bb7c7 + + +# ╔═╡ 09ae27fa-525a-4211-b252-960cdbaf1c1e +b = @bind s html"" + +# ╔═╡ ed78a6f1-d282-4d80-8f42-40701aeadb52 +b + +# ╔═╡ cca2f726-0c25-43c6-85e4-c16ec192d464 +s + +# ╔═╡ c4f51980-3c30-4d3f-a76a-fc0f0fe16944 + + +# ╔═╡ 8f0bd329-36b8-45ed-b80d-24661242129a +b2 = @bind s2 html"" + +# ╔═╡ c55a107f-5d7d-4396-b597-8c1ae07c35be +b2 + +# ╔═╡ a524ff27-a6a3-4f14-8ed4-f55700647bc4 +sleep(1); s2 + +# ╔═╡ 00000000-0000-0000-0000-000000000001 +PLUTO_PROJECT_TOML_CONTENTS = """ +[deps] +AbstractPlutoDingetjes = "6e696c72-6542-2067-7265-42206c756150" + +[compat] +AbstractPlutoDingetjes = "~1.1.2" +""" + +# ╔═╡ 00000000-0000-0000-0000-000000000002 +PLUTO_MANIFEST_TOML_CONTENTS = """ +# This file is machine-generated - editing it directly is not advised + +[[AbstractPlutoDingetjes]] +deps = ["Pkg"] +git-tree-sha1 = "abb72771fd8895a7ebd83d5632dc4b989b022b5b" +uuid = "6e696c72-6542-2067-7265-42206c756150" +version = "1.1.2" + +[[ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" + +[[Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[Downloads]] +deps = ["ArgTools", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" + +[[LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" + +[[LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" + +[[MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" + +[[NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" + +[[Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" + +[[Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" + +[[nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" + +[[p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +""" + +# ╔═╡ Cell order: +# ╠═22dc8ce8-392e-4202-a549-f0fd46152322 +# ╠═9b342ef4-fff9-46d0-b316-cc383fe71a59 +# ╠═635e5ebc-6567-11eb-1d9d-f98bfca7ec27 +# ╠═ad853ac9-a8a0-44ef-8d41-c8cea165ad57 +# ╠═26025270-9b5e-4841-b295-0c47437bc7db +# ╟─1352da54-e567-4f59-a3da-19ed3f4bb7c7 +# ╠═09ae27fa-525a-4211-b252-960cdbaf1c1e +# ╠═ed78a6f1-d282-4d80-8f42-40701aeadb52 +# ╠═cca2f726-0c25-43c6-85e4-c16ec192d464 +# ╟─c4f51980-3c30-4d3f-a76a-fc0f0fe16944 +# ╠═8f0bd329-36b8-45ed-b80d-24661242129a +# ╠═c55a107f-5d7d-4396-b597-8c1ae07c35be +# ╠═a524ff27-a6a3-4f14-8ed4-f55700647bc4 +# ╟─00000000-0000-0000-0000-000000000001 +# ╟─00000000-0000-0000-0000-000000000002 diff --git a/test/runtests.jl b/test/runtests.jl index 2ad685b..a96d6d6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,6 +7,7 @@ if just_run_test_server include("./runtestserver.jl") else ENV["HIDE_PLUTO_EXACT_VERSION_WARNING"] = "true" + include("./staterequest static.jl") include("./configuration.jl") include("./static export.jl") include("./staterequest.jl") diff --git a/test/staterequest static.jl b/test/staterequest static.jl new file mode 100644 index 0000000..826d350 --- /dev/null +++ b/test/staterequest static.jl @@ -0,0 +1,150 @@ +import PlutoSliderServer: PlutoSliderServer, Pluto, list_files_recursive +import HTTP +import Deno_jll +using OrderedCollections + +using Test +using UUIDs +using Base64 + +@testset "HTTP requests" begin + test_dir = tempname(cleanup=false) + cp(@__DIR__, test_dir) + + notebook_paths = ["basic3.jl"] + # notebook_paths = ["basic2.jl", "parallelpaths4.jl"] + + port = rand(12345:65000) + + + still_booting = Ref(true) + ready_result = Ref{Any}(nothing) + function on_ready(result) + ready_result[] = result + still_booting[] = false + end + + + t = Pluto.@asynclog begin + try + run( + `$(Deno_jll.deno()) run --allow-net --allow-read https://deno.land/std@0.115.0/http/file_server.ts $(test_dir) --cors --port $(port)`, + ) + catch e + if !(e isa TaskFailedException) && !(e isa InterruptException) + showerror(stderr, e, stacktrace(catch_backtrace())) + end + end + end + + PlutoSliderServer.export_directory( + test_dir; + Export_static_export_state=true, + notebook_paths, + on_ready, + ) + + @test isdir(joinpath(test_dir, "staterequest")) + @test isdir(joinpath(test_dir, "bondconnections")) + @show list_files_recursive(joinpath(test_dir, "staterequest")) + @show readdir(joinpath(test_dir, "bondconnections")) + @test length(list_files_recursive(joinpath(test_dir, "staterequest"))) == 10 * 10 + + while !occursin( + "Pluto.jl notebook", + read(download("http://localhost:$(port)/basic3.jl"), String), + ) + @info "Waiting for file server to start" + sleep(0.1) + end + + while still_booting[] + sleep(0.1) + end + + + notebook_sessions = ready_result[].notebook_sessions + + @show notebook_paths [ + (s.path, typeof(s.run), s.current_hash) for s in notebook_sessions + ] + + @testset "Bond connections - $(name)" for (i, name) in enumerate(notebook_paths) + s = notebook_sessions[i] + + for ending in [""] + response = HTTP.get( + "http://localhost:$(port)/bondconnections/$(HTTP.URIs.escapeuri(s.current_hash))" * + ending, + ) + + result = Pluto.unpack(response.body) + + @test result == + Dict(String(k) => String.(v) for (k, v) in s.run.bond_connections) + end + end + + + @testset "State request - basic3.jl" begin + i = 1 + s = notebook_sessions[i] + + @testset "Method $(method)" for method in ["GET"], x = 3:7 + + v(x) = OrderedDict("value" => x, "is_first_value" => true) + + bonds = OrderedDict("x" => v(x), "y" => v(7)) + + state = Pluto.unpack(Pluto.pack(s.run.original_state)) + + sum_cell_id = "26025270-9b5e-4841-b295-0c47437bc7db" + + response = if method == "GET" + arg = Pluto.pack(bonds) |> base64encode |> HTTP.URIs.escapeuri + + HTTP.request( + method, + "http://localhost:$(port)/staterequest/$(HTTP.URIs.escapeuri(s.current_hash))/$(arg)", + ) + else + HTTP.request( + method, + "http://localhost:$(port)/staterequest/$(HTTP.URIs.escapeuri(s.current_hash))/", + [], + Pluto.pack(bonds), + ) + end + + result = Pluto.unpack(response.body) + + @test sum_cell_id ∈ result["ids_of_cells_that_ran"] + + for patch in result["patches"] + Pluto.Firebasey.applypatch!( + state, + convert(Pluto.Firebasey.JSONPatch, patch), + ) + end + + @test state["cell_results"][sum_cell_id]["output"]["body"] == + string(bonds["x"]["value"] + bonds["y"]["value"]) + + end + end + + # close(ready_result[].serversocket) + + try + schedule(t, InterruptException(); error=true) + wait(t) + catch e + if !(e isa TaskFailedException) && !(e isa InterruptException) + rethrow(e) + end + end + # schedule(t, InterruptException(), error=true) + @info "DONEZO" + + @test true +end \ No newline at end of file From 072e9a2ef1f34fe1dcc74d1932866e35c1729d82 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 30 Nov 2021 02:21:44 +0100 Subject: [PATCH 08/29] silly rand --- Project.toml | 3 ++- test/staterequest static.jl | 3 ++- test/staterequest.jl | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Project.toml b/Project.toml index b82e05b..5c9e5fe 100644 --- a/Project.toml +++ b/Project.toml @@ -36,7 +36,8 @@ julia = "1.6" [extras] Deno_jll = "04572ae6-984a-583e-9378-9577a1c2574d" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Deno_jll", "Test"] +test = ["Deno_jll", "Random", "Test"] diff --git a/test/staterequest static.jl b/test/staterequest static.jl index 826d350..5538e45 100644 --- a/test/staterequest static.jl +++ b/test/staterequest static.jl @@ -3,6 +3,7 @@ import HTTP import Deno_jll using OrderedCollections +import Random using Test using UUIDs using Base64 @@ -14,7 +15,7 @@ using Base64 notebook_paths = ["basic3.jl"] # notebook_paths = ["basic2.jl", "parallelpaths4.jl"] - port = rand(12345:65000) + port = rand(Random.RandomDevice(), 12345:65000) still_booting = Ref(true) diff --git a/test/staterequest.jl b/test/staterequest.jl index 65f0ff7..e4049fb 100644 --- a/test/staterequest.jl +++ b/test/staterequest.jl @@ -2,6 +2,7 @@ import PlutoSliderServer import PlutoSliderServer.Pluto import PlutoSliderServer.HTTP +import Random using Test using UUIDs using Base64 @@ -12,7 +13,8 @@ using Base64 notebook_paths = ["basic2.jl", "parallelpaths4.jl"] - port = rand(12345:65000) + + port = rand(Random.RandomDevice(), 12345:65000) still_booting = Ref(true) From 81af8718aa7b58d47696d80b85094bfbd038e7e7 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 30 Nov 2021 13:51:10 +0100 Subject: [PATCH 09/29] rng for ProductIterator --- src/Actions.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Actions.jl b/src/Actions.jl index 25c0fa0..1a92e84 100644 --- a/src/Actions.jl +++ b/src/Actions.jl @@ -447,4 +447,12 @@ function generate_static_staterequests( end +end + +function Random.rand( + rng::Random.AbstractRNG, + iterator::Random.SamplerTrivial{Base.Iterators.ProductIterator{T}}, +) where {T} + r(x) = rand(rng, x) + r.(iterator[].iterators) end \ No newline at end of file From 8d4054cfbc5049575831d50b18ee7dacdaf81af4 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 30 Nov 2021 18:53:46 +0100 Subject: [PATCH 10/29] Terminal UI to debug combinations --- Project.toml | 8 +- src/Actions.jl | 301 ++++++++++++++++++++++++++++++++---- test/basic3.jl | 39 ++++- test/staterequest static.jl | 13 +- 4 files changed, 319 insertions(+), 42 deletions(-) diff --git a/Project.toml b/Project.toml index 5c9e5fe..bec06d7 100644 --- a/Project.toml +++ b/Project.toml @@ -8,16 +8,20 @@ Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" BetterFileWatching = "c9fd44ac-77b5-486c-9482-9798bd063cc6" Configurations = "5218b696-f38b-4ac9-8b61-a12ec717816d" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" FromFile = "ff7dd447-1dcb-4ce3-b8ac-22a812192de7" Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" GitHubActions = "6b79fd1a-b13a-48ab-b6b0-aaee1fee41df" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Pluto = "c3e4b0f8-55cb-11ea-2926-15256bba5781" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" TerminalLoggers = "5d786b92-1e48-4d6f-9151-6b4477ca9bed" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" @@ -26,6 +30,7 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" BetterFileWatching = "^0.1.2" Configurations = "0.16" FromFile = "0.1" +Distributions = "0.25" Git = "1" GitHubActions = "0.1" HTTP = "^0.9.3" @@ -36,8 +41,7 @@ julia = "1.6" [extras] Deno_jll = "04572ae6-984a-583e-9378-9577a1c2574d" -Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Deno_jll", "Random", "Test"] +test = ["Deno_jll", "Test"] diff --git a/src/Actions.jl b/src/Actions.jl index 1a92e84..9a5bfa2 100644 --- a/src/Actions.jl +++ b/src/Actions.jl @@ -5,6 +5,10 @@ using SHA using OrderedCollections using FromFile import HTTP.URIs +import Random +import Statistics +using Distributions +import Markdown @from "./MoreAnalysis.jl" import bound_variable_connections_graph @from "./Export.jl" import try_get_exact_pluto_version, try_fromcache, try_tocache @@ -377,42 +381,47 @@ function run_bonds_get_patch_info( ) end -function generate_static_staterequests( - notebook_session::NotebookSession; - settings::PlutoDeploySettings, - pluto_session::Pluto.ServerSession, - output_dir=".", -) - sesh = notebook_session - run = sesh.run - connections = run.bond_connections - current_hash = sesh.current_hash +const Reason = Symbol - @assert run isa RunningNotebook +Base.@kwdef struct VariableGroupPossibilities + names::Vector{Symbol} + file_size_sample_statistics::Union{Nothing,Distribution} = nothing + num_possibilities::Int64 + possible_values::Vector{Any} + not_available::Dict{Symbol,Reason} +end - mkpath(joinpath(output_dir, "bondconnections")) +struct PrecomputedSampleReport + groups::Vector{VariableGroupPossibilities} +end - mkpath(joinpath(output_dir, "staterequest", URIs.escapeuri(current_hash))) +# TODO +# - if one of the fields in the group has no possible values then we cant precompute anything, that should be displayed, but its actually cool that it works - write_path = joinpath(output_dir, "bondconnections", URIs.escapeuri(current_hash)) - write(write_path, Pluto.pack(run.bond_connections)) - @info "Written bond connections to " write_path - for variable_group in Set(values(connections)) +function variable_groups( + connections; + pluto_session::Pluto.ServerSession, + notebook::Pluto.Notebook, +) + map(collect(Set(values(connections)))) do variable_group names = sort(variable_group) + not_available = Dict{Symbol,Reason}() + possible_values = [ let result = Pluto.possible_bond_values( pluto_session::Pluto.ServerSession, - run.notebook::Pluto.Notebook, + notebook::Pluto.Notebook, n::Symbol, ) if result isa Symbol - @error "Failed to get possible values for $(n)" result + # @error "Failed to get possible values for $(n)" result + not_available[n] = result [] else result @@ -420,33 +429,256 @@ function generate_static_staterequests( end for n in names ] - for combination in Iterators.product(possible_values...) - bonds = OrderedDict{Symbol,Any}( - n => OrderedDict{String,Any}("value" => v, "is_first_value" => true) for - (n, v) in zip(names, combination) - ) + VariableGroupPossibilities( + names=names, + # file_size_sample_statistics=( + # Statistics.mean(file_size_sample), + # Statistics.stddev(file_size_sample), + # ), + num_possibilities=prod(Int64.(length.(possible_values))), + possible_values=possible_values, + not_available=not_available, + ) + end +end + + +function combination_iterator(group::VariableGroupPossibilities) + Iterators.map(Iterators.product(group.possible_values...)) do combination + bonds_dict = OrderedDict{Symbol,Any}( + n => OrderedDict{String,Any}("value" => v, "is_first_value" => true) for + (n, v) in zip(group.names, combination) + ) + + return (combination, bonds_dict) + end +end + +format_filesize(x::Real) = isnan(x) ? "NaN" : try + Base.format_bytes(floor(Int64, x)) +catch + "$(x / 1e6) MB" +end + +function format_filesize(x::Distribution) + m, s = mean(x), std(x) + if s / m > 0.05 + format_filesize(m) * " ± " * format_filesize(s) + else + format_filesize(m) + end +end + +sum_distributions(ds; init=Normal(0, 0)) = reduce(convolve, ds; init=init) + +function Base.show(io::IO, m::MIME"text/plain", p::PrecomputedSampleReport) + groups = sort(p.groups; by=g -> mean(g.file_size_sample_statistics), rev=true) + + r = Markdown.parse( + """ +# Summary of precomputed + +Total size estimate: $( + sum(Int64[g.num_possibilities for g in groups]) +) files, $(map(groups) do group + group.file_size_sample_statistics +end |> sum_distributions |> format_filesize) + +$(map(groups) do group +total_size_dist = group.file_size_sample_statistics + +""" +## Group: $(join(["`$(n)`" for n in group.names], ", ")) + +$( + if isempty(group.not_available) + """ + Size estimate for this group: $( + group.num_possibilities + ) files, $( + format_filesize(total_size_dist) + ) + + | Name | Possible values | File size per value | + |---|---|---| + $(map(zip(group.names, group.possible_values)) do (n, vs) + "| `$(n)` | $(length(vs)) | $(format_filesize(total_size_dist / length(vs))) | \n" + end |> join) + """ + else + notgivens = [k for (k,v) in group.not_available if v == :NotGiven] + infitesss = [k for (k,v) in group.not_available if v == :InfinitePossibilities] + + """ + This group could not be precomputed because: + $( + isempty(notgivens) ? "" : "- The set of possible values for $(join(("`$(s)`" for s in notgivens), ", ")) is not known. If you are using PlutoUI, be sure to use an up-to-date version. If this input element is custom-made, take a look at `AbstractPlutoDingetjes.jl`. \n" + )$( + isempty(infitesss) ? "" : "- The set of possible values for $(join(("`$(s)`" for s in infitesss), ", ")) is infinite. \n" + ) + """ + end +) + +""" +end |> join) +""", + ) + show(io, m, r) +end + +# function static_staterequest_report(; connections) + +# groups = variable_groups(connections; pluto_session, notebook=notebook_session.run.notebook) + +# map(Set(values(connections))) do variable_group + +# names = sort(variable_group) + +# not_available = Dict{Symbol,Reason}() + +# possible_values = [ +# let +# result = Pluto.possible_bond_values( +# pluto_session::Pluto.ServerSession, +# run.notebook::Pluto.Notebook, +# n::Symbol, +# ) +# if result isa Symbol +# # @error "Failed to get possible values for $(n)" result +# not_available[n] = result +# [] +# else +# result +# end +# end for n in names +# ] + + +# file_size_sample = map( +# rand(Iterators.product(possible_values...), length(names) * 3), +# ) do combination +# bonds = OrderedDict{Symbol,Any}( +# n => OrderedDict{String,Any}("value" => v, "is_first_value" => true) +# for (n, v) in zip(names, combination) +# ) + +# result = run_bonds_get_patch_info(pluto_session, sesh, bonds) + +# if result !== nothing +# length(Pluto.pack(result)) +# else +# 0 +# end +# end + +# VariableGroupPossibilities( +# variables=variable_group, +# file_size_sample_statistics=( +# Statistics.mean(file_size_sample), +# Statistics.stddev(file_size_sample), +# ), +# num_possible_values=Dict{Symbol,Int64}(n => Int64(length(possible_values[n]))), +# not_available=not_available, +# ) +# end |> PrecomputedSampleReport +# end + + +function generate_static_staterequests_report( + groups::Vector{VariableGroupPossibilities}, + notebook_session::NotebookSession; + settings::PlutoDeploySettings, + pluto_session::Pluto.ServerSession, +) + sesh = notebook_session + run = sesh.run + + @assert run isa RunningNotebook + + map(groups) do group + stat = if !isempty(group.not_available) || isempty(combination_iterator(group)) + Normal(0, 0) + else + iterator = combination_iterator(group) + file_size_sample = + map(rand(iterator, length(group.names) * 3)) do (combination, bonds_dict) + + result = run_bonds_get_patch_info(pluto_session, sesh, bonds_dict) + + if result !== nothing + length(Pluto.pack(result)) + else + 0 + end + end .* length(iterator) # multiply by number of combinations to get an estimate of the total file size + + fit(Normal, file_size_sample) + end + VariableGroupPossibilities( + names=group.names, + possible_values=group.possible_values, + num_possibilities=group.num_possibilities, + not_available=group.not_available, + file_size_sample_statistics=stat, + ) + end |> PrecomputedSampleReport +end + + +function generate_static_staterequests( + notebook_session::NotebookSession; + settings::PlutoDeploySettings, + pluto_session::Pluto.ServerSession, + output_dir=".", +) + + sesh = notebook_session + run = sesh.run + connections = run.bond_connections + current_hash = sesh.current_hash + + @assert run isa RunningNotebook + + mkpath(joinpath(output_dir, "bondconnections")) + mkpath(joinpath(output_dir, "staterequest", URIs.escapeuri(current_hash))) + + bondconnections_path = + joinpath(output_dir, "bondconnections", URIs.escapeuri(current_hash)) + write(bondconnections_path, Pluto.pack(run.bond_connections)) + @info "Written bond connections to " bondconnections_path - result = run_bonds_get_patch_info(pluto_session, sesh, bonds) + groups = + variable_groups(connections; pluto_session, notebook=notebook_session.run.notebook) + + report = generate_static_staterequests_report(groups, sesh; settings, pluto_session) + + println(stderr) + println(stderr) + show(stderr, MIME"text/plain"(), report) + println(stderr) + println(stderr) + + foreach(groups) do group + foreach(combination_iterator(group)) do (combination, bonds_dict) + + result = run_bonds_get_patch_info(pluto_session, sesh, bonds_dict) if result !== nothing write_path = joinpath( output_dir, "staterequest", URIs.escapeuri(current_hash), - Pluto.pack(bonds) |> base64encode |> URIs.escapeuri, + Pluto.pack(bonds_dict) |> base64encode |> URIs.escapeuri, ) write(write_path, Pluto.pack(result)) @info "Written state request to " write_path values = - (; (zip(names, combination))...) - + (; (zip(group.names, combination))...) end end - end - - end function Random.rand( @@ -455,4 +687,11 @@ function Random.rand( ) where {T} r(x) = rand(rng, x) r.(iterator[].iterators) +end + +function Random.rand( + rng::Random.AbstractRNG, + iterator::Random.SamplerTrivial{Base.Generator{T,F}}, +) where {T,F} + iterator[].f(rand(rng, iterator[].iter)) end \ No newline at end of file diff --git a/test/basic3.jl b/test/basic3.jl index 29dfade..1621902 100644 --- a/test/basic3.jl +++ b/test/basic3.jl @@ -20,22 +20,43 @@ import AbstractPlutoDingetjes.Bonds # ╔═╡ 9b342ef4-fff9-46d0-b316-cc383fe71a59 begin struct CoolSlider + max end - function Base.show(io::IO, ::MIME"text/html", ::CoolSlider) - write(io, "") + function Base.show(io::IO, ::MIME"text/html", s::CoolSlider) + write(io, "") end Bonds.initial_value(::CoolSlider) = 1 - Bonds.possible_values(::CoolSlider) = 1:10 + Bonds.possible_values(s::CoolSlider) = 1:s.max +end + +# ╔═╡ 4a683caa-6910-4922-bda4-c3a00950a14b +begin + struct CoolText + end + function Base.show(io::IO, ::MIME"text/html", s::CoolText) + write(io, "") + end + Bonds.initial_value(::CoolText) = "" + Bonds.possible_values(s::CoolText) = Bonds.InfinitePossibilities() end # ╔═╡ 635e5ebc-6567-11eb-1d9d-f98bfca7ec27 -@bind x CoolSlider() +@bind x CoolSlider(10) # ╔═╡ ad853ac9-a8a0-44ef-8d41-c8cea165ad57 -@bind y CoolSlider() +@bind y CoolSlider(20) # ╔═╡ 26025270-9b5e-4841-b295-0c47437bc7db -x + y +repeat(string(x), y) + +# ╔═╡ 8fb4ff71-6f86-4643-a0af-6b6665d63634 + + +# ╔═╡ ea600f99-b634-492f-a1d6-7137ff896e84 +@bind z CoolSlider(15) + +# ╔═╡ 815f3a2d-b5f6-4b5d-84ec-fa660c3dbfe8 +z * 100 # ╔═╡ 1352da54-e567-4f59-a3da-19ed3f4bb7c7 @@ -53,7 +74,7 @@ s # ╔═╡ 8f0bd329-36b8-45ed-b80d-24661242129a -b2 = @bind s2 html"" +b2 = @bind s2 CoolText() # ╔═╡ c55a107f-5d7d-4396-b597-8c1ae07c35be b2 @@ -193,9 +214,13 @@ uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" # ╔═╡ Cell order: # ╠═22dc8ce8-392e-4202-a549-f0fd46152322 # ╠═9b342ef4-fff9-46d0-b316-cc383fe71a59 +# ╠═4a683caa-6910-4922-bda4-c3a00950a14b # ╠═635e5ebc-6567-11eb-1d9d-f98bfca7ec27 # ╠═ad853ac9-a8a0-44ef-8d41-c8cea165ad57 # ╠═26025270-9b5e-4841-b295-0c47437bc7db +# ╟─8fb4ff71-6f86-4643-a0af-6b6665d63634 +# ╠═ea600f99-b634-492f-a1d6-7137ff896e84 +# ╠═815f3a2d-b5f6-4b5d-84ec-fa660c3dbfe8 # ╟─1352da54-e567-4f59-a3da-19ed3f4bb7c7 # ╠═09ae27fa-525a-4211-b252-960cdbaf1c1e # ╠═ed78a6f1-d282-4d80-8f42-40701aeadb52 diff --git a/test/staterequest static.jl b/test/staterequest static.jl index 5538e45..3cc1952 100644 --- a/test/staterequest static.jl +++ b/test/staterequest static.jl @@ -49,7 +49,12 @@ using Base64 @test isdir(joinpath(test_dir, "bondconnections")) @show list_files_recursive(joinpath(test_dir, "staterequest")) @show readdir(joinpath(test_dir, "bondconnections")) - @test length(list_files_recursive(joinpath(test_dir, "staterequest"))) == 10 * 10 + @test length(list_files_recursive(joinpath(test_dir, "staterequest"))) == let + x = 10 + y = 20 + z = 15 + x * y + z + end while !occursin( "Pluto.jl notebook", @@ -129,7 +134,11 @@ using Base64 end @test state["cell_results"][sum_cell_id]["output"]["body"] == - string(bonds["x"]["value"] + bonds["y"]["value"]) + let + x = bonds["x"]["value"] + y = bonds["y"]["value"] + repeat(string(x), y) + end |> repr end end From 984a4d7175f69f06c70f60cf3ebc0787e13b6e7b Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Wed, 1 Dec 2021 01:44:32 +0100 Subject: [PATCH 11/29] Live debugging mode --- src/Actions.jl | 384 +-------------------------------------- src/HTTPRouter.jl | 4 +- src/PlutoSliderServer.jl | 2 + src/precomputed/debug.jl | 64 +++++++ src/precomputed/index.jl | 162 +++++++++++++++++ src/precomputed/types.jl | 133 ++++++++++++++ src/run_bonds.jl | 65 +++++++ test/basic3.jl | 14 +- 8 files changed, 438 insertions(+), 390 deletions(-) create mode 100644 src/precomputed/debug.jl create mode 100644 src/precomputed/index.jl create mode 100644 src/precomputed/types.jl create mode 100644 src/run_bonds.jl diff --git a/src/Actions.jl b/src/Actions.jl index 9a5bfa2..b191ecf 100644 --- a/src/Actions.jl +++ b/src/Actions.jl @@ -2,19 +2,16 @@ import Pluto: Pluto, without_pluto_file_extension, generate_html, @asynclog, withtoken, Firebasey using Base64 using SHA -using OrderedCollections using FromFile import HTTP.URIs -import Random -import Statistics -using Distributions -import Markdown @from "./MoreAnalysis.jl" import bound_variable_connections_graph @from "./Export.jl" import try_get_exact_pluto_version, try_fromcache, try_tocache @from "./Types.jl" import NotebookSession, RunningNotebook, FinishedNotebook @from "./Configuration.jl" import PlutoDeploySettings @from "./FileHelpers.jl" import find_notebook_files_recursive +@from "./precomputed/index.jl" import generate_precomputed_staterequests + myhash = base64encode ∘ sha256 function path_hash(path) myhash(read(path)) @@ -126,7 +123,7 @@ function process( run=run, ) if settings.Export.static_export_state - generate_static_staterequests( + generate_precomputed_staterequests( new_session; settings, pluto_session=server_session, @@ -320,378 +317,3 @@ function remove_static_export(path; settings, output_dir) end - -function run_bonds_get_patch_info( - server_session, - notebook_session::NotebookSession, - bonds::AbstractDict{Symbol,<:Any}, -)::Union{AbstractDict{String,Any},Nothing} - sesh = notebook_session - - notebook = sesh.run.notebook - - topological_order, new_state = withtoken(sesh.run.token) do - try - notebook.bonds = bonds - - names::Vector{Symbol} = Symbol.(keys(bonds)) - - topological_order = Pluto.set_bond_values_reactive( - session=server_session, - notebook=notebook, - bound_sym_names=names, - run_async=false, - )::Pluto.TopologicalOrder - - new_state = Pluto.notebook_to_js(notebook) - - topological_order, new_state - catch e - @error "Failed to set bond values" exception = (e, catch_backtrace()) - nothing, nothing - end - end - if topological_order === nothing - return nothing - end - - ids_of_cells_that_ran = [c.cell_id for c in topological_order.runnable] - - @debug "Finished running!" length(ids_of_cells_that_ran) - - # We only want to send state updates about... - function only_relevant(state) - new = copy(state) - # ... the cells that just ran and ... - new["cell_results"] = filter(state["cell_results"]) do (id, cell_state) - id ∈ ids_of_cells_that_ran - end - # ... nothing about bond values, because we don't want to synchronize among clients. - new["bonds"] = Dict{String,Dict{String,Any}}() - new - end - - patches = - Firebasey.diff(only_relevant(sesh.run.original_state), only_relevant(new_state)) - patches_as_dicts::Array{Dict} = patches - - Dict{String,Any}( - "patches" => patches_as_dicts, - "ids_of_cells_that_ran" => ids_of_cells_that_ran, - ) -end - -const Reason = Symbol - -Base.@kwdef struct VariableGroupPossibilities - names::Vector{Symbol} - file_size_sample_statistics::Union{Nothing,Distribution} = nothing - num_possibilities::Int64 - possible_values::Vector{Any} - not_available::Dict{Symbol,Reason} -end - -struct PrecomputedSampleReport - groups::Vector{VariableGroupPossibilities} -end - -# TODO -# - if one of the fields in the group has no possible values then we cant precompute anything, that should be displayed, but its actually cool that it works - - - - -function variable_groups( - connections; - pluto_session::Pluto.ServerSession, - notebook::Pluto.Notebook, -) - map(collect(Set(values(connections)))) do variable_group - - names = sort(variable_group) - - not_available = Dict{Symbol,Reason}() - - possible_values = [ - let - result = Pluto.possible_bond_values( - pluto_session::Pluto.ServerSession, - notebook::Pluto.Notebook, - n::Symbol, - ) - if result isa Symbol - # @error "Failed to get possible values for $(n)" result - not_available[n] = result - [] - else - result - end - end for n in names - ] - - VariableGroupPossibilities( - names=names, - # file_size_sample_statistics=( - # Statistics.mean(file_size_sample), - # Statistics.stddev(file_size_sample), - # ), - num_possibilities=prod(Int64.(length.(possible_values))), - possible_values=possible_values, - not_available=not_available, - ) - end -end - - -function combination_iterator(group::VariableGroupPossibilities) - Iterators.map(Iterators.product(group.possible_values...)) do combination - bonds_dict = OrderedDict{Symbol,Any}( - n => OrderedDict{String,Any}("value" => v, "is_first_value" => true) for - (n, v) in zip(group.names, combination) - ) - - return (combination, bonds_dict) - end -end - -format_filesize(x::Real) = isnan(x) ? "NaN" : try - Base.format_bytes(floor(Int64, x)) -catch - "$(x / 1e6) MB" -end - -function format_filesize(x::Distribution) - m, s = mean(x), std(x) - if s / m > 0.05 - format_filesize(m) * " ± " * format_filesize(s) - else - format_filesize(m) - end -end - -sum_distributions(ds; init=Normal(0, 0)) = reduce(convolve, ds; init=init) - -function Base.show(io::IO, m::MIME"text/plain", p::PrecomputedSampleReport) - groups = sort(p.groups; by=g -> mean(g.file_size_sample_statistics), rev=true) - - r = Markdown.parse( - """ -# Summary of precomputed - -Total size estimate: $( - sum(Int64[g.num_possibilities for g in groups]) -) files, $(map(groups) do group - group.file_size_sample_statistics -end |> sum_distributions |> format_filesize) - -$(map(groups) do group -total_size_dist = group.file_size_sample_statistics - -""" -## Group: $(join(["`$(n)`" for n in group.names], ", ")) - -$( - if isempty(group.not_available) - """ - Size estimate for this group: $( - group.num_possibilities - ) files, $( - format_filesize(total_size_dist) - ) - - | Name | Possible values | File size per value | - |---|---|---| - $(map(zip(group.names, group.possible_values)) do (n, vs) - "| `$(n)` | $(length(vs)) | $(format_filesize(total_size_dist / length(vs))) | \n" - end |> join) - """ - else - notgivens = [k for (k,v) in group.not_available if v == :NotGiven] - infitesss = [k for (k,v) in group.not_available if v == :InfinitePossibilities] - - """ - This group could not be precomputed because: - $( - isempty(notgivens) ? "" : "- The set of possible values for $(join(("`$(s)`" for s in notgivens), ", ")) is not known. If you are using PlutoUI, be sure to use an up-to-date version. If this input element is custom-made, take a look at `AbstractPlutoDingetjes.jl`. \n" - )$( - isempty(infitesss) ? "" : "- The set of possible values for $(join(("`$(s)`" for s in infitesss), ", ")) is infinite. \n" - ) - """ - end -) - -""" -end |> join) -""", - ) - show(io, m, r) -end - -# function static_staterequest_report(; connections) - -# groups = variable_groups(connections; pluto_session, notebook=notebook_session.run.notebook) - -# map(Set(values(connections))) do variable_group - -# names = sort(variable_group) - -# not_available = Dict{Symbol,Reason}() - -# possible_values = [ -# let -# result = Pluto.possible_bond_values( -# pluto_session::Pluto.ServerSession, -# run.notebook::Pluto.Notebook, -# n::Symbol, -# ) -# if result isa Symbol -# # @error "Failed to get possible values for $(n)" result -# not_available[n] = result -# [] -# else -# result -# end -# end for n in names -# ] - - -# file_size_sample = map( -# rand(Iterators.product(possible_values...), length(names) * 3), -# ) do combination -# bonds = OrderedDict{Symbol,Any}( -# n => OrderedDict{String,Any}("value" => v, "is_first_value" => true) -# for (n, v) in zip(names, combination) -# ) - -# result = run_bonds_get_patch_info(pluto_session, sesh, bonds) - -# if result !== nothing -# length(Pluto.pack(result)) -# else -# 0 -# end -# end - -# VariableGroupPossibilities( -# variables=variable_group, -# file_size_sample_statistics=( -# Statistics.mean(file_size_sample), -# Statistics.stddev(file_size_sample), -# ), -# num_possible_values=Dict{Symbol,Int64}(n => Int64(length(possible_values[n]))), -# not_available=not_available, -# ) -# end |> PrecomputedSampleReport -# end - - -function generate_static_staterequests_report( - groups::Vector{VariableGroupPossibilities}, - notebook_session::NotebookSession; - settings::PlutoDeploySettings, - pluto_session::Pluto.ServerSession, -) - sesh = notebook_session - run = sesh.run - - @assert run isa RunningNotebook - - map(groups) do group - stat = if !isempty(group.not_available) || isempty(combination_iterator(group)) - Normal(0, 0) - else - iterator = combination_iterator(group) - file_size_sample = - map(rand(iterator, length(group.names) * 3)) do (combination, bonds_dict) - - result = run_bonds_get_patch_info(pluto_session, sesh, bonds_dict) - - if result !== nothing - length(Pluto.pack(result)) - else - 0 - end - end .* length(iterator) # multiply by number of combinations to get an estimate of the total file size - - fit(Normal, file_size_sample) - end - VariableGroupPossibilities( - names=group.names, - possible_values=group.possible_values, - num_possibilities=group.num_possibilities, - not_available=group.not_available, - file_size_sample_statistics=stat, - ) - end |> PrecomputedSampleReport -end - - -function generate_static_staterequests( - notebook_session::NotebookSession; - settings::PlutoDeploySettings, - pluto_session::Pluto.ServerSession, - output_dir=".", -) - - sesh = notebook_session - run = sesh.run - connections = run.bond_connections - current_hash = sesh.current_hash - - @assert run isa RunningNotebook - - mkpath(joinpath(output_dir, "bondconnections")) - mkpath(joinpath(output_dir, "staterequest", URIs.escapeuri(current_hash))) - - bondconnections_path = - joinpath(output_dir, "bondconnections", URIs.escapeuri(current_hash)) - write(bondconnections_path, Pluto.pack(run.bond_connections)) - @info "Written bond connections to " bondconnections_path - - groups = - variable_groups(connections; pluto_session, notebook=notebook_session.run.notebook) - - report = generate_static_staterequests_report(groups, sesh; settings, pluto_session) - - println(stderr) - println(stderr) - show(stderr, MIME"text/plain"(), report) - println(stderr) - println(stderr) - - foreach(groups) do group - foreach(combination_iterator(group)) do (combination, bonds_dict) - - result = run_bonds_get_patch_info(pluto_session, sesh, bonds_dict) - - if result !== nothing - write_path = joinpath( - output_dir, - "staterequest", - URIs.escapeuri(current_hash), - Pluto.pack(bonds_dict) |> base64encode |> URIs.escapeuri, - ) - - write(write_path, Pluto.pack(result)) - - @info "Written state request to " write_path values = - (; (zip(group.names, combination))...) - end - end - end -end - -function Random.rand( - rng::Random.AbstractRNG, - iterator::Random.SamplerTrivial{Base.Iterators.ProductIterator{T}}, -) where {T} - r(x) = rand(rng, x) - r.(iterator[].iterators) -end - -function Random.rand( - rng::Random.AbstractRNG, - iterator::Random.SamplerTrivial{Base.Generator{T,F}}, -) where {T,F} - iterator[].f(rand(rng, iterator[].iter)) -end \ No newline at end of file diff --git a/src/HTTPRouter.jl b/src/HTTPRouter.jl index 1b0cdd5..97da3d2 100644 --- a/src/HTTPRouter.jl +++ b/src/HTTPRouter.jl @@ -16,7 +16,7 @@ using Sockets using Logging: global_logger using GitHubActions: GitHubActionsLogger -@from "./Actions.jl" import run_bonds_get_patch_info +@from "./run_bonds.jl" import run_bonds_get_patches @from "./Export.jl" import generate_index_html @from "./Types.jl" import NotebookSession, RunningNotebook @from "./Configuration.jl" import PlutoDeploySettings, get_configuration @@ -102,7 +102,7 @@ function make_router( end ## - result = run_bonds_get_patch_info(server_session, sesh, bonds) + result = run_bonds_get_patches(server_session, sesh.run, bonds) ## if result === nothing diff --git a/src/PlutoSliderServer.jl b/src/PlutoSliderServer.jl index f66e7e1..aff187a 100644 --- a/src/PlutoSliderServer.jl +++ b/src/PlutoSliderServer.jl @@ -16,6 +16,7 @@ using FromFile @from "./ReloadFolder.jl" import update_sessions!, select @from "./HTTPRouter.jl" import make_router @from "./gitpull.jl" import fetch_pull +@from "./precomputed/debug.jl" import start_debugging import Pluto import Pluto: @@ -494,4 +495,5 @@ function kind_of_debounced(f) end + end diff --git a/src/precomputed/debug.jl b/src/precomputed/debug.jl new file mode 100644 index 0000000..6e583e5 --- /dev/null +++ b/src/precomputed/debug.jl @@ -0,0 +1,64 @@ + +# this file is included in src/PlutoSliderServer.jl +# yolo + +import Pluto: Pluto, @asynclog, tamepath, is_pluto_notebook +import Configurations +using FromFile + +@from "./index.jl" import variable_groups, generate_precomputed_staterequests_report +@from "../Actions.jl" import path_hash +@from "../Types.jl" import RunningNotebook +@from "../Configuration.jl" import PlutoDeploySettings +@from "../MoreAnalysis.jl" import bound_variable_connections_graph + + +function start_debugging(notebook_path::String; kwargs...) + notebook_path = tamepath(notebook_path) + @assert is_pluto_notebook(notebook_path) + + settings = Configurations.from_kwargs(PlutoDeploySettings; kwargs...) + + + @info "Running notebook..." + + pluto_session = Pluto.ServerSession(; options=settings.Pluto) + notebook = Pluto.SessionActions.open(pluto_session, notebook_path; run_async=false) + + @info "Notebook ready! Starting server..." + + pluto_session.options.server.show_file_system = false + t = @asynclog Pluto.run(pluto_session) + sleep(1) + + repeat = true + while repeat + # current_hash = path_hash(notebook_path) + connections = bound_variable_connections_graph(notebook) + + run = RunningNotebook(; + path=notebook_path, + notebook=notebook, + bond_connections=connections, + original_state=Pluto.notebook_to_js(notebook), + ) + + groups = variable_groups(connections; pluto_session, notebook=run.notebook) + + report = + generate_precomputed_staterequests_report(groups, run; settings, pluto_session) + + for _ = 1:first(displaysize(stdout)) + println(stdout) + end + show(stdout, MIME"text/plain"(), report) + println(stdout) + println(stdout) + + + repeat = Base.prompt("Run again? (y/n)"; default="y") == "y" + end + + wait(t) + +end \ No newline at end of file diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl new file mode 100644 index 0000000..07a0e1a --- /dev/null +++ b/src/precomputed/index.jl @@ -0,0 +1,162 @@ +import Pluto +using Base64 +using OrderedCollections +using FromFile +import HTTP.URIs +import Random +import Statistics +using Distributions +import Markdown + +# @from "../MoreAnalysis.jl" import bound_variable_connections_graph +@from "../Types.jl" import NotebookSession, RunningNotebook +@from "../Configuration.jl" import PlutoDeploySettings +@from "../run_bonds.jl" import run_bonds_get_patches + +@from "./types.jl" import VariableGroupPossibilities, PrecomputedSampleReport, Reason + + +function variable_groups( + connections; + pluto_session::Pluto.ServerSession, + notebook::Pluto.Notebook, +) + map(collect(Set(values(connections)))) do variable_group + + names = sort(variable_group) + + not_available = Dict{Symbol,Reason}() + + possible_values = [ + let + result = Pluto.possible_bond_values( + pluto_session::Pluto.ServerSession, + notebook::Pluto.Notebook, + n::Symbol, + ) + if result isa Symbol + # @error "Failed to get possible values for $(n)" result + not_available[n] = result + [] + else + result + end + end for n in names + ] + + VariableGroupPossibilities(; + names=names, + num_possibilities=prod(Int64.(length.(possible_values))), + possible_values=possible_values, + not_available=not_available, + ) + end +end + + +function combination_iterator(group::VariableGroupPossibilities) + Iterators.map(Iterators.product(group.possible_values...)) do combination + bonds_dict = OrderedDict{Symbol,Any}( + n => OrderedDict{String,Any}("value" => v, "is_first_value" => true) for + (n, v) in zip(group.names, combination) + ) + + return (combination, bonds_dict) + end +end + +function generate_precomputed_staterequests_report( + groups::Vector{VariableGroupPossibilities}, + run::RunningNotebook; + settings::PlutoDeploySettings, + pluto_session::Pluto.ServerSession, +) + map(groups) do group + stat = if !isempty(group.not_available) + Normal(0, 0) + else + iterator = combination_iterator(group) + if isempty(iterator) + Normal(0, 0) + else + file_size_sample = + map( + rand(iterator, length(group.names) * 3), + ) do (combination, bonds_dict) + + result = run_bonds_get_patches(pluto_session, run, bonds_dict) + + if result !== nothing + length(Pluto.pack(result)) + else + 0 + end + end .* length(iterator) # multiply by number of combinations to get an estimate of the total file size + + fit(Normal, file_size_sample) + end + end + VariableGroupPossibilities(; + names=group.names, + possible_values=group.possible_values, + num_possibilities=group.num_possibilities, + not_available=group.not_available, + file_size_sample_distribution=stat, + ) + end |> PrecomputedSampleReport +end + + +function generate_precomputed_staterequests( + notebook_session::NotebookSession; + settings::PlutoDeploySettings, + pluto_session::Pluto.ServerSession, + output_dir=".", +) + + sesh = notebook_session + run = sesh.run + connections = run.bond_connections + current_hash = sesh.current_hash + + @assert run isa RunningNotebook + + mkpath(joinpath(output_dir, "bondconnections")) + mkpath(joinpath(output_dir, "staterequest", URIs.escapeuri(current_hash))) + + bondconnections_path = + joinpath(output_dir, "bondconnections", URIs.escapeuri(current_hash)) + write(bondconnections_path, Pluto.pack(run.bond_connections)) + @info "Written bond connections to " bondconnections_path + + groups = variable_groups(connections; pluto_session, notebook=run.notebook) + + report = generate_precomputed_staterequests_report(groups, run; settings, pluto_session) + + println(stderr) + println(stderr) + show(stderr, MIME"text/plain"(), report) + println(stderr) + println(stderr) + + foreach(groups) do group + foreach(combination_iterator(group)) do (combination, bonds_dict) + + result = run_bonds_get_patches(pluto_session, run, bonds_dict) + + if result !== nothing + write_path = joinpath( + output_dir, + "staterequest", + URIs.escapeuri(current_hash), + Pluto.pack(bonds_dict) |> base64encode |> URIs.escapeuri, + ) + + write(write_path, Pluto.pack(result)) + + @info "Written state request to " write_path values = + (; (zip(group.names, combination))...) + end + end + end +end diff --git a/src/precomputed/types.jl b/src/precomputed/types.jl new file mode 100644 index 0000000..e9a1a91 --- /dev/null +++ b/src/precomputed/types.jl @@ -0,0 +1,133 @@ + +using FromFile +import Random +import Statistics +using Distributions +import Markdown + + +const Reason = Symbol + +Base.@kwdef struct VariableGroupPossibilities + names::Vector{Symbol} + possible_values::Vector{Any} + not_available::Dict{Symbol,Reason} + # size info: + file_size_sample_distribution::Union{Nothing,Distribution} = nothing + num_possibilities::Int64 +end + +Base.@kwdef struct PrecomputedSampleReport + groups::Vector{VariableGroupPossibilities} + # size info: + file_size_sample_distribution::Union{Nothing,Distribution} = nothing + num_possibilities::Int64 +end + +function PrecomputedSampleReport(groups::Vector{VariableGroupPossibilities}) + num_possibilities = sum(Int64[g.num_possibilities for g in groups]) + + file_size_sample_distribution = + map(groups) do group + group.file_size_sample_distribution + end |> sum_distributions + + PrecomputedSampleReport(; groups, num_possibilities, file_size_sample_distribution) +end + +function Base.show(io::IO, m::MIME"text/plain", p::PrecomputedSampleReport) + groups = sort(p.groups; by=g -> mean(g.file_size_sample_distribution), rev=true) + + r = Markdown.parse( + """ +# Precomputed state summary + +Total size estimate: $(p.num_possibilities) files, $( + p.file_size_sample_distribution |> format_filesize +) + +$(map(groups) do group +total_size_dist = group.file_size_sample_distribution + +""" +## Group: $(join(["`$(n)`" for n in group.names], ", ")) + +$(if isempty(group.not_available) + """ + Size estimate for this group: $( + group.num_possibilities + ) files, $( + format_filesize(total_size_dist) + ) + + | Name | Possible values | File size per value | + |---|---|---| + $(map(zip(group.names, group.possible_values)) do (n, vs) + "| `$(n)` | **$(length(vs))** | $(format_filesize(total_size_dist / length(vs))) | \n" + end |> join) + """ +else + notgivens = [k for (k,v) in group.not_available if v == :NotGiven] + infinites = [k for (k,v) in group.not_available if v == :InfinitePossibilities] + remainder = setdiff(keys(group.not_available), notgivens ∪ infinites) + + """ + This group could not be precomputed because: + $( + isempty(notgivens) ? "" : "- The set of possible values for $(join(("`$(s)`" for s in notgivens), ", ")) is not known. If you are using PlutoUI, be sure to use an up-to-date version. If this input element is custom-made, take a look at `AbstractPlutoDingetjes.jl`. \n" + )$( + isempty(infinites) ? "" : "- The set of possible values for $(join(("`$(s)`" for s in infinites), ", ")) is infinite. \n" + )$( + isempty(remainder) ? "" : "- The set of possible values for $(join(("`$(s)`" for s in remainder), ", ")) could not be determined because of an unknown reason: $( + join((group.not_available[k] for k in remainder), ", ") + ). \n" + ) + """ +end) + +""" +end |> join) +""", + ) + show(io, m, r) +end + + +format_filesize(x::Real) = isnan(x) ? "NaN" : try + Base.format_bytes(floor(Int64, x)) +catch + "$(x / 1e6) MB" +end + +function format_filesize(x::Distribution) + m, s = mean(x), std(x) + if s / m > 0.05 + format_filesize(m) * " ± " * format_filesize(s) + else + format_filesize(m) + end +end + + + + +# Some missing functionality + + +function Random.rand( + rng::Random.AbstractRNG, + iterator::Random.SamplerTrivial{Base.Iterators.ProductIterator{T}}, +) where {T} + r(x) = rand(rng, x) + r.(iterator[].iterators) +end + +function Random.rand( + rng::Random.AbstractRNG, + iterator::Random.SamplerTrivial{Base.Generator{T,F}}, +) where {T,F} + iterator[].f(rand(rng, iterator[].iter)) +end + +sum_distributions(ds; init=Normal(0, 0)) = + any(isnothing, ds) ? nothing : reduce(convolve, ds; init=init) diff --git a/src/run_bonds.jl b/src/run_bonds.jl new file mode 100644 index 0000000..47e8c47 --- /dev/null +++ b/src/run_bonds.jl @@ -0,0 +1,65 @@ +import Pluto: + Pluto, without_pluto_file_extension, generate_html, @asynclog, withtoken, Firebasey +using Base64 +using SHA +using FromFile + +@from "./Types.jl" import RunningNotebook + +function run_bonds_get_patches( + server_session::Pluto.ServerSession, + run::RunningNotebook, + bonds::AbstractDict{Symbol,<:Any}, +)::Union{AbstractDict{String,Any},Nothing} + + notebook = run.notebook + + topological_order, new_state = withtoken(run.token) do + try + notebook.bonds = bonds + + names::Vector{Symbol} = Symbol.(keys(bonds)) + + topological_order = Pluto.set_bond_values_reactive( + session=server_session, + notebook=notebook, + bound_sym_names=names, + run_async=false, + )::Pluto.TopologicalOrder + + new_state = Pluto.notebook_to_js(notebook) + + topological_order, new_state + catch e + @error "Failed to set bond values" exception = (e, catch_backtrace()) + nothing, nothing + end + end + if topological_order === nothing + return nothing + end + + ids_of_cells_that_ran = [c.cell_id for c in topological_order.runnable] + + @debug "Finished running!" length(ids_of_cells_that_ran) + + # We only want to send state updates about... + function only_relevant(state) + new = copy(state) + # ... the cells that just ran and ... + new["cell_results"] = filter(state["cell_results"]) do (id, cell_state) + id ∈ ids_of_cells_that_ran + end + # ... nothing about bond values, because we don't want to synchronize among clients. + new["bonds"] = Dict{String,Dict{String,Any}}() + new + end + + patches = Firebasey.diff(only_relevant(run.original_state), only_relevant(new_state)) + patches_as_dicts::Array{Dict} = patches + + Dict{String,Any}( + "patches" => patches_as_dicts, + "ids_of_cells_that_ran" => ids_of_cells_that_ran, + ) +end diff --git a/test/basic3.jl b/test/basic3.jl index 1621902..223cfd5 100644 --- a/test/basic3.jl +++ b/test/basic3.jl @@ -41,13 +41,10 @@ begin end # ╔═╡ 635e5ebc-6567-11eb-1d9d-f98bfca7ec27 -@bind x CoolSlider(10) +@bind x CoolSlider(100) # ╔═╡ ad853ac9-a8a0-44ef-8d41-c8cea165ad57 -@bind y CoolSlider(20) - -# ╔═╡ 26025270-9b5e-4841-b295-0c47437bc7db -repeat(string(x), y) +@bind y CoolSlider(200) # ╔═╡ 8fb4ff71-6f86-4643-a0af-6b6665d63634 @@ -55,6 +52,9 @@ repeat(string(x), y) # ╔═╡ ea600f99-b634-492f-a1d6-7137ff896e84 @bind z CoolSlider(15) +# ╔═╡ 26025270-9b5e-4841-b295-0c47437bc7db +repeat(string(x), y + z) + # ╔═╡ 815f3a2d-b5f6-4b5d-84ec-fa660c3dbfe8 z * 100 @@ -213,8 +213,8 @@ uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" # ╔═╡ Cell order: # ╠═22dc8ce8-392e-4202-a549-f0fd46152322 -# ╠═9b342ef4-fff9-46d0-b316-cc383fe71a59 -# ╠═4a683caa-6910-4922-bda4-c3a00950a14b +# ╟─9b342ef4-fff9-46d0-b316-cc383fe71a59 +# ╟─4a683caa-6910-4922-bda4-c3a00950a14b # ╠═635e5ebc-6567-11eb-1d9d-f98bfca7ec27 # ╠═ad853ac9-a8a0-44ef-8d41-c8cea165ad57 # ╠═26025270-9b5e-4841-b295-0c47437bc7db From af8de518efa9f5da790d53873c468eb57be47a44 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 6 Dec 2021 17:12:59 +0100 Subject: [PATCH 12/29] fix tests --- test/basic3.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/basic3.jl b/test/basic3.jl index 223cfd5..9623d04 100644 --- a/test/basic3.jl +++ b/test/basic3.jl @@ -41,10 +41,10 @@ begin end # ╔═╡ 635e5ebc-6567-11eb-1d9d-f98bfca7ec27 -@bind x CoolSlider(100) +@bind x CoolSlider(10) # ╔═╡ ad853ac9-a8a0-44ef-8d41-c8cea165ad57 -@bind y CoolSlider(200) +@bind y CoolSlider(20) # ╔═╡ 8fb4ff71-6f86-4643-a0af-6b6665d63634 @@ -53,7 +53,7 @@ end @bind z CoolSlider(15) # ╔═╡ 26025270-9b5e-4841-b295-0c47437bc7db -repeat(string(x), y + z) +repeat(string(x), y) # ╔═╡ 815f3a2d-b5f6-4b5d-84ec-fa660c3dbfe8 z * 100 From 001d96db9979b1fafd905ea43b39f4265e967b56 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 6 Dec 2021 17:13:33 +0100 Subject: [PATCH 13/29] Judgement day --- src/Actions.jl | 4 +- src/Configuration.jl | 14 ++++++- src/precomputed/index.jl | 7 ++-- src/precomputed/types.jl | 84 ++++++++++++++++++++++++++++++++++--- test/staterequest static.jl | 14 ++++--- 5 files changed, 104 insertions(+), 19 deletions(-) diff --git a/src/Actions.jl b/src/Actions.jl index b191ecf..0d59999 100644 --- a/src/Actions.jl +++ b/src/Actions.jl @@ -70,7 +70,7 @@ function process( @warn "Notebook file does not have desired hash. This probably means that the file changed too quickly. Continuing and hoping for the best!" s.path new_hash s.desired_hash end - keep_running = settings.SliderServer.enabled || settings.Export.static_export_state + keep_running = settings.SliderServer.enabled || settings.Precompute.enabled skip_cache = keep_running || path ∈ settings.Export.ignore_cache cached_state = skip_cache ? nothing : try_fromcache(settings.Export.cache_dir, new_hash) @@ -122,7 +122,7 @@ function process( desired_hash=s.desired_hash, run=run, ) - if settings.Export.static_export_state + if settings.Precompute.enabled generate_precomputed_staterequests( new_session; settings, diff --git a/src/Configuration.jl b/src/Configuration.jl index 8011497..56692c6 100644 --- a/src/Configuration.jl +++ b/src/Configuration.jl @@ -1,7 +1,8 @@ using Configurations import TOML import Pluto -export SliderServerSettings, ExportSettings, PlutoDeploySettings, get_configuration +export SliderServerSettings, + ExportSettings, PrecomputeSettings, PlutoDeploySettings, get_configuration using TerminalLoggers: TerminalLogger using Logging: global_logger using FromFile @@ -23,10 +24,18 @@ using FromFile simulated_lag::Real = 0 end + +@extract_docs @option struct PrecomputeSettings + "Precompute slider server requests?" + enabled::Bool = false + "List of notebook files to skip precomputation. Provide paths relative to `start_dir`." + exclude::Vector{String} = String[] + max_filesize_per_group::Integer = 10_000_000 +end + @extract_docs @option struct ExportSettings "Generate static HTML files? This setting can only be `false` if you are also running a slider server." enabled::Bool = true - static_export_state::Bool = false "Folder to write generated HTML files to (will create directories to preserve the input folder structure). The behaviour of the default value depends on whether you are running the slider server, or just exporting. If running the slider server, we use a temporary directory; otherwise, we use `start_dir` (i.e. we generate each HTML file in the same folder as the notebook file)." output_dir::Union{Nothing,String} = nothing "List of notebook files to skip. Provide paths relative to `start_dir`." @@ -54,6 +63,7 @@ end @option struct PlutoDeploySettings SliderServer::SliderServerSettings = SliderServerSettings() Export::ExportSettings = ExportSettings() + Precompute::PrecomputeSettings = PrecomputeSettings() Pluto::Pluto.Configuration.Options = Pluto.Configuration.Options() end diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl index 07a0e1a..a784b78 100644 --- a/src/precomputed/index.jl +++ b/src/precomputed/index.jl @@ -46,9 +46,9 @@ function variable_groups( VariableGroupPossibilities(; names=names, - num_possibilities=prod(Int64.(length.(possible_values))), possible_values=possible_values, not_available=not_available, + num_possibilities=prod(Int64.(length.(possible_values))), ) end end @@ -102,6 +102,7 @@ function generate_precomputed_staterequests_report( num_possibilities=group.num_possibilities, not_available=group.not_available, file_size_sample_distribution=stat, + settings, ) end |> PrecomputedSampleReport end @@ -127,7 +128,7 @@ function generate_precomputed_staterequests( bondconnections_path = joinpath(output_dir, "bondconnections", URIs.escapeuri(current_hash)) write(bondconnections_path, Pluto.pack(run.bond_connections)) - @info "Written bond connections to " bondconnections_path + @debug "Written bond connections to " bondconnections_path groups = variable_groups(connections; pluto_session, notebook=run.notebook) @@ -154,7 +155,7 @@ function generate_precomputed_staterequests( write(write_path, Pluto.pack(result)) - @info "Written state request to " write_path values = + @debug "Written state request to " write_path values = (; (zip(group.names, combination))...) end end diff --git a/src/precomputed/types.jl b/src/precomputed/types.jl index e9a1a91..2eea541 100644 --- a/src/precomputed/types.jl +++ b/src/precomputed/types.jl @@ -5,16 +5,66 @@ import Statistics using Distributions import Markdown +@from "../Configuration.jl" import PlutoDeploySettings const Reason = Symbol -Base.@kwdef struct VariableGroupPossibilities +Base.@kwdef struct Judgement + should_precompute::Bool = false + not_available::Bool = false + close_to_filesize_limit::Bool = false + exceeds_filesize_limit::Bool = false +end + +struct VariableGroupPossibilities names::Vector{Symbol} - possible_values::Vector{Any} + possible_values::Vector not_available::Dict{Symbol,Reason} # size info: - file_size_sample_distribution::Union{Nothing,Distribution} = nothing + file_size_sample_distribution::Union{Nothing,Distribution} num_possibilities::Int64 + + judgement::Judgement +end + +function VariableGroupPossibilities(; + names::Vector{Symbol}, + possible_values::Vector, + not_available::Dict{Symbol,Reason}, + # size info: + num_possibilities::Int64, + file_size_sample_distribution::Union{Nothing,Distribution}=nothing, + settings::Union{Nothing,PlutoDeploySettings}=nothing, +) + is_not_available = !isempty(not_available) + + if !isa(file_size_sample_distribution, Nothing) + @assert settings isa PlutoDeploySettings + + limit = settings.Precompute.max_filesize_per_group + current = mean(file_size_sample_distribution) + + exceeds_filesize_limit = current > limit + close_to_filesize_limit = current > limit * 0.7 + else + exceeds_filesize_limit = close_to_filesize_limit = false + end + + j = Judgement(; + should_precompute=!is_not_available && !exceeds_filesize_limit, + exceeds_filesize_limit, + close_to_filesize_limit, + not_available=is_not_available, + ) + + VariableGroupPossibilities( + names, + possible_values, + not_available, + file_size_sample_distribution, + num_possibilities, + j, + ) end Base.@kwdef struct PrecomputedSampleReport @@ -22,6 +72,7 @@ Base.@kwdef struct PrecomputedSampleReport # size info: file_size_sample_distribution::Union{Nothing,Distribution} = nothing num_possibilities::Int64 + judgement::Judgement end function PrecomputedSampleReport(groups::Vector{VariableGroupPossibilities}) @@ -32,15 +83,28 @@ function PrecomputedSampleReport(groups::Vector{VariableGroupPossibilities}) group.file_size_sample_distribution end |> sum_distributions - PrecomputedSampleReport(; groups, num_possibilities, file_size_sample_distribution) + judgement = Judgement(; + should_precompute=any(g.judgement.should_precompute for g in groups), + not_available=any(g.judgement.not_available for g in groups), + exceeds_filesize_limit=any(g.judgement.exceeds_filesize_limit for g in groups), + close_to_filesize_limit=any(g.judgement.close_to_filesize_limit for g in groups), + ) + + PrecomputedSampleReport(; + groups, + num_possibilities, + file_size_sample_distribution, + judgement, + ) end + function Base.show(io::IO, m::MIME"text/plain", p::PrecomputedSampleReport) groups = sort(p.groups; by=g -> mean(g.file_size_sample_distribution), rev=true) r = Markdown.parse( """ -# Precomputed state summary +# $(pretty(p.judgement)) Precomputed state summary Total size estimate: $(p.num_possibilities) files, $( p.file_size_sample_distribution |> format_filesize @@ -50,7 +114,7 @@ $(map(groups) do group total_size_dist = group.file_size_sample_distribution """ -## Group: $(join(["`$(n)`" for n in group.names], ", ")) +## $(pretty(group.judgement)) Group: $(join(["`$(n)`" for n in group.names], ", ")) $(if isempty(group.not_available) """ @@ -131,3 +195,11 @@ end sum_distributions(ds; init=Normal(0, 0)) = any(isnothing, ds) ? nothing : reduce(convolve, ds; init=init) + + +pretty(j::Judgement) = + if j.should_precompute + j.close_to_filesize_limit ? "⚠️" : "✓" + else + "❌" + end diff --git a/test/staterequest static.jl b/test/staterequest static.jl index 3cc1952..227dc98 100644 --- a/test/staterequest static.jl +++ b/test/staterequest static.jl @@ -38,12 +38,14 @@ using Base64 end end - PlutoSliderServer.export_directory( - test_dir; - Export_static_export_state=true, - notebook_paths, - on_ready, - ) + withenv("JULIA_DEBUG" => nothing) do + PlutoSliderServer.export_directory( + test_dir; + Precompute_enabled=true, + notebook_paths, + on_ready, + ) + end @test isdir(joinpath(test_dir, "staterequest")) @test isdir(joinpath(test_dir, "bondconnections")) From ff138486aea0807c4097d75a3442c8517a1d1a52 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 6 Dec 2021 20:37:38 +0100 Subject: [PATCH 14/29] Match /~https://github.com/fonsp/Pluto.jl/pull/1703 --- src/precomputed/index.jl | 2 +- test/staterequest static.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl index a784b78..9eeddec 100644 --- a/src/precomputed/index.jl +++ b/src/precomputed/index.jl @@ -57,7 +57,7 @@ end function combination_iterator(group::VariableGroupPossibilities) Iterators.map(Iterators.product(group.possible_values...)) do combination bonds_dict = OrderedDict{Symbol,Any}( - n => OrderedDict{String,Any}("value" => v, "is_first_value" => true) for + n => OrderedDict{String,Any}("value" => v) for (n, v) in zip(group.names, combination) ) diff --git a/test/staterequest static.jl b/test/staterequest static.jl index 227dc98..a4581ff 100644 --- a/test/staterequest static.jl +++ b/test/staterequest static.jl @@ -100,7 +100,7 @@ using Base64 @testset "Method $(method)" for method in ["GET"], x = 3:7 - v(x) = OrderedDict("value" => x, "is_first_value" => true) + v(x) = OrderedDict("value" => x) bonds = OrderedDict("x" => v(x), "y" => v(7)) From f72fcaa46c7c15f1899d9e3d48a1300d3f5dd383 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 6 Dec 2021 20:37:51 +0100 Subject: [PATCH 15/29] whitespace --- src/precomputed/index.jl | 85 ++++++++++++++++++++++------------------ src/precomputed/types.jl | 20 ++++++---- 2 files changed, 59 insertions(+), 46 deletions(-) diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl index 9eeddec..6f8df62 100644 --- a/src/precomputed/index.jl +++ b/src/precomputed/index.jl @@ -20,37 +20,38 @@ function variable_groups( connections; pluto_session::Pluto.ServerSession, notebook::Pluto.Notebook, -) - map(collect(Set(values(connections)))) do variable_group - - names = sort(variable_group) - - not_available = Dict{Symbol,Reason}() - - possible_values = [ - let - result = Pluto.possible_bond_values( - pluto_session::Pluto.ServerSession, - notebook::Pluto.Notebook, - n::Symbol, - ) - if result isa Symbol - # @error "Failed to get possible values for $(n)" result - not_available[n] = result - [] - else - result - end - end for n in names - ] - - VariableGroupPossibilities(; - names=names, - possible_values=possible_values, - not_available=not_available, - num_possibilities=prod(Int64.(length.(possible_values))), - ) - end +)::Vector{VariableGroupPossibilities} + VariableGroupPossibilities[ + let + names = sort(variable_group) + + not_available = Dict{Symbol,Reason}() + + possible_values = [ + let + result = Pluto.possible_bond_values( + pluto_session::Pluto.ServerSession, + notebook::Pluto.Notebook, + n::Symbol, + ) + if result isa Symbol + # @error "Failed to get possible values for $(n)" result + not_available[n] = result + [] + else + result + end + end for n in names + ] + + VariableGroupPossibilities(; + names=names, + possible_values=possible_values, + not_available=not_available, + num_possibilities=prod(Int64.(length.(possible_values))), + ) + end for variable_group in Set(values(connections)) + ] end @@ -70,7 +71,7 @@ function generate_precomputed_staterequests_report( run::RunningNotebook; settings::PlutoDeploySettings, pluto_session::Pluto.ServerSession, -) +)::PrecomputedSampleReport map(groups) do group stat = if !isempty(group.not_available) Normal(0, 0) @@ -130,15 +131,21 @@ function generate_precomputed_staterequests( write(bondconnections_path, Pluto.pack(run.bond_connections)) @debug "Written bond connections to " bondconnections_path - groups = variable_groups(connections; pluto_session, notebook=run.notebook) + unanalyzed_groups = variable_groups(connections; pluto_session, notebook=run.notebook) - report = generate_precomputed_staterequests_report(groups, run; settings, pluto_session) + report = generate_precomputed_staterequests_report( + unanalyzed_groups, + run; + settings, + pluto_session, + ) + groups = report.groups - println(stderr) - println(stderr) - show(stderr, MIME"text/plain"(), report) - println(stderr) - println(stderr) + if report.judgement.should_precompute_all + @info "Notebook can be fully precomputed!" report + else + @warn "Notebook cannot be (fully) precomputed" report + end foreach(groups) do group foreach(combination_iterator(group)) do (combination, bonds_dict) diff --git a/src/precomputed/types.jl b/src/precomputed/types.jl index 2eea541..9596e3a 100644 --- a/src/precomputed/types.jl +++ b/src/precomputed/types.jl @@ -10,7 +10,7 @@ import Markdown const Reason = Symbol Base.@kwdef struct Judgement - should_precompute::Bool = false + should_precompute_all::Bool = false not_available::Bool = false close_to_filesize_limit::Bool = false exceeds_filesize_limit::Bool = false @@ -51,7 +51,7 @@ function VariableGroupPossibilities(; end j = Judgement(; - should_precompute=!is_not_available && !exceeds_filesize_limit, + should_precompute_all=!is_not_available && !exceeds_filesize_limit, exceeds_filesize_limit, close_to_filesize_limit, not_available=is_not_available, @@ -84,7 +84,7 @@ function PrecomputedSampleReport(groups::Vector{VariableGroupPossibilities}) end |> sum_distributions judgement = Judgement(; - should_precompute=any(g.judgement.should_precompute for g in groups), + should_precompute_all=all(g.judgement.should_precompute_all for g in groups), not_available=any(g.judgement.not_available for g in groups), exceeds_filesize_limit=any(g.judgement.exceeds_filesize_limit for g in groups), close_to_filesize_limit=any(g.judgement.close_to_filesize_limit for g in groups), @@ -98,17 +98,23 @@ function PrecomputedSampleReport(groups::Vector{VariableGroupPossibilities}) ) end +function exceeds_limit(j::Judgement, prefix::String="") + + j.exceeds_filesize_limit ? "*($(prefix)exceeding filesize limit)*" : + j.close_to_filesize_limit ? "*($(prefix)close to filesize limit)*" : "" +end + function Base.show(io::IO, m::MIME"text/plain", p::PrecomputedSampleReport) groups = sort(p.groups; by=g -> mean(g.file_size_sample_distribution), rev=true) r = Markdown.parse( """ -# $(pretty(p.judgement)) Precomputed state summary +# Precomputed state summary Total size estimate: $(p.num_possibilities) files, $( p.file_size_sample_distribution |> format_filesize -) +) $(exceeds_limit(p.judgement, "some groups are ")) $(map(groups) do group total_size_dist = group.file_size_sample_distribution @@ -122,7 +128,7 @@ $(if isempty(group.not_available) group.num_possibilities ) files, $( format_filesize(total_size_dist) - ) + ) $(exceeds_limit(group.judgement)) | Name | Possible values | File size per value | |---|---|---| @@ -198,7 +204,7 @@ sum_distributions(ds; init=Normal(0, 0)) = pretty(j::Judgement) = - if j.should_precompute + if j.should_precompute_all j.close_to_filesize_limit ? "⚠️" : "✓" else "❌" From 3a6122515c785aa0424211ba8e0c0a425121fbd3 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Wed, 8 Dec 2021 16:10:32 +0100 Subject: [PATCH 16/29] Update Project.toml --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index bec06d7..60c8e8c 100644 --- a/Project.toml +++ b/Project.toml @@ -35,7 +35,7 @@ Git = "1" GitHubActions = "0.1" HTTP = "^0.9.3" OrderedCollections = "1" -Pluto = "0.15, 0.16, 0.17" +Pluto = "0.17.3" TerminalLoggers = "0.1" julia = "1.6" From 207a21caccc965939dc9420b17678abe759ff2ca Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Wed, 8 Dec 2021 16:22:57 +0100 Subject: [PATCH 17/29] hopla --- src/Configuration.jl | 2 +- src/precomputed/index.jl | 32 +++++++++++++++++--------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/Configuration.jl b/src/Configuration.jl index 56692c6..dbea9df 100644 --- a/src/Configuration.jl +++ b/src/Configuration.jl @@ -30,7 +30,7 @@ end enabled::Bool = false "List of notebook files to skip precomputation. Provide paths relative to `start_dir`." exclude::Vector{String} = String[] - max_filesize_per_group::Integer = 10_000_000 + max_filesize_per_group::Integer = 1_000_000 end @extract_docs @option struct ExportSettings diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl index 6f8df62..ed5e5a2 100644 --- a/src/precomputed/index.jl +++ b/src/precomputed/index.jl @@ -147,23 +147,25 @@ function generate_precomputed_staterequests( @warn "Notebook cannot be (fully) precomputed" report end - foreach(groups) do group - foreach(combination_iterator(group)) do (combination, bonds_dict) - - result = run_bonds_get_patches(pluto_session, run, bonds_dict) - - if result !== nothing - write_path = joinpath( - output_dir, - "staterequest", - URIs.escapeuri(current_hash), - Pluto.pack(bonds_dict) |> base64encode |> URIs.escapeuri, - ) + foreach(groups) do group::VariableGroupPossibilities + if group.judgement.should_precompute_all + foreach(combination_iterator(group)) do (combination, bonds_dict) + + result = run_bonds_get_patches(pluto_session, run, bonds_dict) + + if result !== nothing + write_path = joinpath( + output_dir, + "staterequest", + URIs.escapeuri(current_hash), + Pluto.pack(bonds_dict) |> base64encode |> URIs.escapeuri, + ) - write(write_path, Pluto.pack(result)) + write(write_path, Pluto.pack(result)) - @debug "Written state request to " write_path values = - (; (zip(group.names, combination))...) + @debug "Written state request to " write_path values = + (; (zip(group.names, combination))...) + end end end end From 996b13a53469b941a92dd553a0ae041d21b14329 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Wed, 8 Dec 2021 16:32:28 +0100 Subject: [PATCH 18/29] Update Actions.jl --- src/Actions.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Actions.jl b/src/Actions.jl index 0d59999..21f65c6 100644 --- a/src/Actions.jl +++ b/src/Actions.jl @@ -233,7 +233,11 @@ function generate_static_export( slider_server_running_somewhere = settings.Export.slider_server_url !== nothing || - (settings.SliderServer.serve_static_export_folder && settings.SliderServer.enabled) + ( + settings.SliderServer.serve_static_export_folder && + settings.SliderServer.enabled + ) || + settings.Precompute.enabled notebookfile_js = if settings.Export.offer_binder || slider_server_running_somewhere if settings.Export.baked_notebookfile From e00d4d57727ce19963c2cf80e000491cbc59021c Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Wed, 8 Dec 2021 17:02:28 +0100 Subject: [PATCH 19/29] try catch --- src/precomputed/index.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl index ed5e5a2..aa78820 100644 --- a/src/precomputed/index.jl +++ b/src/precomputed/index.jl @@ -29,11 +29,15 @@ function variable_groups( possible_values = [ let - result = Pluto.possible_bond_values( - pluto_session::Pluto.ServerSession, - notebook::Pluto.Notebook, - n::Symbol, - ) + result = try + Pluto.possible_bond_values( + pluto_session::Pluto.ServerSession, + notebook::Pluto.Notebook, + n::Symbol, + ) + catch e + Symbol("Failed ", string(e)) + end if result isa Symbol # @error "Failed to get possible values for $(n)" result not_available[n] = result From 5b853e0dff7da7b437d842b7be4f1c93cea6d263 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Wed, 8 Dec 2021 17:20:17 +0100 Subject: [PATCH 20/29] bigint to avoid overflow --- src/precomputed/index.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl index aa78820..fc29802 100644 --- a/src/precomputed/index.jl +++ b/src/precomputed/index.jl @@ -70,6 +70,9 @@ function combination_iterator(group::VariableGroupPossibilities) end end +biglength(pr::Iterators.ProductIterator) = prod(BigInt[biglength(i) for i in pr.iterators]) +biglength(x) = BigInt(length(x)) + function generate_precomputed_staterequests_report( groups::Vector{VariableGroupPossibilities}, run::RunningNotebook; @@ -96,7 +99,7 @@ function generate_precomputed_staterequests_report( else 0 end - end .* length(iterator) # multiply by number of combinations to get an estimate of the total file size + end .* biglength(iterator) # multiply by number of combinations to get an estimate of the total file size fit(Normal, file_size_sample) end From 5e27a9ac6e49296e4042e519c659f7ed26f23f67 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Wed, 8 Dec 2021 17:25:15 +0100 Subject: [PATCH 21/29] more bigints --- src/precomputed/index.jl | 6 +++--- src/precomputed/types.jl | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl index fc29802..41150e4 100644 --- a/src/precomputed/index.jl +++ b/src/precomputed/index.jl @@ -52,7 +52,7 @@ function variable_groups( names=names, possible_values=possible_values, not_available=not_available, - num_possibilities=prod(Int64.(length.(possible_values))), + num_possibilities=prod(BigInt.(length.(possible_values))), ) end for variable_group in Set(values(connections)) ] @@ -81,11 +81,11 @@ function generate_precomputed_staterequests_report( )::PrecomputedSampleReport map(groups) do group stat = if !isempty(group.not_available) - Normal(0, 0) + Normal(0.0, 0.0) else iterator = combination_iterator(group) if isempty(iterator) - Normal(0, 0) + Normal(0.0, 0.0) else file_size_sample = map( diff --git a/src/precomputed/types.jl b/src/precomputed/types.jl index 9596e3a..f242c46 100644 --- a/src/precomputed/types.jl +++ b/src/precomputed/types.jl @@ -22,7 +22,7 @@ struct VariableGroupPossibilities not_available::Dict{Symbol,Reason} # size info: file_size_sample_distribution::Union{Nothing,Distribution} - num_possibilities::Int64 + num_possibilities::BigInt judgement::Judgement end @@ -32,7 +32,7 @@ function VariableGroupPossibilities(; possible_values::Vector, not_available::Dict{Symbol,Reason}, # size info: - num_possibilities::Int64, + num_possibilities::BigInt, file_size_sample_distribution::Union{Nothing,Distribution}=nothing, settings::Union{Nothing,PlutoDeploySettings}=nothing, ) @@ -71,12 +71,12 @@ Base.@kwdef struct PrecomputedSampleReport groups::Vector{VariableGroupPossibilities} # size info: file_size_sample_distribution::Union{Nothing,Distribution} = nothing - num_possibilities::Int64 + num_possibilities::BigInt judgement::Judgement end function PrecomputedSampleReport(groups::Vector{VariableGroupPossibilities}) - num_possibilities = sum(Int64[g.num_possibilities for g in groups]) + num_possibilities = sum(BigInt[g.num_possibilities for g in groups]) file_size_sample_distribution = map(groups) do group From d96ace48aa08301edac2a673869a172bf960ea59 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Wed, 8 Dec 2021 17:57:29 +0100 Subject: [PATCH 22/29] tweak --- src/precomputed/index.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl index 41150e4..eec4a8c 100644 --- a/src/precomputed/index.jl +++ b/src/precomputed/index.jl @@ -84,7 +84,7 @@ function generate_precomputed_staterequests_report( Normal(0.0, 0.0) else iterator = combination_iterator(group) - if isempty(iterator) + if isempty(iterator) || isempty(group.names) Normal(0.0, 0.0) else file_size_sample = From b534d58951d82af4dd9f9f1a3805790244c1ee61 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Wed, 8 Dec 2021 18:18:33 +0100 Subject: [PATCH 23/29] more tweakies --- src/precomputed/index.jl | 14 +++++++++----- src/precomputed/types.jl | 6 +++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl index eec4a8c..0bc6a59 100644 --- a/src/precomputed/index.jl +++ b/src/precomputed/index.jl @@ -54,7 +54,7 @@ function variable_groups( not_available=not_available, num_possibilities=prod(BigInt.(length.(possible_values))), ) - end for variable_group in Set(values(connections)) + end for variable_group in filter(!isempty, Set(values(connections))) ] end @@ -98,7 +98,7 @@ function generate_precomputed_staterequests_report( length(Pluto.pack(result)) else 0 - end + end |> BigInt end .* biglength(iterator) # multiply by number of combinations to get an estimate of the total file size fit(Normal, file_size_sample) @@ -156,8 +156,12 @@ function generate_precomputed_staterequests( foreach(groups) do group::VariableGroupPossibilities if group.judgement.should_precompute_all - foreach(combination_iterator(group)) do (combination, bonds_dict) - + for (combination, bonds_dict) in combination_iterator(group) + filename = Pluto.pack(bonds_dict) |> base64encode |> URIs.escapeuri + if length(filename) > 255 + @warn "Filename is too long, stopping this group" group.names + break + end result = run_bonds_get_patches(pluto_session, run, bonds_dict) if result !== nothing @@ -165,7 +169,7 @@ function generate_precomputed_staterequests( output_dir, "staterequest", URIs.escapeuri(current_hash), - Pluto.pack(bonds_dict) |> base64encode |> URIs.escapeuri, + filename, ) write(write_path, Pluto.pack(result)) diff --git a/src/precomputed/types.jl b/src/precomputed/types.jl index f242c46..dba2e3f 100644 --- a/src/precomputed/types.jl +++ b/src/precomputed/types.jl @@ -148,9 +148,9 @@ else )$( isempty(infinites) ? "" : "- The set of possible values for $(join(("`$(s)`" for s in infinites), ", ")) is infinite. \n" )$( - isempty(remainder) ? "" : "- The set of possible values for $(join(("`$(s)`" for s in remainder), ", ")) could not be determined because of an unknown reason: $( + isempty(remainder) ? "" : "- The set of possible values for $(join(("`$(s)`" for s in remainder), ", ")) could not be determined because of an unknown reason: `$( join((group.not_available[k] for k in remainder), ", ") - ). \n" + )`. \n" ) """ end) @@ -164,7 +164,7 @@ end format_filesize(x::Real) = isnan(x) ? "NaN" : try - Base.format_bytes(floor(Int64, x)) + Base.format_bytes(floor(BigInt, x)) catch "$(x / 1e6) MB" end From 2c9b88d224cb2a4bde432f4178eb85d12ed67601 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Wed, 8 Dec 2021 18:39:23 +0100 Subject: [PATCH 24/29] more bigint --- src/precomputed/index.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl index 0bc6a59..e24f82a 100644 --- a/src/precomputed/index.jl +++ b/src/precomputed/index.jl @@ -71,6 +71,7 @@ function combination_iterator(group::VariableGroupPossibilities) end biglength(pr::Iterators.ProductIterator) = prod(BigInt[biglength(i) for i in pr.iterators]) +biglength(g::Base.Generator) = biglength(g.iter) biglength(x) = BigInt(length(x)) function generate_precomputed_staterequests_report( From 1a960938d1ab7da7c0ba23552fc064cc93c63928 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 11 Apr 2022 21:47:53 +0200 Subject: [PATCH 25/29] merge part 2 --- src/run_bonds.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/run_bonds.jl b/src/run_bonds.jl index 47e8c47..8637df8 100644 --- a/src/run_bonds.jl +++ b/src/run_bonds.jl @@ -24,6 +24,7 @@ function run_bonds_get_patches( session=server_session, notebook=notebook, bound_sym_names=names, + is_first_values=[false for _n in names], # because requests should be stateless. We might want to do something special for the (actual) initial request (containing every initial bond value) in the future. run_async=false, )::Pluto.TopologicalOrder From c709acf6de9d7603c1b0806f712d6aad60134470 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Thu, 14 Apr 2022 15:04:23 +0200 Subject: [PATCH 26/29] oops! --- src/precomputed/index.jl | 3 ++- test/staterequest static.jl | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl index e24f82a..5d5e3c9 100644 --- a/src/precomputed/index.jl +++ b/src/precomputed/index.jl @@ -12,6 +12,7 @@ import Markdown @from "../Types.jl" import NotebookSession, RunningNotebook @from "../Configuration.jl" import PlutoDeploySettings @from "../run_bonds.jl" import run_bonds_get_patches +@from "../PlutoHash.jl" import base64urlencode @from "./types.jl" import VariableGroupPossibilities, PrecomputedSampleReport, Reason @@ -158,7 +159,7 @@ function generate_precomputed_staterequests( foreach(groups) do group::VariableGroupPossibilities if group.judgement.should_precompute_all for (combination, bonds_dict) in combination_iterator(group) - filename = Pluto.pack(bonds_dict) |> base64encode |> URIs.escapeuri + filename = Pluto.pack(bonds_dict) |> base64urlencode if length(filename) > 255 @warn "Filename is too long, stopping this group" group.names break diff --git a/test/staterequest static.jl b/test/staterequest static.jl index a4581ff..bc4adf2 100644 --- a/test/staterequest static.jl +++ b/test/staterequest static.jl @@ -109,16 +109,16 @@ using Base64 sum_cell_id = "26025270-9b5e-4841-b295-0c47437bc7db" response = if method == "GET" - arg = Pluto.pack(bonds) |> base64encode |> HTTP.URIs.escapeuri + arg = Pluto.pack(bonds) |> PlutoSliderServer.base64urlencode HTTP.request( method, - "http://localhost:$(port)/staterequest/$(HTTP.URIs.escapeuri(s.current_hash))/$(arg)", + "http://localhost:$(port)/staterequest/$(s.current_hash)/$(arg)", ) else HTTP.request( method, - "http://localhost:$(port)/staterequest/$(HTTP.URIs.escapeuri(s.current_hash))/", + "http://localhost:$(port)/staterequest/$(s.current_hash)/", [], Pluto.pack(bonds), ) From df2d1a9848427b8bfae0ae07a3fcfd2adcd5f2e7 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Wed, 20 Apr 2022 14:07:41 +0200 Subject: [PATCH 27/29] Easier possibilities report Co-Authored-By: Benedikt Ehinger Co-Authored-By: llips <38983684+llips@users.noreply.github.com> --- src/precomputed/types.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/precomputed/types.jl b/src/precomputed/types.jl index dba2e3f..05e6e39 100644 --- a/src/precomputed/types.jl +++ b/src/precomputed/types.jl @@ -112,7 +112,7 @@ function Base.show(io::IO, m::MIME"text/plain", p::PrecomputedSampleReport) """ # Precomputed state summary -Total size estimate: $(p.num_possibilities) files, $( +Total size estimate (based on a radom sample): $(p.num_possibilities) files, $( p.file_size_sample_distribution |> format_filesize ) $(exceeds_limit(p.judgement, "some groups are ")) @@ -124,16 +124,16 @@ total_size_dist = group.file_size_sample_distribution $(if isempty(group.not_available) """ - Size estimate for this group: $( + Size estimate for this group (based on a radom sample): $( group.num_possibilities ) files, $( format_filesize(total_size_dist) ) $(exceeds_limit(group.judgement)) - | Name | Possible values | File size per value | - |---|---|---| + | Name | Possible values | + |---|---| $(map(zip(group.names, group.possible_values)) do (n, vs) - "| `$(n)` | **$(length(vs))** | $(format_filesize(total_size_dist / length(vs))) | \n" + "| `$(n)` | **$(length(vs))** | \n" end |> join) """ else @@ -172,7 +172,7 @@ end function format_filesize(x::Distribution) m, s = mean(x), std(x) if s / m > 0.05 - format_filesize(m) * " ± " * format_filesize(s) + "$(format_filesize(m)) (𝜎 $(format_filesize(s)))" else format_filesize(m) end From d8f3a977cfd599de3a3a14d22b27ed287ce23df6 Mon Sep 17 00:00:00 2001 From: Connor Burns Date: Tue, 13 Jun 2023 11:10:18 -0600 Subject: [PATCH 28/29] Fix #117 by updating --- Project.toml | 3 +- src/HTTPRouter.jl | 34 +--------------------- src/precomputed/debug.jl | 2 -- src/run_bonds.jl | 2 +- test/basic3.jl | 56 ++++++++++++++++++++++++++++++++----- test/staterequest static.jl | 3 +- 6 files changed, 54 insertions(+), 46 deletions(-) diff --git a/Project.toml b/Project.toml index ca36483..83bf6af 100644 --- a/Project.toml +++ b/Project.toml @@ -48,8 +48,7 @@ julia = "1.6" [extras] Deno_jll = "04572ae6-984a-583e-9378-9577a1c2574d" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [targets] test = ["Deno_jll", "Test", "Random"] -Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/src/HTTPRouter.jl b/src/HTTPRouter.jl index 52c193c..71b1511 100644 --- a/src/HTTPRouter.jl +++ b/src/HTTPRouter.jl @@ -115,39 +115,7 @@ function make_router( with_not_cacheable! end - ids_of_cells_that_ran = [c.cell_id for c in topological_order.runnable] - - @debug "Finished running!" length(ids_of_cells_that_ran) - - # We only want to send state updates about... - function only_relevant(state) - new = copy(state) - # ... the cells that just ran and ... - new["cell_results"] = filter(state["cell_results"]) do (id, cell_state) - id ∈ ids_of_cells_that_ran - end - # ... nothing about bond values, because we don't want to synchronize among clients. and... - delete!(new, "bonds") - # ... we ignore changes to the status tree caused by a running bonds. - delete!(new, "status_tree") - new - end - - patches = Firebasey.diff( - only_relevant(sesh.run.original_state), - only_relevant(new_state), - ) - patches_as_dicts::Array{Dict} = Firebasey._convert(Array{Dict}, patches) - - HTTP.Response( - 200, - Pluto.pack( - Dict{String,Any}( - "patches" => patches_as_dicts, - "ids_of_cells_that_ran" => ids_of_cells_that_ran, - ), - ), - ) |> + HTTP.Response(200, Pluto.pack(result)) |> with_cacheable! |> with_cors! |> with_msgpack! diff --git a/src/precomputed/debug.jl b/src/precomputed/debug.jl index 6e583e5..a4b1ca4 100644 --- a/src/precomputed/debug.jl +++ b/src/precomputed/debug.jl @@ -7,7 +7,6 @@ import Configurations using FromFile @from "./index.jl" import variable_groups, generate_precomputed_staterequests_report -@from "../Actions.jl" import path_hash @from "../Types.jl" import RunningNotebook @from "../Configuration.jl" import PlutoDeploySettings @from "../MoreAnalysis.jl" import bound_variable_connections_graph @@ -33,7 +32,6 @@ function start_debugging(notebook_path::String; kwargs...) repeat = true while repeat - # current_hash = path_hash(notebook_path) connections = bound_variable_connections_graph(notebook) run = RunningNotebook(; diff --git a/src/run_bonds.jl b/src/run_bonds.jl index 8637df8..de5c937 100644 --- a/src/run_bonds.jl +++ b/src/run_bonds.jl @@ -57,7 +57,7 @@ function run_bonds_get_patches( end patches = Firebasey.diff(only_relevant(run.original_state), only_relevant(new_state)) - patches_as_dicts::Array{Dict} = patches + patches_as_dicts::Array{Dict} = Firebasey._convert(Array{Dict}, patches) Dict{String,Any}( "patches" => patches_as_dicts, diff --git a/test/basic3.jl b/test/basic3.jl index 9623d04..9bca2ce 100644 --- a/test/basic3.jl +++ b/test/basic3.jl @@ -1,5 +1,5 @@ ### A Pluto.jl notebook ### -# v0.17.2 +# v0.19.26 using Markdown using InteractiveUtils @@ -46,21 +46,39 @@ end # ╔═╡ ad853ac9-a8a0-44ef-8d41-c8cea165ad57 @bind y CoolSlider(20) +# ╔═╡ 26025270-9b5e-4841-b295-0c47437bc7db +repeat(string(x), y) + # ╔═╡ 8fb4ff71-6f86-4643-a0af-6b6665d63634 # ╔═╡ ea600f99-b634-492f-a1d6-7137ff896e84 @bind z CoolSlider(15) -# ╔═╡ 26025270-9b5e-4841-b295-0c47437bc7db -repeat(string(x), y) - # ╔═╡ 815f3a2d-b5f6-4b5d-84ec-fa660c3dbfe8 z * 100 # ╔═╡ 1352da54-e567-4f59-a3da-19ed3f4bb7c7 +# ╔═╡ dca76d5d-bb71-4a81-9f9e-19ddfbc258cd +@bind a1 CoolSlider(10) + +# ╔═╡ a3ab6582-3d30-4594-8eb6-17f3c4103076 +@bind a2 CoolSlider(10) + +# ╔═╡ 6ee0893c-45f3-490e-a53c-7b3d92bc8f70 +f = () -> a1 + a2 + +# ╔═╡ 7ad5c51b-76e4-44d3-aeca-f81a8f22f8b4 +f() + +# ╔═╡ 1681132d-050d-4ef7-9caa-14be6eade03d + + +# ╔═╡ 6ca3eca5-2a0b-4c9c-84c7-307fd4a29eae + + # ╔═╡ 09ae27fa-525a-4211-b252-960cdbaf1c1e b = @bind s html"" @@ -103,6 +121,7 @@ version = "1.1.2" [[ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" [[Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" @@ -115,8 +134,12 @@ deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" [[Downloads]] -deps = ["ArgTools", "LibCURL", "NetworkOptions"] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" [[InteractiveUtils]] deps = ["Markdown"] @@ -125,10 +148,12 @@ uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" [[LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" [[LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" [[LibGit2]] deps = ["Base64", "NetworkOptions", "Printf", "SHA"] @@ -137,6 +162,7 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" [[LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" [[Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -151,16 +177,20 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+0" [[MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.10.11" [[NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" [[Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.9.0" [[Printf]] deps = ["Unicode"] @@ -171,11 +201,12 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[Random]] -deps = ["Serialization"] +deps = ["SHA", "Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" @@ -186,10 +217,12 @@ uuid = "6462fe0b-24de-5631-8697-dd941f90decc" [[TOML]] deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" [[Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" [[UUIDs]] deps = ["Random", "SHA"] @@ -201,14 +234,17 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+0" [[nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" [[p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" """ # ╔═╡ Cell order: @@ -222,6 +258,12 @@ uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" # ╠═ea600f99-b634-492f-a1d6-7137ff896e84 # ╠═815f3a2d-b5f6-4b5d-84ec-fa660c3dbfe8 # ╟─1352da54-e567-4f59-a3da-19ed3f4bb7c7 +# ╠═dca76d5d-bb71-4a81-9f9e-19ddfbc258cd +# ╠═a3ab6582-3d30-4594-8eb6-17f3c4103076 +# ╠═6ee0893c-45f3-490e-a53c-7b3d92bc8f70 +# ╠═7ad5c51b-76e4-44d3-aeca-f81a8f22f8b4 +# ╠═1681132d-050d-4ef7-9caa-14be6eade03d +# ╟─6ca3eca5-2a0b-4c9c-84c7-307fd4a29eae # ╠═09ae27fa-525a-4211-b252-960cdbaf1c1e # ╠═ed78a6f1-d282-4d80-8f42-40701aeadb52 # ╠═cca2f726-0c25-43c6-85e4-c16ec192d464 diff --git a/test/staterequest static.jl b/test/staterequest static.jl index bc4adf2..4ca42ec 100644 --- a/test/staterequest static.jl +++ b/test/staterequest static.jl @@ -55,7 +55,8 @@ using Base64 x = 10 y = 20 z = 15 - x * y + z + a1, a2 = 10, 10 + x * y + z + a1 * a2 end while !occursin( From 83194ebe2b8cc66efedfe76669932db871a253b7 Mon Sep 17 00:00:00 2001 From: Connor Burns Date: Mon, 10 Jul 2023 23:34:35 +0000 Subject: [PATCH 29/29] Option to precompute only if all groups can be --- src/Configuration.jl | 2 ++ src/precomputed/index.jl | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/Configuration.jl b/src/Configuration.jl index e86adf3..6228cff 100644 --- a/src/Configuration.jl +++ b/src/Configuration.jl @@ -31,6 +31,8 @@ end enabled::Bool = false "List of notebook files to skip precomputation. Provide paths relative to `start_dir`." exclude::Vector{String} = String[] + "Whether or not to partially precompute notebooks. If `true`, notebooks will only be precomputed if **all** their sliders can be precomputed" + only_fully::Bool = false max_filesize_per_group::Integer = 1_000_000 end diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl index 5d5e3c9..4a64a8f 100644 --- a/src/precomputed/index.jl +++ b/src/precomputed/index.jl @@ -152,6 +152,9 @@ function generate_precomputed_staterequests( if report.judgement.should_precompute_all @info "Notebook can be fully precomputed!" report + elseif settings.Precompute.only_fully + @warn "Notebook cannot be fully precomputed - skipping it!" report + return else @warn "Notebook cannot be (fully) precomputed" report end