🧾 Options to store notebook in visual order or executable order

sample/Basic no cell order.jl

@@ -0,0 +1,31 @@
+### A Pluto.jl notebook ###
+# v0.20.14
+
+using Markdown
+using InteractiveUtils
+
+# ╔═╡ b2d786ec-7f73-11ea-1a0c-f38d7b6bbc1e
+md"""
+# The Basel problem
+
+_Leonard Euler_ proved in 1741 that the series
+
+```math
+\frac{1}{1} + \frac{1}{4} + \frac{1}{9} + \cdots
+```
+
+converges to
+
+```math
+\frac{\pi^2}{6}.
+```
+"""
+
+# ╔═╡ b2d792c2-7f73-11ea-0c65-a5042701e9f3
+sqrt(sum(seq) * 6.0)
+
+# ╔═╡ b2d79330-7f73-11ea-0d1c-a9aad1efaae1
+n = 1:100000
+
+# ╔═╡ b2d79376-7f73-11ea-2dce-cb9c449eece6
+seq = n .^ -2

sample/Basic.jl

@@ -1,5 +1,5 @@
 ### A Pluto.jl notebook ###
-# v0.14.0
+# v0.20.14
 
 using Markdown
 using InteractiveUtils
@@ -21,15 +21,15 @@ converges to
 ```
 """
 
+# ╔═╡ b2d792c2-7f73-11ea-0c65-a5042701e9f3
+sqrt(sum(seq) * 6.0)
+
 # ╔═╡ b2d79330-7f73-11ea-0d1c-a9aad1efaae1
 n = 1:100000
 
 # ╔═╡ b2d79376-7f73-11ea-2dce-cb9c449eece6
 seq = n .^ -2
 
-# ╔═╡ b2d792c2-7f73-11ea-0c65-a5042701e9f3
-sqrt(sum(seq) * 6.0)
-
 # ╔═╡ Cell order:
 # ╟─b2d786ec-7f73-11ea-1a0c-f38d7b6bbc1e
 # ╠═b2d792c2-7f73-11ea-0c65-a5042701e9f3

src/Configuration.jl

@@ -54,12 +54,16 @@ const DISABLE_WRITING_NOTEBOOK_FILES_DEFAULT = false
 const AUTO_RELOAD_FROM_FILE_DEFAULT = false
 const AUTO_RELOAD_FROM_FILE_COOLDOWN_DEFAULT = 0.4
 const AUTO_RELOAD_FROM_FILE_IGNORE_PKG_DEFAULT = false
+const STORE_IN_EXECUTABLE_ORDER_NEW_DEFAULT = true
+const STORE_IN_EXECUTABLE_ORDER_EXISTING_DEFAULT = nothing
 const NOTEBOOK_DEFAULT = nothing
 const SIMULATED_LAG_DEFAULT = 0.0
 const SIMULATED_PKG_LAG_DEFAULT = 0.0
 const INJECTED_JAVASCRIPT_DATA_URL_DEFAULT = "data:text/javascript;base64,"
 const ON_EVENT_DEFAULT = function(a) #= @info "$(typeof(a))" =# end
 
+const exec_order_doc = "should the notebook file store cells in executable order, so that the notebook file can run as a standalone Julia file? If false, the visual order will be used as the file order, and the `Cell order` section is ommited"
+
 """
     ServerOptions([; kwargs...])
 
@@ -80,7 +84,9 @@ The HTTP server options. See [`SecurityOptions`](@ref) for additional settings.
 - `auto_reload_from_file::Bool = $AUTO_RELOAD_FROM_FILE_DEFAULT` Watch notebook files for outside changes and update running notebook state automatically
 - `auto_reload_from_file_cooldown::Real = $AUTO_RELOAD_FROM_FILE_COOLDOWN_DEFAULT` Experimental, will be removed
 - `auto_reload_from_file_ignore_pkg::Bool = $AUTO_RELOAD_FROM_FILE_IGNORE_PKG_DEFAULT` Experimental flag, will be removed
-- `notebook::Union{Nothing,String} = $NOTEBOOK_DEFAULT` Optional path of notebook to launch at start
+- `store_in_executable_order_new::Bool = $STORE_IN_EXECUTABLE_ORDER_NEW_DEFAULT` For newly created files, $(exec_order_doc).
+- `store_in_executable_order_existing::Union{Nothing,Bool} = $STORE_IN_EXECUTABLE_ORDER_EXISTING_DEFAULT` After opening an existing notebook, $(exec_order_doc). If `nothing`, the notebook will be saved in the format it was opened in.
+- `notebook::Union{Nothing,String,Vector{<:String}} = $NOTEBOOK_DEFAULT` Optional path of notebook to launch at start
 - `simulated_lag::Real=$SIMULATED_LAG_DEFAULT` (internal) Extra lag to add to our server responses. Will be multiplied by `0.5 + rand()`.
 - `simulated_pkg_lag::Real=$SIMULATED_PKG_LAG_DEFAULT` (internal) Extra lag to add to operations done by Pluto's package manager. Will be multiplied by `0.5 + rand()`.
 - `injected_javascript_data_url::String = "$INJECTED_JAVASCRIPT_DATA_URL_DEFAULT"` (internal) Optional javascript injectables to the front-end. Can be used to customize the editor, but this API is not meant for general use yet.
@@ -104,6 +110,8 @@ The HTTP server options. See [`SecurityOptions`](@ref) for additional settings.
     auto_reload_from_file::Bool = AUTO_RELOAD_FROM_FILE_DEFAULT
     auto_reload_from_file_cooldown::Real = AUTO_RELOAD_FROM_FILE_COOLDOWN_DEFAULT
     auto_reload_from_file_ignore_pkg::Bool = AUTO_RELOAD_FROM_FILE_IGNORE_PKG_DEFAULT
+    store_in_executable_order_new::Bool = STORE_IN_EXECUTABLE_ORDER_NEW_DEFAULT
+    store_in_executable_order_existing::Union{Nothing,Bool} = STORE_IN_EXECUTABLE_ORDER_EXISTING_DEFAULT
     notebook::Union{Nothing,String,Vector{<:String}} = NOTEBOOK_DEFAULT
     simulated_lag::Real = SIMULATED_LAG_DEFAULT
     simulated_pkg_lag::Real = SIMULATED_PKG_LAG_DEFAULT
@@ -298,6 +306,8 @@ function from_flat_kwargs(;
         auto_reload_from_file::Bool = AUTO_RELOAD_FROM_FILE_DEFAULT,
         auto_reload_from_file_cooldown::Real = AUTO_RELOAD_FROM_FILE_COOLDOWN_DEFAULT,
         auto_reload_from_file_ignore_pkg::Bool = AUTO_RELOAD_FROM_FILE_IGNORE_PKG_DEFAULT,
+        store_in_executable_order_new::Bool = STORE_IN_EXECUTABLE_ORDER_NEW_DEFAULT,
+        store_in_executable_order_existing::Union{Nothing,Bool} = STORE_IN_EXECUTABLE_ORDER_EXISTING_DEFAULT,
         notebook::Union{Nothing,String,Vector{<:String}} = NOTEBOOK_DEFAULT,
         simulated_lag::Real = SIMULATED_LAG_DEFAULT,
         simulated_pkg_lag::Real = SIMULATED_PKG_LAG_DEFAULT,
@@ -348,6 +358,8 @@ function from_flat_kwargs(;
         auto_reload_from_file,
         auto_reload_from_file_cooldown,
         auto_reload_from_file_ignore_pkg,
+        store_in_executable_order_new,
+        store_in_executable_order_existing,
         notebook,
         simulated_lag,
         simulated_pkg_lag,

src/notebook/Notebook.jl

@@ -69,6 +69,7 @@ Base.@kwdef mutable struct Notebook
     bonds::Dict{Symbol,BondValue}=Dict{Symbol,BondValue}()
 
     metadata::Dict{String, Any}=copy(DEFAULT_NOTEBOOK_METADATA)
+    store_in_executable_order::Bool=true
 end
 
 function _initial_nb_status()

src/notebook/saving and loading.jl

@@ -33,6 +33,8 @@ Have a look at our [JuliaCon 2020 presentation](https://youtu.be/IAF8DjrQSSk?t=1
 function save_notebook(io::IO, notebook::Notebook)
     println(io, _notebook_header)
     println(io, "# ", PLUTO_VERSION_STR)
+
+    exe_order = notebook.store_in_executable_order
 
     # Notebook metadata
     let nb_metadata_toml = strip(sprint(TOML.print, get_metadata_no_default(notebook)))
@@ -57,14 +59,14 @@ function save_notebook(io::IO, notebook::Notebook)
     end
     println(io)
 
-    cells_ordered = collect(topological_order(notebook))
+    cells_ordered = exe_order ? collect(topological_order(notebook)) : notebook.cells
 
     # NOTE: the notebook topological is cached on every update_dependency! call
     # ....  so it is possible that a cell was added/removed since this last update.
     # ....  in this case, it will not contain that cell since it is build from its
     # ....  store notebook topology. therefore, we compute an updated topological
     # ....  order in this unlikely case.
-    if length(cells_ordered) != length(notebook.cells_dict)
+    if exe_order && length(cells_ordered) != length(notebook.cells_dict)
         cells = notebook.cells
         updated_topo = updated_topology(notebook.topology, notebook, cells)
         cells_ordered = collect(topological_order(updated_topo, cells))
@@ -122,15 +124,16 @@ function save_notebook(io::IO, notebook::Notebook)
         print(io, _cell_suffix)
     end
 
-
-    println(io, _cell_id_delimiter, "Cell order:")
-    for c in notebook.cells
-        delim = c.code_folded ? _order_delimiter_folded : _order_delimiter
-        println(io, delim, string(c.cell_id))
-    end
-    if write_package
-        println(io, _order_delimiter_folded, string(_ptoml_cell_id))
-        println(io, _order_delimiter_folded, string(_mtoml_cell_id))
+    if exe_order
+        println(io, _cell_id_delimiter, "Cell order:")
+        for c in notebook.cells
+            delim = c.code_folded ? _order_delimiter_folded : _order_delimiter
+            println(io, delim, string(c.cell_id))
+        end
+        if write_package
+            println(io, _order_delimiter_folded, string(_ptoml_cell_id))
+            println(io, _order_delimiter_folded, string(_mtoml_cell_id))
+        end
     end
 
     notebook
@@ -199,12 +202,23 @@ end
 
 function _read_notebook_collected_cells!(@nospecialize(io::IO))
     collected_cells = Dict{UUID,Cell}()
+    collected_cells_order = UUID[]
     while !eof(io)
         cell_id_str = String(readline(io))
         if cell_id_str == "Cell order:"
             break
         else
-            cell_id = UUID(cell_id_str)
+            cell_id_parsed = tryparse(UUID, cell_id_str)
+            cell_id = if cell_id_parsed isa UUID
+                if haskey(collected_cells, cell_id_parsed)
+                    @warn "Cell ID appears multiple times in the file. Generating a new one."
+                    uuid1()
+                else
+                    cell_id_parsed
+                end
+            else
+                uuid1()
+            end
 
             metadata_toml_lines = String[]
             initial_code_line = ""
@@ -239,9 +253,10 @@ function _read_notebook_collected_cells!(@nospecialize(io::IO))
 
             read_cell = Cell(; cell_id, code, metadata)
             collected_cells[cell_id] = read_cell
+            push!(collected_cells_order, cell_id)
         end
     end
-    return collected_cells
+    return collected_cells, collected_cells_order
 end
 
 function _read_notebook_cell_order!(@nospecialize(io::IO), collected_cells)
@@ -300,13 +315,13 @@ function _read_notebook_nbpkg_ctx(cell_order::Vector{UUID}, collected_cells::Dic
     return nbpkg_ctx
 end
 
-function _read_notebook_appeared_order!(cell_order::Vector{UUID}, collected_cells::Dict{Base.UUID, Cell})
+function _notebook_appeared_order(cell_order::Vector{UUID}, collected_cells_order::Vector{UUID})
     setdiff!(
         union!(
             # don't include cells that only appear in the order, but no code was given
-            intersect!(cell_order, keys(collected_cells)),
+            intersect(cell_order, collected_cells_order),
             # add cells that appeared in code, but not in the order.
-            keys(collected_cells)
+            collected_cells_order
         ),
         # remove Pkg cells
         (_ptoml_cell_id, _mtoml_cell_id)
@@ -316,15 +331,16 @@ end
 "Load a notebook without saving it or creating a backup; returns a `Notebook`. REMEMBER TO CHANGE THE NOTEBOOK PATH after loading it to prevent it from autosaving and overwriting the original file."
 function load_notebook_nobackup(@nospecialize(io::IO), @nospecialize(path::AbstractString); skip_nbpkg::Bool=false)::Notebook
     notebook_metadata = _read_notebook_metadata!(io)
-    collected_cells = _read_notebook_collected_cells!(io)
+    collected_cells, collected_cells_order = _read_notebook_collected_cells!(io)
     cell_order = _read_notebook_cell_order!(io, collected_cells)
     nbpkg_ctx = skip_nbpkg ? nothing : _read_notebook_nbpkg_ctx(cell_order, collected_cells)
-    appeared_order = _read_notebook_appeared_order!(cell_order, collected_cells)
 
+    appeared_order = _notebook_appeared_order(cell_order, collected_cells_order)
     appeared_cells_dict = filter(collected_cells) do (k, v)
         k ∈ appeared_order
     end
     topology = _initial_topology(appeared_cells_dict, appeared_order)
+    was_stored_in_executable_order = !isempty(cell_order)
 
     Notebook(;
         cells_dict=appeared_cells_dict,
@@ -335,6 +351,7 @@ function load_notebook_nobackup(@nospecialize(io::IO), @nospecialize(path::Abstr
         nbpkg_ctx,
         nbpkg_installed_versions_cache=nbpkg_cache(nbpkg_ctx),
         metadata=notebook_metadata,
+        store_in_executable_order=was_stored_in_executable_order,
     )
 end
 

src/webserver/SessionActions.jl

@@ -73,6 +73,7 @@ function open(session::ServerSession, path::AbstractString;
         notebook.metadata["risky_file_source"] = risky_file_source
     end
     notebook.process_status = execution_allowed ? ProcessStatus.starting : ProcessStatus.waiting_for_permission
+    notebook.store_in_executable_order = something(session.options.server.store_in_executable_order_existing, notebook.store_in_executable_order)
 
     # overwrites the notebook environment if specified
     if compiler_options !== nothing
@@ -222,6 +223,7 @@ function new(session::ServerSession; run_async=true, notebook_id::UUID=uuid1())
     # Run NewNotebookEvent handler before assigning ID
     isid = try_event_call(session, NewNotebookEvent())
     notebook.notebook_id = isnothing(isid) ? notebook_id : isid
+    notebook.store_in_executable_order = session.options.server.store_in_executable_order_new
 
     update_save_run!(session, notebook, notebook.cells; run_async, prerender_text=true)
     add(session, notebook; run_async)