Introduce Fly.io runtime (#2708)

.formatter.exs

@@ -1,5 +1,5 @@
 [
   import_deps: [:phoenix, :ecto],
   plugins: [Phoenix.LiveView.HTMLFormatter],
-  inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}"]
+  inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "rel/*/overlays/**/*.exs"]
 ]

.github/workflows/test.yml

@@ -10,6 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       MIX_ENV: test
+      ELIXIR_ERL_OPTIONS: "-epmd_module Elixir.Livebook.EPMD"
     steps:
       - name: Checkout git repo
         uses: actions/checkout@v3
@@ -59,41 +60,6 @@ jobs:
       - name: Run assets tests
         run: npm test --prefix assets
 
-  epmdless:
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push'
-    env:
-      MIX_ENV: test
-      LIVEBOOK_EPMDLESS: true
-      ELIXIR_ERL_OPTIONS: "-epmd_module Elixir.Livebook.EPMD -start_epmd false -erl_epmd_port 0"
-    steps:
-      - name: Checkout git repo
-        uses: actions/checkout@v3
-      - name: Read ./versions
-        run: |
-          . versions
-          echo "elixir=$elixir" >> $GITHUB_ENV
-          echo "otp=$otp" >> $GITHUB_ENV
-          echo "openssl=$openssl" >> $GITHUB_ENV
-      - name: Install Erlang & Elixir
-        uses: erlef/setup-beam@v1
-        with:
-          otp-version: ${{ env.otp }}
-          elixir-version: ${{ env.elixir }}
-      - name: Cache Mix
-        uses: actions/cache@v3
-        with:
-          path: |
-            deps
-            _build
-          key: ${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}-${{ hashFiles('**/mix.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}-
-      - name: Install mix dependencies
-        run: mix deps.get
-      - name: Run tests
-        run: mix test
-
   windows:
     runs-on: windows-latest
     if: github.event_name == 'push'

.gitignore

@@ -33,6 +33,3 @@ npm-debug.log
 
 # The built Escript
 /livebook
-
-# The priv directory with the EPMD file
-/priv/epmd

README.md

@@ -217,12 +217,9 @@ The following environment variables can be used to configure Livebook on boot:
 
   * `LIVEBOOK_DEFAULT_RUNTIME` - sets the runtime type that is used by default
     when none is started explicitly for the given notebook. Must be either
-    "standalone" (Elixir standalone), "attached:NODE:COOKIE" (Attached node)
+    "standalone" (Standalone), "attached:NODE:COOKIE" (Attached node)
     or "embedded" (Embedded). Defaults to "standalone".
 
-  * `LIVEBOOK_EPMDLESS` - if set to "true", it disables the usage of EPMD. This is
-    only supported within releases and defaults to true for the Desktop app.
-
   * `LIVEBOOK_FIPS` - if set to "true", it enables the FIPS mode on startup.
     See more details in [the documentation](https://hexdocs.pm/livebook/fips.html).
 

assets/js/events.js

@@ -61,13 +61,16 @@ export function registerGlobalEventHandlers() {
   });
 
   window.addEventListener("lb:scroll_into_view", (event) => {
+    const options = event.detail || {};
+
     // If the element is going to be shown, we want to wait for that
     waitUntilVisible(event.target).then(() => {
       scrollIntoView(event.target, {
         scrollMode: "if-needed",
         behavior: "smooth",
         block: "nearest",
         inline: "nearest",
+        ...options,
       });
     });
   });

config/config.exs

@@ -30,7 +30,6 @@ config :livebook,
   app_service_url: nil,
   authentication: :token,
   aws_credentials: false,
-  epmdless: false,
   feature_flags: [],
   force_ssl_host: nil,
   learn_notebooks: [],

lib/livebook.ex

@@ -149,22 +149,19 @@ defmodule Livebook do
       config :livebook, :aws_credentials, true
     end
 
-    if Livebook.Config.boolean!("LIVEBOOK_EPMDLESS", false) do
-      config :livebook, :epmdless, true
-    end
-
     config :livebook,
            :default_runtime,
            Livebook.Config.default_runtime!("LIVEBOOK_DEFAULT_RUNTIME") ||
-             Livebook.Runtime.ElixirStandalone.new()
+             Livebook.Runtime.Standalone.new()
 
-    config :livebook, :default_app_runtime, Livebook.Runtime.ElixirStandalone.new()
+    config :livebook, :default_app_runtime, Livebook.Runtime.Standalone.new()
 
     config :livebook,
            :runtime_modules,
            [
-             Livebook.Runtime.ElixirStandalone,
-             Livebook.Runtime.Attached
+             Livebook.Runtime.Standalone,
+             Livebook.Runtime.Attached,
+             Livebook.Runtime.Fly
            ]
 
     if home = Livebook.Config.writable_dir!("LIVEBOOK_HOME") do

lib/livebook/application.ex

@@ -7,14 +7,8 @@ defmodule Livebook.Application do
     ensure_directories!()
     set_local_file_system!()
 
-    if Livebook.Config.epmdless?() do
-      validate_epmdless!()
-      ensure_distribution!()
-    else
-      ensure_epmd!()
-      ensure_distribution!()
-    end
-
+    validate_epmd_module!()
+    start_distribution!()
     set_cookie()
 
     children =
@@ -48,6 +42,8 @@ defmodule Livebook.Application do
           Livebook.EPMD.NodePool,
           # Start the server responsible for associating files with sessions
           Livebook.Session.FileGuard,
+          # Start the supervisor dynamically managing runtimes
+          {DynamicSupervisor, name: Livebook.RuntimeSupervisor, strategy: :one_for_one},
           # Start the supervisor dynamically managing sessions
           {DynamicSupervisor, name: Livebook.SessionSupervisor, strategy: :one_for_one},
           # Start the registry for managing unique connections
@@ -124,60 +120,33 @@ defmodule Livebook.Application do
     :persistent_term.put(:livebook_local_file_system, local_file_system)
   end
 
-  defp validate_epmdless!() do
-    with {:ok, [[~c"Elixir.Livebook.EPMD"]]} <- :init.get_argument(:epmd_module),
-         {:ok, [[~c"false"]]} <- :init.get_argument(:start_epmd),
-         {:ok, [[~c"0"]]} <- :init.get_argument(:erl_epmd_port) do
-      :ok
-    else
+  defp validate_epmd_module!() do
+    # We use a custom EPMD module. In releases and Escript, we make
+    # sure the necessary erl flags are set. When running from source,
+    # those need to be passed explicitly.
+    case :init.get_argument(:epmd_module) do
+      {:ok, [[~c"Elixir.Livebook.EPMD"]]} ->
+        :ok
+
       _ ->
         Livebook.Config.abort!("""
-        You must specify ELIXIR_ERL_OPTIONS=\"-epmd_module Elixir.Livebook.EPMD -start_epmd false -erl_epmd_port 0\" with LIVEBOOK_EPMDLESS. \
-        The epmd module can be found inside #{Application.app_dir(:livebook, "priv/ebin")}.
+        You must set the environment variable ELIXIR_ERL_OPTIONS="-epmd_module Elixir.Livebook.EPMD"
         """)
     end
   end
 
-  defp ensure_epmd!() do
-    unless Node.alive?() do
-      case System.cmd("epmd", ["-daemon"]) do
-        {_, 0} ->
-          :ok
-
-        _ ->
-          Livebook.Config.abort!("""
-          Could not start epmd (Erlang Port Mapper Daemon). Livebook uses epmd to \
-          talk to different runtimes. You may have to start epmd explicitly by calling:
-
-              epmd -daemon
-
-          Or by calling:
-
-              elixir --sname test -e "IO.puts node()"
-
-          Then you can try booting Livebook again
-          """)
-      end
-    end
-  end
-
-  defp ensure_distribution!() do
-    unless Node.alive?() do
-      node = get_node_name()
+  defp start_distribution!() do
+    node = get_node_name()
 
-      case Node.start(node, :longnames) do
-        {:ok, _} ->
-          :ok
+    case Node.start(node, :longnames) do
+      {:ok, _} ->
+        :ok
 
-        {:error, reason} ->
-          Livebook.Config.abort!("Could not start distributed node: #{inspect(reason)}")
-      end
+      {:error, reason} ->
+        Livebook.Config.abort!("Could not start distributed node: #{inspect(reason)}")
     end
   end
 
-  import Record
-  defrecordp :hostent, Record.extract(:hostent, from_lib: "kernel/include/inet.hrl")
-
   defp set_cookie() do
     cookie = Application.fetch_env!(:livebook, :cookie)
     Node.set_cookie(cookie)
@@ -356,10 +325,10 @@ defmodule Livebook.Application do
     })
   end
 
-  # We set ELIXIR_ERL_OPTIONS when LIVEBOOK_EPMDLESS is set to true.
-  # By design, we don't allow ELIXIR_ERL_OPTIONS to pass through.
-  # Use ERL_AFLAGS and ERL_ZFLAGS if you want to configure both
-  # Livebook and spawned runtimes.
+  # We set ELIXIR_ERL_OPTIONS to set our custom EPMD module when
+  # running from source. By design, we don't allow ELIXIR_ERL_OPTIONS
+  # to pass through. Use ERL_AFLAGS and ERL_ZFLAGS if you want to
+  # configure both Livebook and spawned runtimes.
   defp config_env_var?("ELIXIR_ERL_OPTIONS"), do: true
   defp config_env_var?("LIVEBOOK_" <> _), do: true
   defp config_env_var?("RELEASE_" <> _), do: true

lib/livebook/config.ex

@@ -60,12 +60,26 @@ defmodule Livebook.Config do
           })
   def docker_images() do
     version = app_version()
-    base = if version =~ "dev", do: "latest", else: version
+
+    {version, version_cuda} =
+      if version =~ "dev" do
+        {"edge", "latest"}
+      else
+        {version, version}
+      end
 
     [
-      %{tag: base, name: "Livebook", env: []},
-      %{tag: "#{base}-cuda11.8", name: "Livebook + CUDA 11.8", env: [{"XLA_TARGET", "cuda118"}]},
-      %{tag: "#{base}-cuda12.1", name: "Livebook + CUDA 12.1", env: [{"XLA_TARGET", "cuda120"}]}
+      %{tag: version, name: "Livebook", env: []},
+      %{
+        tag: "#{version_cuda}-cuda11.8",
+        name: "Livebook + CUDA 11.8",
+        env: [{"XLA_TARGET", "cuda118"}]
+      },
+      %{
+        tag: "#{version_cuda}-cuda12.1",
+        name: "Livebook + CUDA 12.1",
+        env: [{"XLA_TARGET", "cuda120"}]
+      }
     ]
   end
 
@@ -158,7 +172,7 @@ defmodule Livebook.Config do
   @spec tmp_path() :: String.t()
   def tmp_path() do
     tmp_dir = System.tmp_dir!() |> Path.expand()
-    Path.join(tmp_dir, "livebook")
+    Path.join([tmp_dir, "livebook", app_version()])
   end
 
   @doc """
@@ -353,13 +367,6 @@ defmodule Livebook.Config do
     Application.fetch_env!(:livebook, :update_instructions_url)
   end
 
-  @doc """
-  Returns a boolean if epmdless mode is configured.
-  """
-  def epmdless? do
-    Application.fetch_env!(:livebook, :epmdless)
-  end
-
   @doc """
   Returns the force ssl host if any.
   """
@@ -673,7 +680,7 @@ defmodule Livebook.Config do
         nil
 
       "standalone" ->
-        Livebook.Runtime.ElixirStandalone.new()
+        Livebook.Runtime.Standalone.new()
 
       "embedded" ->
         Livebook.Runtime.Embedded.new()