Sorry! I ended up forgetting to report back, but the solution was what @josevalim suggested (thank you, by the way!). We now have one base runtime.exs in the umbrella root, as well as one runtime.exs for each app that needs custom runtime configuration.

The code in mix.exs looks like this, with some explanation added:

  defp releases do
    [
      release_1: [
        version: "1.0.0",
        applications: [
          app_1: :permanent,
          app_2: :permanent,
          app_3: :permanent,
          app_4: :permanent, # assuming this app doesn't have runtime config
        ],
        include_executables_for: [:unix],
        config_providers: config_providers_for_apps([
          :app_1,
          :app_2,
          :app_3,
        ]),
        steps: [:assemble, &copy_configs/1],
      ],
      release_2: [
        version: "0.0.1",
        applications: [
          app_1: :permanent,
          app_2: :permanent,
          app_5: :permanent,
        ],
        include_executables_for: [:unix],
        runtime_config_path: "apps/app_5/config/runtime.exs", # In this case the base runtime is not the one from root, but the one in app_5
        config_providers: config_providers_for_apps([
          :app_1,
          :app_2,
        ]),
        steps: [:assemble, &copy_configs/1],
      ]
    ]
  end

  # Add the runtime.exs configuration for each provided app
  defp config_providers_for_apps(apps) do
    for app <- apps do
      {Config.Reader, path: {:system, "RELEASE_ROOT", "/apps/#{app}/config/runtime.exs"}, env: Mix.env()}
    end
  end

  # When assembling the release, we copy all the runtime.exs files defined in `config_providers` to it, keeping the relative app path to avoid collisions.
  defp copy_configs(%Mix.Release{path: release_directory_path, config_providers: config_providers} = release) do
    for {_module, path: {_context, _root, config_file_path}, env: _} <- config_providers do
      config_directory = Path.join(release_directory_path, Path.dirname(config_file_path))

      # Clean the config directory to make sure we are only including the files defined in the config_providers
      File.rm_rf!(config_directory)
      File.mkdir_p!(config_directory)

      File.cp!(Path.relative(config_file_path), Path.join(config_directory, Path.basename(config_file_path)))
    end

    release
  end

Does that help you @dylan-chong ?

Elixir has long moved away from configuration per application because it is confusing and error prone.

Regarding the original inquiry in this thread, note it is also possible to set up a base configuration and then have each individual release override only the parts it needs. This is preferred to having each configuration scattered around per application:

https://hexdocs.pm/elixir/1.12/Config.Provider.html#module-multiple-config-files

Thank you for the comments @josevalim and @gabrielpra! That was indeed very helpful