Added Docker registry speed up proxy.

Signed-off-by: Edmondfrank <[email protected]>

config/dev.exs

@@ -70,7 +70,11 @@ config :compass_admin, CompassAdminWeb.Endpoint,
   # Binding to loopback ipv4 address prevents access from other machines.
   # Change to `ip: {0, 0, 0, 0}` to allow access from other machines.
   server: true,
-  http: [ip: {0, 0, 0, 0}, port: 4000],
+  http: [
+    ip: {0, 0, 0, 0},
+    port: 4000,
+    protocol_options: [idle_timeout: 3_600_000]
+  ],
   check_origin: false,
   code_reloader: true,
   debug_errors: true,

config/runtime.exs

@@ -59,7 +59,7 @@ if config_env() == :prod do
       # for details about using IPv6 vs IPv4 and loopback vs public addresses.
       ip: {0, 0, 0, 0, 0, 0, 0, 0},
       port: port,
-      protocol_options: [max_header_value_length: 10240]
+      protocol_options: [max_header_value_length: 10240, idle_timeout: 1_800_000]
     ],
     secret_key_base: secret_key_base
 

lib/compass_admin/application.ex

@@ -36,6 +36,7 @@ defmodule CompassAdmin.Application do
       CompassAdminWeb.Telemetry,
       # Start the PubSub system
       {Phoenix.PubSub, name: CompassAdmin.PubSub},
+      CompassAdmin.DockerTokenCacher,
       # Start the Endpoint (http/https)
       CompassAdminWeb.Endpoint,
       # Start Redix

lib/compass_admin/docker_token_cacher.ex

@@ -0,0 +1,96 @@
+defmodule CompassAdmin.DockerTokenCacher do
+  use GenServer
+  @config Application.get_env(:compass_admin, CompassAdmin.Services.ExportMetrics, %{})
+
+  defmodule Token do
+    @enforce_keys [:token, :expires_at]
+    @refresh_lead_time_s 30 # Refresh 30 seconds before expiry
+    defstruct token: nil, expires_at: nil
+
+    def valid?(%__MODULE__{} = token) do
+      refresh_time = DateTime.add(DateTime.utc_now(), -@refresh_lead_time_s, :second)
+      DateTime.compare(token.expires_at, refresh_time) == :gt
+    end
+  end
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, [], name: __MODULE__)
+  end
+
+  def get(conn) do
+    GenServer.call(__MODULE__, {:get, conn})
+  end
+
+  @impl true
+  def init(_) do
+    {:ok, nil}
+  end
+
+  @impl true
+  def handle_call({:get, conn}, _from, nil) do
+    get_token(conn, %{})
+  end
+
+  def handle_call({:get, conn}, _from, state) do
+    token = Map.get(state, extract_key(conn))
+    if token && Token.valid?(token) do
+        {:reply, token, state}
+    else
+      get_token(conn, state)
+    end
+  end
+
+  defp extract_key(conn) do
+    [_|rest] = conn.path_info
+    path = if rest, do: Enum.join(rest, "/"), else: ""
+    cond do
+      route = Regex.named_captures(~r/(?<namespace>[^\s]+)\/manifests\/.*?/, path) ->
+        route["namespace"]
+      route = Regex.named_captures(~r/(?<namespace>[^\s]+)\/blobs\/.*?/, path) ->
+        route["namespace"]
+      route = Regex.named_captures(~r/(?<namespace>[^\s]+)\/tags\/.*?/, path) ->
+        route["namespace"]
+      true ->
+        ""
+    end
+  end
+
+  defp get_token(conn, state) do
+    key = extract_key(conn)
+    case fetch_token(key) do
+      {:ok, token, expiration} ->
+        new_token = %Token{token: token, expires_at: DateTime.add(DateTime.utc_now(), expiration)}
+        {:reply, new_token, Map.put(state, key, new_token)}
+
+      {:error, reason} ->
+        {:reply, {:error, reason}, state}
+    end
+  end
+
+  defp fetch_token(key) do
+    # Replace this with your actual token fetching logic
+    # Example using HTTPoison
+    base = "https://auth.docker.io/token?service=registry.docker.io"
+
+    url =
+    if String.length(key) > 1 do
+      "#{base}&scope=repository:#{key}:pull"
+    else
+      base
+    end
+
+    case HTTPoison.get(url, [], [proxy: @config[:proxy]]) do
+      {:ok, %{body: body}} ->
+        case Jason.decode(body) do
+          {:ok, %{"access_token" => token, "expires_in" => expires_in}} ->
+            {:ok, token, expires_in}
+
+          {:error, reason} ->
+            {:error, reason}
+        end
+
+      {:error, reason} ->
+        {:error, reason}
+    end
+  end
+end

lib/compass_admin_web/controllers/debug_controller.ex

@@ -1,7 +1,15 @@
 defmodule CompassAdminWeb.DebugController do
+  alias Plug.Conn
+  alias CompassAdmin.DockerTokenCacher
+  alias CompassAdmin.DockerTokenCacher.Token
+
   use CompassAdminWeb, :controller
+
   import CompassAdminWeb.Helpers
 
+  @config Application.get_env(:compass_admin, CompassAdmin.Services.ExportMetrics, %{})
+  @client_options [proxy: @config[:proxy], timeout: 180_000, recv_timeout: 3_600_000]
+
   def webhook(conn, _params) do
     {:ok, body, conn} = Plug.Conn.read_body(conn)
     pretty_json(conn,
@@ -13,4 +21,57 @@ defmodule CompassAdminWeb.DebugController do
       }
     )
   end
+
+  def docker_registry_proxy(conn, _params) do
+    params =
+      ReverseProxyPlug.init(
+        upstream: "https://registry-1.docker.io",
+        client_options: @client_options,
+        response_mode: :buffer,
+        preserve_host_header: false
+      )
+
+    {:ok, body, conn} = Plug.Conn.read_body(conn)
+
+    case DockerTokenCacher.get(conn) do
+      %Token{token: token} ->
+        auth_conn =
+          %Conn{}
+          |> Map.merge(conn)
+          |> Conn.put_req_header("Authorization", "bearer #{token}")
+
+        auth_conn
+        |> ReverseProxyPlug.request(body, params)
+        |> handle_redirect(auth_conn)
+        |> ReverseProxyPlug.response(conn, params)
+      {:error, reason} ->
+        json(conn, %{error: reason})
+    end
+  end
+
+  def handle_redirect({:ok, %{status_code: code, headers: headers, request: %{headers: req_headers}}}, conn)  when code > 300 and code < 400 do
+    next = get_location(headers)
+    method =
+      conn.method
+      |> String.downcase()
+      |> String.to_existing_atom()
+    apply(HTTPoison, method, [next, remove_host(req_headers), @client_options])
+  end
+
+  def handle_redirect(resp, _), do: resp
+
+  defp get_location(headers) do
+    {_h, location} =
+      Enum.find(headers, fn {header, _location} ->
+        String.downcase(header) == "location"
+      end)
+
+    location
+  end
+
+  defp remove_host(headers) do
+    Enum.reject(headers, fn {header, _} ->
+      String.downcase(header) == "host"
+    end)
+  end
 end

lib/compass_admin_web/router.ex

@@ -17,6 +17,11 @@ defmodule CompassAdminWeb.Router do
     plug :accepts, ["json"]
   end
 
+  scope "/", CompassAdminWeb do
+    pipe_through :api
+    match :*, "/v2/*path", DebugController, :docker_registry_proxy
+  end
+
   scope "/admin", CompassAdminWeb do
     pipe_through :browser
     live "/", PageLive, :index

mix.exs

@@ -72,6 +72,8 @@ defmodule CompassAdmin.MixProject do
       {:jason, "~> 1.2"},
       {:sitemapper, "~> 0.6"},
       {:plug_cowboy, "~> 2.5"},
+      {:reverse_proxy_plug, "~> 3.0"},
+      {:httpoison, "~> 2.2"},
       {:petal_components, "~> 0.18.0"},
       {:ex_indexea, "~> 0.1.0"},
       {:backoffice, path: "backoffice"},

mix.lock

@@ -26,7 +26,7 @@
   "exile": {:hex, :exile, "0.9.1", "832b6340cf800661e90e52cebc760b795450f803c0e9265ccdc54150423fbb32", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "553a1847b27118c843d3dc6912adbc36d60336811d15ad70a31b82eb5a416328"},
   "fastglobal": {:hex, :fastglobal, "1.0.0", "f3133a0cda8e9408aac7281ec579c4b4a8386ce0e99ca55f746b9f58192f455b", [:mix], [], "hexpm", "cfdb7ed63910bc75f579cd09e2517618fa9418b56731d51d03f7ba4b400798d0"},
   "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
-  "finch": {:hex, :finch, "0.16.0", "40733f02c89f94a112518071c0a91fe86069560f5dbdb39f9150042f44dcfb1a", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f660174c4d519e5fec629016054d60edd822cdfe2b7270836739ac2f97735ec5"},
+  "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"},
   "flatten_map": {:hex, :flatten_map, "0.1.1", "0847c7db67500866eb8b037169bafa704df165e085887479397b24547b70d0ee", [:mix], [], "hexpm", "566f621ab4063eb30a868196cd4d0958833f63566a29b63d8942dba8b7bbef2d"},
   "floki": {:hex, :floki, "0.33.1", "f20f1eb471e726342b45ccb68edb9486729e7df94da403936ea94a794f072781", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "461035fd125f13fdf30f243c85a0b1e50afbec876cbf1ceefe6fddd2e6d712c6"},
   "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"},
@@ -73,6 +73,7 @@
   "recon": {:hex, :recon, "2.5.3", "739107b9050ea683c30e96de050bc59248fd27ec147696f79a8797ff9fa17153", [:mix, :rebar3], [], "hexpm", "6c6683f46fd4a1dfd98404b9f78dcabc7fcd8826613a89dcb984727a8c3099d7"},
   "redix": {:hex, :redix, "1.4.1", "8303e13bad38ca80c15bdf79ea9cbd6eb879554c9cbb815b35df1602d7b1549d", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "676b5ce37d7b1d46931d506e3208786bd8334a1625ecb591d87d790b23ffbd1f"},
   "redlock": {:hex, :redlock, "1.0.21", "7c6b0eaa8470fb6fea24d565fd116ac98a6a0e309fda5366607bceef6733cdbe", [:mix], [{:ex_hash_ring, "~> 3.0", [hex: :ex_hash_ring, repo: "hexpm", optional: false]}, {:fastglobal, "~> 1.0.0", [hex: :fastglobal, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}, {:redix, "~> 1.3", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "7e72e5b13148a3a0f5565513992543c01bdf20d082385cb91a11ea3b3e8c9cdd"},
+  "reverse_proxy_plug": {:hex, :reverse_proxy_plug, "3.0.2", "38fde2f59bca8b219ef4f1ec0c0849a67c6d9705160e426a2354f35399db5c7b", [:mix], [{:finch, "~> 0.18", [hex: :finch, repo: "hexpm", optional: true]}, {:httpoison, "~> 1.2 or ~> 2.0", [hex: :httpoison, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: false]}, {:req, "~> 0.3.0 or ~> 0.4.0 or ~> 0.5.0", [hex: :req, repo: "hexpm", optional: true]}, {:tesla, "~> 1.4", [hex: :tesla, repo: "hexpm", optional: true]}], "hexpm", "31ae5e068f7f504fba1b5c17c31c87966c720809ac15140c6c181440fbd24eda"},
   "sitemapper": {:hex, :sitemapper, "0.7.0", "4aee7930327a9a01b1c9b81d1d42f60c1a295e9f420108eb2d130c317415abd7", [:mix], [{:ex_aws_s3, "~> 2.0", [hex: :ex_aws_s3, repo: "hexpm", optional: true]}, {:xml_builder, "~> 2.1", [hex: :xml_builder, repo: "hexpm", optional: false]}], "hexpm", "60f7a684e5e9fe7f10ac5b69f48b0be2bcbba995afafcb3c143fc0c8ef1f223f"},
   "snap": {:hex, :snap, "0.8.1", "d95a7ecbc911a5a12f419c7108ce9e44db161b9177d137f14d22a8c1fe85cf4e", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:finch, "~> 0.8", [hex: :finch, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "929e2d3254697c85c0226cabe6b40eda6703cfece4fe8285c4b82f3e66ee6291"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
@@ -85,7 +86,7 @@
   "thoas": {:hex, :thoas, "1.0.0", "567c03902920827a18a89f05b79a37b5bf93553154b883e0131801600cf02ce0", [:rebar3], [], "hexpm", "fc763185b932ecb32a554fb735ee03c3b6b1b31366077a2427d2a97f3bd26735"},
   "timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"},
   "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"},
-  "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"},
+  "tzdata": {:hex, :tzdata, "1.1.2", "45e5f1fcf8729525ec27c65e163be5b3d247ab1702581a94674e008413eef50b", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "cec7b286e608371602318c414f344941d5eb0375e14cfdab605cca2fe66cba8b"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
   "vapor": {:hex, :vapor, "0.10.0", "547a94b381093dea61a4ca2200109385b6e44b86d72d1ebf93e5ac1a8873bc3c", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:norm, "~> 0.9", [hex: :norm, repo: "hexpm", optional: false]}, {:toml, "~> 0.3", [hex: :toml, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.1", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "ee6d089a71309647a0a2a2ae6cf3bea61739a983e8c1310d53ff04b1493afbc1"},
   "xml_builder": {:hex, :xml_builder, "2.2.0", "cc5f1eeefcfcde6e90a9b77fb6c490a20bc1b856a7010ce6396f6da9719cbbab", [:mix], [], "hexpm", "9d66d52fb917565d358166a4314078d39ef04d552904de96f8e73f68f64a62c9"},