defmodule Req.Steps do @moduledoc """ The collection of built-in steps. Req is composed of: * `Req` - the high-level API * `Req.Request` - the low-level API and the request struct * `Req.Steps` - the collection of built-in steps (you're here!) * `Req.Test` - the testing conveniences """ require Logger @doc false def attach(req) do req |> Req.Request.register_options([ # request steps :user_agent, :compressed, :range, :base_url, :params, :path_params, :path_params_style, :auth, :form, :form_multipart, :json, :compress_body, :checksum, :aws_sigv4, # response steps :raw, :http_errors, :decode_body, :decode_json, :redirect, :redirect_trusted, :redirect_log_level, :max_redirects, :retry, :retry_delay, :retry_log_level, :max_retries, :cache, :cache_dir, :plug, :finch, :finch_request, :finch_private, :connect_options, :inet6, :receive_timeout, :pool_timeout, :unix_socket, :pool_max_idle_time, # TODO: Remove on Req 1.0 :output, :follow_redirects, :location_trusted, :redact_auth ]) |> Req.Request.prepend_request_steps( put_user_agent: &Req.Steps.put_user_agent/1, compressed: &Req.Steps.compressed/1, encode_body: &Req.Steps.encode_body/1, put_base_url: &Req.Steps.put_base_url/1, auth: &Req.Steps.auth/1, put_params: &Req.Steps.put_params/1, put_path_params: &Req.Steps.put_path_params/1, put_range: &Req.Steps.put_range/1, cache: &Req.Steps.cache/1, put_plug: &Req.Steps.put_plug/1, compress_body: &Req.Steps.compress_body/1, checksum: &Req.Steps.checksum/1, put_aws_sigv4: &Req.Steps.put_aws_sigv4/1 ) |> Req.Request.prepend_response_steps( retry: &Req.Steps.retry/1, handle_http_errors: &Req.Steps.handle_http_errors/1, redirect: &Req.Steps.redirect/1, decompress_body: &Req.Steps.decompress_body/1, verify_checksum: &Req.Steps.verify_checksum/1, decode_body: &Req.Steps.decode_body/1, output: &Req.Steps.output/1 ) |> Req.Request.prepend_error_steps(retry: &Req.Steps.retry/1) end ## Request steps @doc """ Sets base URL for all requests. ## Request Options * `:base_url` - if set, the request URL is merged with this base URL. The base url can be a string, a `%URI{}` struct, a 0-arity function, or a `{mod, fun, args}` tuple describing a function to call. ## Examples iex> req = Req.new(base_url: "https://httpbin.org") iex> Req.get!(req, url: "/status/200").status 200 iex> Req.get!(req, url: "/status/201").status 201 """ @doc step: :request def put_base_url(request) def put_base_url(%{options: %{base_url: base_url}} = request) do if request.url.scheme != nil do request else base_url = case base_url do binary when is_binary(binary) -> binary %URI{} = url -> URI.to_string(url) fun when is_function(fun, 0) -> case fun.() do binary when is_binary(binary) -> binary %URI{} = url -> URI.to_string(url) end {mod, fun, args} when is_atom(mod) and is_atom(fun) and is_list(args) -> case apply(mod, fun, args) do binary when is_binary(binary) -> binary %URI{} = url -> URI.to_string(url) end end %{request | url: URI.parse(join(base_url, request.url))} end end def put_base_url(request) do request end defp join(base, url) do case {:binary.last(base), to_string(url)} do {?/, "/" <> rest} -> base <> rest {?/, rest} -> base <> rest {_, ""} -> base {_, "/" <> rest} -> base <> "/" <> rest {_, rest} -> base <> "/" <> rest end end @doc """ Sets request authentication. ## Request Options * `:auth` - sets the `authorization` header: * `string` - sets to this value; * `{:basic, userinfo}` - uses Basic HTTP authentication; * `{:bearer, token}` - uses Bearer HTTP authentication; * `:netrc` - load credentials from `.netrc` at path specified in `NETRC` environment variable. If `NETRC` is not set, load `.netrc` in user's home directory; * `{:netrc, path}` - load credentials from `path` * `fn -> {:bearer, "eyJ0eXAi..." } end` - a 0-arity function that returns one of the aforementioned types. ## Examples iex> Req.get!("https://httpbin.org/basic-auth/foo/bar", auth: {:basic, "foo:foo"}).status 401 iex> Req.get!("https://httpbin.org/basic-auth/foo/bar", auth: {:basic, "foo:bar"}).status 200 iex> Req.get!("https://httpbin.org/basic-auth/foo/bar", auth: fn -> {:basic, "foo:bar"} end).status 200 iex> Req.get!("https://httpbin.org/bearer", auth: {:bearer, ""}).status 401 iex> Req.get!("https://httpbin.org/bearer", auth: {:bearer, "foo"}).status 200 iex> Req.get!("https://httpbin.org/bearer", auth: fn -> {:bearer, "foo"} end).status 200 iex> System.put_env("NETRC", "./test/my_netrc") iex> Req.get!("https://httpbin.org/basic-auth/foo/bar", auth: :netrc).status 200 iex> Req.get!("https://httpbin.org/basic-auth/foo/bar", auth: {:netrc, "./test/my_netrc"}).status 200 iex> Req.get!("https://httpbin.org/basic-auth/foo/bar", auth: fn -> {:netrc, "./test/my_netrc"} end).status 200 """ @doc step: :request def auth(request) do auth(request, request.options[:auth]) end defp auth(request, nil) do request end defp auth(request, authorization) when is_binary(authorization) do Req.Request.put_header(request, "authorization", authorization) end defp auth(request, {:basic, userinfo}) when is_binary(userinfo) do Req.Request.put_header(request, "authorization", "Basic " <> Base.encode64(userinfo)) end defp auth(request, {:bearer, token}) when is_binary(token) do Req.Request.put_header(request, "authorization", "Bearer " <> token) end defp auth(request, fun) when is_function(fun, 0) do value = fun.() if is_function(value, 0) do raise ArgumentError, "setting `auth: fn -> ... end` should not return another function" end auth(request, value) end defp auth(request, :netrc) do path = System.get_env("NETRC") || Path.join(System.user_home!(), ".netrc") authenticate_with_netrc(request, path) end defp auth(request, {:netrc, path}) do authenticate_with_netrc(request, path) end defp auth(request, {username, password}) when is_binary(username) and is_binary(password) do IO.warn( "setting `auth: {username, password}` is deprecated in favour of `auth: {:basic, userinfo}`" ) Req.Request.put_header( request, "authorization", "Basic " <> Base.encode64("#{username}:#{password}") ) end defp authenticate_with_netrc(request, path_or_device) do case Map.fetch(Req.Utils.load_netrc(path_or_device), request.url.host) do {:ok, {username, password}} -> auth(request, {:basic, "#{username}:#{password}"}) :error -> request end end @user_agent "req/#{Mix.Project.config()[:version]}" @doc """ Sets the user-agent header. ## Request Options * `:user_agent` - sets the `user-agent` header. Defaults to `"#{@user_agent}"`. ## Examples iex> Req.get!("https://httpbin.org/user-agent").body %{"user-agent" => "#{@user_agent}"} iex> Req.get!("https://httpbin.org/user-agent", user_agent: "foo").body %{"user-agent" => "foo"} """ @doc step: :request def put_user_agent(request) do user_agent = Req.Request.get_option(request, :user_agent, @user_agent) Req.Request.put_new_header(request, "user-agent", user_agent) end @doc """ Asks the server to return compressed response. Supported formats: * `gzip` * `br` (if [brotli] is installed) * `zstd` (if [ezstd] is installed) ## Request Options * `:compressed` - if set to `true`, sets the `accept-encoding` header with compression algorithms that Req supports. Defaults to `true`. When streaming response body (`into: fun | collectable`), `compressed` defaults to `false`. ## Examples Req automatically decompresses response body (`decompress_body/1` step) so let's disable that by passing `raw: true`. By default, we ask the server to send compressed response. Let's look at the headers and the raw body. Notice the body starts with `<<31, 139>>` (`<<0x1F, 0x8B>>`), the "magic bytes" for gzip: iex> response = Req.get!("https://elixir-lang.org", raw: true) iex> Req.Response.get_header(response, "content-encoding") ["gzip"] iex> response.body |> binary_part(0, 2) <<31, 139>> Now, let's pass `compressed: false` and notice the raw body was not compressed: iex> response = Req.get!("https://elixir-lang.org", raw: true, compressed: false) iex> response.body |> binary_part(0, 15) "" The Brotli and Zstandard compression algorithms are also supported if the optional packages are installed: Mix.install([ :req, {:brotli, "~> 0.3.0"}, {:ezstd, "~> 1.0"} ]) response = Req.get!("https://httpbin.org/anything") response.body["headers"]["Accept-Encoding"] #=> "zstd, br, gzip" [brotli]: https://hex.pm/packages/brotli [ezstd]: https://hex.pm/packages/ezstd """ @doc step: :request def compressed(%Req.Request{into: nil} = request) do case Req.Request.get_option(request, :compressed, true) do true -> Req.Request.put_new_header(request, "accept-encoding", supported_accept_encoding()) false -> request end end def compressed(request) do request end defmacrop brotli_loaded? do Code.ensure_loaded?(:brotli) end defmacrop ezstd_loaded? do Code.ensure_loaded?(:ezstd) end defp supported_accept_encoding do value = "gzip" value = if brotli_loaded?(), do: "br, " <> value, else: value if ezstd_loaded?(), do: "zstd, " <> value, else: value end @doc """ Encodes the request body. ## Request Options * `:form` - if set, encodes the request body as `application/x-www-form-urlencoded` (using `URI.encode_query/1`). * `:form_multipart` - if set, encodes the request body as `multipart/form-data`. It accepts `name` / `value` pairs. `value` can be one of: * integer (automatically encoded as string) * iodata * `File.Stream` * `Enumerable` * `{value, options}` tuple. `value` can be any of the values mentioned above. Supported options are: `:filename`, `:content_type`, and `:size`. When `value` is an `Enumerable`, option `:size` can be set with the binary size of the `value`. The size will be used to calculate and send the `content-length` header which might be required for some servers. There is no need to pass `:size` for `integer`, `iodata`, and `File.Stream` values as it's automatically calculated. * `:json` - if set, encodes the request body as JSON (using `Jason.encode_to_iodata!/1`), sets the `accept` header to `application/json`, and the `content-type` header to `application/json`. ## Examples Encoding form (`application/x-www-form-urlencoded`): iex> Req.post!("https://httpbin.org/anything", form: [a: 1]).body["form"] %{"a" => "1"} Encoding form (`multipart/form-data`): iex> fields = [a: 1, b: {"2", filename: "b.txt"}] iex> resp = Req.post!("https://httpbin.org/anything", form_multipart: fields) iex> resp.body["form"] %{"a" => "1"} iex> resp.body["files"] %{"b" => "2"} Encoding streaming form (`multipart/form-data`): iex> stream = Stream.cycle(["abc"]) |> Stream.take(3) iex> fields = [file: {stream, filename: "b.txt"}] iex> resp = Req.post!("https://httpbin.org/anything", form_multipart: fields) iex> resp.body["files"] %{"file" => "abcabcabc"} # with explicit :size iex> stream = Stream.cycle(["abc"]) |> Stream.take(3) iex> fields = [file: {stream, filename: "b.txt", size: 9}] iex> resp = Req.post!("https://httpbin.org/anything", form_multipart: fields) iex> resp.body["files"] %{"file" => "abcabcabc"} Encoding JSON: iex> Req.post!("https://httpbin.org/post", json: %{a: 2}).body["json"] %{"a" => 2} """ @doc step: :request def encode_body(request) do cond do data = request.options[:form] -> %{request | body: URI.encode_query(data)} |> Req.Request.put_new_header("content-type", "application/x-www-form-urlencoded") data = request.options[:form_multipart] -> multipart = Req.Utils.encode_form_multipart(data) %{request | body: multipart.body} |> Req.Request.put_new_header("content-type", multipart.content_type) |> then(&maybe_put_content_length(&1, multipart.size)) data = request.options[:json] -> %{request | body: Jason.encode_to_iodata!(data)} |> Req.Request.put_new_header("content-type", "application/json") |> Req.Request.put_new_header("accept", "application/json") true -> request end end defp maybe_put_content_length(req, nil), do: req defp maybe_put_content_length(req, size) do Req.Request.put_new_header(req, "content-length", Integer.to_string(size)) end @doc """ Uses a templated request path. By default, params in the URL path are expressed as strings prefixed with `:`. For example, `:code` in `https://httpbin.org/status/:code`. If you want to use the `{code}` syntax, set `path_params_style: :curly`. Param names must start with a letter and can contain letters, digits, and underscores; this is true both for `:colon_params` as well as `{curly_params}`. Path params are replaced in the request URL path. The path params are specified as a keyword list of parameter names and values, as in the examples below. The values of the parameters are converted to strings using the `String.Chars` protocol (`to_string/1`). ## Request Options * `:path_params` - params to add to the templated path. Defaults to `[]`. * `:path_params_style` (*available since v0.5.1*) - how path params are expressed. Can be one of: * `:colon` - (default) for Plug-style parameters, such as `:code` in `https://httpbin.org/status/:code`. * `:curly` - for [OpenAPI](https://swagger.io/specification/)-style parameters, such as `{code}` in `https://httpbin.org/status/{code}`. ## Examples iex> Req.get!("https://httpbin.org/status/:code", path_params: [code: 201]).status 201 iex> Req.get!("https://httpbin.org/status/{code}", path_params: [code: 201], path_params_style: :curly).status 201 """ @doc step: :request def put_path_params(request) do put_path_params(request, Req.Request.get_option(request, :path_params, [])) end defp put_path_params(request, []) do request end defp put_path_params(request, params) do request |> Req.Request.put_private(:path_params_template, request.url.path) |> apply_path_params(params) end defp apply_path_params(request, params) do regex = case Req.Request.get_option(request, :path_params_style, :colon) do :colon -> ~r/:([a-zA-Z]{1}[\w_]*)/ :curly -> ~r/\{([a-zA-Z]{1}[\w_]*)\}/ end update_in(request.url.path, fn nil -> nil path -> Regex.replace(regex, path, fn match, key -> case params[String.to_existing_atom(key)] do nil -> match value -> value |> to_string() |> URI.encode() end end) end) end @doc """ Adds params to request query string. ## Request Options * `:params` - params to add to the request query string. Defaults to `[]`. ## Examples iex> Req.get!("https://httpbin.org/anything/query", params: [x: 1, y: 2]).body["args"] %{"x" => "1", "y" => "2"} """ @doc step: :request def put_params(request) do put_params(request, Req.Request.get_option(request, :params, [])) end defp put_params(request, []) do request end defp put_params(request, params) do encoded = URI.encode_query(params) update_in(request.url.query, fn nil -> encoded query -> query <> "&" <> encoded end) end @doc """ Sets the "Range" request header. ## Request Options * `:range` - can be one of the following: * a string - returned as is * a `first..last` range - converted to `"bytes=-"` ## Examples iex> response = Req.get!("https://httpbin.org/range/100", range: 0..3) iex> response.status 206 iex> response.body "abcd" iex> Req.Response.get_header(response, "content-range") ["bytes 0-3/100"] """ @doc step: :request def put_range(%{options: %{range: range}} = request) when is_binary(range) do Req.Request.put_header(request, "range", range) end def put_range(%{options: %{range: first..last//1}} = request) do Req.Request.put_header(request, "range", "bytes=#{first}-#{last}") end def put_range(request) do request end @doc """ Performs HTTP caching using `if-modified-since` header. Only successful (200 OK) responses are cached. This step also _prepends_ a response step that loads and writes the cache. Be careful when _prepending_ other response steps, make sure the cache is loaded/written as soon as possible. ## Options * `:cache` - if `true`, performs simple caching using `if-modified-since` header. Defaults to `false`. * `:cache_dir` - the directory to store the cache, defaults to `/req` (see: `:filename.basedir/3`) ## Examples iex> url = "https://elixir-lang.org" iex> response1 = Req.get!(url, cache: true) iex> response2 = Req.get!(url, cache: true) iex> response1 == response2 true """ @doc step: :request def cache(request) do case request.options[:cache] do true -> dir = request.options[:cache_dir] || :filename.basedir(:user_cache, ~c"req") cache_path = cache_path(dir, request) request |> put_if_modified_since(cache_path) |> Req.Request.prepend_response_steps(handle_cache: &handle_cache(&1, cache_path)) other when other in [false, nil] -> request end end defp put_if_modified_since(request, cache_path) do case File.stat(cache_path) do {:ok, stat} -> http_datetime_string = stat.mtime |> NaiveDateTime.from_erl!() |> DateTime.from_naive!("Etc/UTC") |> Req.Utils.format_http_date() Req.Request.put_new_header(request, "if-modified-since", http_datetime_string) _ -> request end end defp handle_cache({request, response}, cache_path) do cond do response.status == 200 -> write_cache(cache_path, response) {request, response} response.status == 304 -> response = load_cache(cache_path) {request, response} true -> {request, response} end end @doc """ Compresses the request body. ## Request Options * `:compress_body` - if set to `true`, compresses the request body using gzip. Defaults to `false`. """ @doc step: :request def compress_body(request) do if request.body && request.options[:compress_body] do body = case request.body do iodata when is_binary(iodata) or is_list(iodata) -> :zlib.gzip(iodata) enumerable -> Req.Utils.stream_gzip(enumerable) end request |> Map.replace!(:body, body) |> Req.Request.put_header("content-encoding", "gzip") else request end end @default_finch_options Req.Finch.pool_options(%{}) @doc """ Runs the request using `Finch`. This is the default Req _adapter_. See ["Adapter" section in the `Req.Request`](Req.Request.html#module-adapter) module documentation for more information on adapters. Finch returns `Mint.TransportError` exceptions on HTTP connection problems. These are automatically converted to `Req.TransportError` exceptions. Similarly, HTTP-protocol-related errors, `Mint.HTTPError` and `Finch.Error`, and converted to `Req.HTTPError`. ## HTTP/1 Pools On HTTP/1 connections, Finch creates a pool per `{scheme, host, port}` tuple. These pools are kept around to re-use connections as much as possible, however they are **not automatically terminated**. To do so, you can configure custom Finch pool: {:ok, _} = Finch.start_link( name: MyFinch, pools: %{ default: [ # terminate idle {scheme, host, port} pool after 60s pool_max_idle_time: 60_000 ] } ) Req.get!("https://httpbin.org/json", finch: MyFinch) More commonly you'd add the the custom Finch pool as part of your supervision tree in your `application.ex`: children = [ {Finch, name: MyFinch, pools: %{ default: [size: 70] }} ] That way you can also configure a bigger pool size for the HTTP pool. You just mustn't forget to pass along `finch: MyFinch` as discussed above. You could use `Req.default_options/1` to make it a global default but it's generally discouraged. For documentation about the possible pool options and their meaning, please check out the [Finch docs on pool configuration options](https://hexdocs.pm/finch/Finch.html#start_link/1-pool-configuration-options). ## Request Options * `:finch` - the name of the Finch pool. Defaults to a pool automatically started by Req. * `:connect_options` - dynamically starts (or re-uses already started) Finch pool with the given connection options: * `:timeout` - socket connect timeout in milliseconds, defaults to `30_000`. * `:protocols` - the HTTP protocols to use, defaults to `#{inspect(Keyword.fetch!(@default_finch_options, :protocols))}`. * `:hostname` - Mint explicit hostname, see `Mint.HTTP.connect/4` for more information. * `:transport_opts` - Mint transport options, see `Mint.HTTP.connect/4` for more information. * `:proxy_headers` - Mint proxy headers, see `Mint.HTTP.connect/4` for more information. * `:proxy` - Mint HTTP/1 proxy settings, a `{schema, address, port, options}` tuple. See `Mint.HTTP.connect/4` for more information. * `:client_settings` - Mint HTTP/2 client settings, see `Mint.HTTP.connect/4` for more information. * `:inet6` - if set to true, uses IPv6. If the request URL looks like IPv6 address, i.e., say, `[::1]`, it defaults to `true` and otherwise defaults to `false`. This is a shortcut for setting `connect_options: [transport_opts: [inet6: true]]`. * `:pool_timeout` - pool checkout timeout in milliseconds, defaults to `5000`. * `:receive_timeout` - socket receive timeout in milliseconds, defaults to `15_000`. * `:unix_socket` - if set, connect through the given UNIX domain socket. * `:pool_max_idle_time` - the maximum number of milliseconds that a pool can be idle before being terminated, used only by HTTP1 pools. Default to `:infinity`. * `:finch_private` - a map or keyword list of private metadata to add to the Finch request. May be useful for adding custom data when handling telemetry with `Finch.Telemetry`. * `:finch_request` - a function that executes the Finch request, defaults to using `Finch.request/3`. The function should accept 4 arguments: * `request` - the `%Req.Request{}` struct * `finch_request` - the Finch request * `finch_name` - the Finch name * `finch_options` - the Finch options And it should return either `{request, response}` or `{request, exception}`. ## Examples Custom `:receive_timeout`: iex> Req.get!(url, receive_timeout: 1000) Connecting through UNIX socket: iex> Req.get!("http:///v1.41/_ping", unix_socket: "/var/run/docker.sock").body "OK" Custom connection options: iex> Req.get!(url, connect_options: [timeout: 5000]) iex> Req.get!(url, connect_options: [protocols: [:http2]]) Connecting without certificate check (useful in development, not recommended in production): iex> Req.get!(url, connect_options: [transport_opts: [verify: :verify_none]]) Connecting with custom certificates: iex> Req.get!(url, connect_options: [transport_opts: [cacertfile: "certs.pem"]]) Connecting through a proxy with basic authentication: iex> Req.new( ...> url: "https://elixir-lang.org", ...> connect_options: [ ...> proxy: {:http, "your.proxy.com", 8888, []}, ...> proxy_headers: [{"proxy-authorization", "Basic " <> Base.encode64("user:pass")}] ...> ] ...> ) iex> |> Req.get!() Transport errors are represented as `Req.TransportError` exceptions: iex> Req.get("https://httpbin.org/delay/1", receive_timeout: 0, retry: false) {:error, %Req.TransportError{reason: :timeout}} """ @doc step: :request def run_finch(request) do Req.Finch.run(request) end @doc """ Sets adapter to `run_plug/1`. See `run_plug/1` for more information. ## Request Options * `:plug` - if set, the plug to run the request through. """ @doc step: :request def put_plug(request) do if request.options[:plug] do %{request | adapter: &run_plug/1} else request end end @doc """ Runs the request against a plug instead of over the network. This step is a Req _adapter_. It is set as the adapter by the `put_plug/1` step if the `:plug` option is set. It requires [`:plug`](https://hexdocs.pm/plug) dependency: {:plug, "~> 1.0"} ## Request Options * `:plug` - the plug to run the request through. It can be one of: * A _function_ plug: a `fun(conn)` or `fun(conn, options)` function that takes a `Plug.Conn` and returns a `Plug.Conn`. * A _module_ plug: a `module` name or a `{module, options}` tuple. Req automatically calls `Plug.Conn.fetch_query_params/2` before your plug, so you can get query params using `conn.query_params`. Req also automatically parses request body using `Plug.Parsers` for JSON, urlencoded and multipart requests and you can access it with `conn.body_params`. The raw request body of the request is available by calling `Req.Test.raw_body/1` with the `conn` in your tests. ## Examples This step is particularly useful to test plugs: defmodule Echo do def call(conn, _) do "/" <> path = conn.request_path Plug.Conn.send_resp(conn, 200, path) end end test "echo" do assert Req.get!("http:///hello", plug: Echo).body == "hello" end You can define plugs as functions too: test "echo" do echo = fn conn -> "/" <> path = conn.request_path Plug.Conn.send_resp(conn, 200, path) end assert Req.get!("http:///hello", plug: echo).body == "hello" end which is particularly useful to create HTTP service stubs, similar to tools like [Bypass](https://github.com/PSPDFKit-labs/bypass). Response streaming is also supported however at the moment the entire response body is emitted as one chunk: test "echo" do plug = fn conn -> conn = Plug.Conn.send_chunked(conn, 200) {:ok, conn} = Plug.Conn.chunk(conn, "echo") {:ok, conn} = Plug.Conn.chunk(conn, "echo") conn end assert Req.get!(plug: plug, into: []).body == ["echoecho"] end When testing JSON APIs, it's common to use the `Req.Test.json/2` helper: test "JSON" do plug = fn conn -> Req.Test.json(conn, %{message: "Hello, World!"}) end resp = Req.get!(plug: plug) assert resp.status == 200 assert resp.headers["content-type"] == ["application/json; charset=utf-8"] assert resp.body == %{"message" => "Hello, World!"} end You can simulate network errors by calling `Req.Test.transport_error/2` in your plugs: test "network issues" do plug = fn conn -> Req.Test.transport_error(conn, :timeout) end assert Req.get(plug: plug, retry: false) == {:error, %Req.TransportError{reason: :timeout}} end """ @doc step: :request def run_plug(request) if Code.ensure_loaded?(Plug.Test) do def run_plug(request) do plug = request.options.plug req_body = case request.body do iodata when is_binary(iodata) or is_list(iodata) -> IO.iodata_to_binary(iodata) nil -> "" enumerable -> enumerable |> Enum.to_list() |> IO.iodata_to_binary() end req_headers = if unquote(Req.MixProject.legacy_headers_as_lists?()) do request.headers else for {name, values} <- request.headers, value <- values do {name, value} end end parser_opts = Plug.Parsers.init( parsers: [:urlencoded, :multipart, :json], pass: ["*/*"], json_decoder: Jason ) conn = Req.Test.Adapter.conn(%Plug.Conn{}, request.method, request.url, req_body) |> Map.replace!(:req_headers, req_headers) |> Plug.Conn.fetch_query_params() |> Plug.Parsers.call(parser_opts) # Handle cases where the body isn't read with Plug.Parsers {mod, state} = conn.adapter state = %{state | body_read: true} conn = %{conn | adapter: {mod, state}} conn = call_plug(conn, plug) unless match?(%Plug.Conn{}, conn) do raise ArgumentError, "expected to return %Plug.Conn{}, got: #{inspect(conn)}" end if exception = conn.private[:req_test_exception] do {request, exception} else handle_plug_result(conn, request) end end defp handle_plug_result(conn, request) do # consume messages sent by Plug.Test adapter {_, %{ref: ref}} = conn.adapter if conn.state == :unset do raise """ expected connection to have a response but no response was set/sent. Please verify that you are using Plug.Conn.send_resp/3 in your plug: Req.Test.stub(MyStub, fn conn, -> Plug.Conn.send_resp(conn, 200, "Hello, World!") end) """ end receive do {^ref, {_status, _headers, _body}} -> :ok after 0 -> :ok end receive do {:plug_conn, :sent} -> :ok after 0 -> :ok end case request.into do nil -> response = Req.Response.new( status: conn.status, headers: conn.resp_headers, body: conn.resp_body ) {request, response} fun when is_function(fun, 2) -> response = Req.Response.new( status: conn.status, headers: conn.resp_headers ) case fun.({:data, conn.resp_body}, {request, response}) do {:cont, acc} -> acc {:halt, acc} -> acc other -> raise ArgumentError, "expected {:cont, acc}, got: #{inspect(other)}" end :self -> async = %Req.Response.Async{ pid: self(), ref: make_ref(), stream_fun: &plug_parse_message/2, cancel_fun: &plug_cancel/1 } resp = Req.Response.new(status: conn.status, headers: conn.resp_headers, body: async) send(self(), {async.ref, {:data, conn.resp_body}}) send(self(), {async.ref, :done}) {request, resp} collectable -> response = Req.Response.new( status: conn.status, headers: conn.resp_headers ) if conn.status == 200 do {acc, collector} = Collectable.into(collectable) acc = collector.(acc, {:cont, conn.resp_body}) acc = collector.(acc, :done) {request, %{response | body: acc}} else {request, %{response | body: conn.resp_body}} end end end defp plug_parse_message(ref, {ref, {:data, data}}) do {:ok, [data: data]} end defp plug_parse_message(ref, {ref, :done}) do {:ok, [:done]} end defp plug_parse_message(_, _) do :unknown end defp plug_cancel(ref) do plug_clean_responses(ref) :ok end defp plug_clean_responses(ref) do receive do {^ref, _} -> plug_clean_responses(ref) after 0 -> :ok end end defp call_plug(conn, plug) when is_atom(plug) do plug.call(conn, []) end defp call_plug(conn, {plug, options}) when is_atom(plug) do plug.call(conn, plug.init(options)) end defp call_plug(conn, plug) when is_function(plug, 1) do plug.(conn) end defp call_plug(conn, plug) when is_function(plug, 2) do plug.(conn, []) end else def run_plug(_request) do Logger.error(""" Could not find plug dependency. Please add :plug to your dependencies: {:plug, "~> 1.0"} """) raise "missing plug dependency" end end @doc """ Sets expected response body checksum. ## Request Options * `:checksum` - if set, this is the expected response body checksum. Can be one of: * `"md5:(...)"` * `"sha1:(...)"` * `"sha256:(...)"` ## Examples iex> resp = Req.get!("https://httpbin.org/json", checksum: "sha1:9274ffd9cf273d4a008750f44540c4c5d4c8227c") iex> resp.status 200 iex> Req.get!("https://httpbin.org/json", checksum: "sha1:bad") ** (Req.ChecksumMismatchError) checksum mismatch expected: sha1:bad actual: sha1:9274ffd9cf273d4a008750f44540c4c5d4c8227c """ @doc step: :request def checksum(request) do case Req.Request.get_option(request, :checksum) do nil -> request checksum when is_binary(checksum) -> type = checksum_type(checksum) case request.into do nil -> Req.Request.put_private(request, :req_checksum, %{ type: type, expected: checksum, hash: :body }) fun when is_function(fun, 2) -> hash = hash_init(type) into = fn {:data, chunk}, {req, resp} -> req = update_in(req.private.req_checksum.hash, &:crypto.hash_update(&1, chunk)) fun.({:data, chunk}, {req, resp}) end request |> Req.Request.put_private(:req_checksum, %{ type: type, expected: checksum, hash: hash }) |> Map.replace!(:into, into) :self -> raise ArgumentError, ":checksum cannot be used with `into: :self`" collectable -> into = Req.Utils.collect_with_hash(collectable, type) request |> Req.Request.put_private(:req_checksum, %{ type: type, expected: checksum, hash: :collectable }) |> Map.replace!(:into, into) end end end defp checksum_type("md5:" <> _), do: :md5 defp checksum_type("sha1:" <> _), do: :sha1 defp checksum_type("sha256:" <> _), do: :sha256 defp hash_init(:sha1), do: hash_init(:sha) defp hash_init(type), do: :crypto.hash_init(type) @doc """ Signs request with AWS Signature Version 4. ## Request Options * `:aws_sigv4` - if set, the AWS options to sign request: * `:access_key_id` - the AWS access key id. * `:secret_access_key` - the AWS secret access key. * `:token` - if set, the AWS security token, for example returned from AWS STS. * `:service` - the AWS service. We try to automatically detect the service (e.g. `s3.amazonaws.com` host sets service to `:s3`) * `:region` - the AWS region. Defaults to `"us-east-1"`. * `:datetime` - the request datetime, defaults to `DateTime.utc_now(:second)`. ## Examples iex> req = ...> Req.new( ...> base_url: "https://s3.amazonaws.com", ...> aws_sigv4: [ ...> access_key_id: System.get_env("AWS_ACCESS_KEY_ID"), ...> secret_access_key: System.get_env("AWS_SECRET_ACCESS_KEY") ...> ] ...> ) iex> iex> %{status: 200} = Req.put!(req, url: "/bucket1/key1", body: "Hello, World!") iex> resp = Req.get!(req, url: "/bucket1/key1").body "Hello, World!" Request body streaming also works though `content-length` header must be explicitly set: iex> path = "a.txt" iex> File.write!(path, String.duplicate("a", 100_000)) iex> size = File.stat!(path).size iex> chunk_size = 10 * 1024 iex> stream = File.stream!(path, chunk_size) iex> %{status: 200} = Req.put!(req, url: "/key1", headers: [content_length: size], body: stream) iex> byte_size(Req.get!(req, "/bucket1/key1").body) 100_000 """ @doc step: :request def put_aws_sigv4(request) do if aws_options = request.options[:aws_sigv4] do aws_options = case aws_options do list when is_list(list) -> list map when is_map(map) -> Enum.to_list(map) other -> raise ArgumentError, ":aws_sigv4 must be a keywords list or a map, got: #{inspect(other)}" end aws_options = aws_options |> Keyword.put_new(:region, "us-east-1") |> Keyword.put_new(:datetime, DateTime.utc_now()) # aws_credentials returns this key so let's ignore it |> Keyword.drop([:credential_provider]) Req.Request.validate_options(aws_options, [ :access_key_id, :secret_access_key, :token, :service, :region, :datetime, # for req_s3 :expires ]) unless aws_options[:access_key_id] do raise ArgumentError, "missing :access_key_id in :aws_sigv4 option" end unless aws_options[:secret_access_key] do raise ArgumentError, "missing :secret_access_key in :aws_sigv4 option" end aws_options = ensure_aws_service(aws_options, request.url) {body, options} = case request.body do nil -> {"", []} iodata when is_binary(iodata) or is_list(iodata) -> {iodata, []} _enumerable -> if Req.Request.get_header(request, "content-length") == [] do raise "content-length header must be explicitly set when streaming request body" end {"", [body_digest: "UNSIGNED-PAYLOAD"]} end request = Req.Request.put_new_header(request, "host", request.url.host) headers = for {name, values} <- request.headers, value <- values, do: {name, value} headers = Req.Utils.aws_sigv4_headers( aws_options ++ [ method: request.method, url: to_string(request.url), headers: headers, body: body ] ++ options ) Req.merge(request, headers: headers) else request end end defp ensure_aws_service(options, url) do if options[:service] do options else if service = detect_aws_service(url) do Keyword.put(options, :service, service) else raise ArgumentError, "missing :service in :aws_sigv4 option" end end end defp detect_aws_service(%URI{} = url) do parts = (url.host || "") |> String.split(".") |> Enum.reverse() with ["com", "amazonaws" | rest] <- parts do case rest do # s3 ["s3" | _] -> :s3 [_region, "s3" | _] -> :s3 # sqs ["sqs" | _] -> :sqs [_region, "sqs" | _] -> :sqs # ses ["email" | _] -> :ses [_region, "email" | _] -> :ses # iam ["iam"] -> :iam _ -> nil end else _ -> nil end end ## Response steps @doc """ Verifies the response body checksum. See `checksum/1` for more information. """ @doc step: :response def verify_checksum({request, response}) do if config = request.private[:req_checksum] do {response, hash} = case config.hash do # The most efficient way to do this would be to calculate checksum one chunk # at a time but it's not easy to implemenet. :body -> hash = hash_init(config.type) hash = :crypto.hash_update(hash, response.body) {response, :crypto.hash_final(hash)} :collectable -> {body, hash} = response.body {put_in(response.body, body), hash} hash -> {response, :crypto.hash_final(hash)} end actual = "#{config.type}:" <> Base.encode16(hash, case: :lower, padding: false) if config.expected == actual do request = Req.Request.delete_option(request, :req_checksum) {request, response} else exception = Req.ChecksumMismatchError.exception(expected: config.expected, actual: actual) {request, exception} end else {request, response} end end @doc """ Decompresses the response body based on the `content-encoding` header. This step is disabled on response body streaming. If response body is not a binary, in other words it has been transformed by another step, it is left as is. Supported formats: | Format | Decoder | | ------------- | ----------------------------------------------- | | gzip, x-gzip | `:zlib.gunzip/1` | | br | `:brotli.decode/1` (if [brotli] is installed) | | zstd | `:ezstd.decompress/1` (if [ezstd] is installed) | | _other_ | Returns data as is | This step updates the following headers to reflect the changes: * `content-encoding` is removed * `content-length` is removed ## Options * `:raw` - if set to `true`, disables response body decompression. Defaults to `false`. Note: setting `raw: true` also disables response body decoding in the `decode_body/1` step. ## Examples iex> response = Req.get!("https://httpbin.org/gzip") iex> response.body["gzipped"] true If the [brotli] package is installed, Brotli is also supported: Mix.install([ :req, {:brotli, "~> 0.3.0"} ]) response = Req.get!("https://httpbin.org/brotli") Req.Response.get_header(response, "content-encoding") #=> ["br"] response.body["brotli"] #=> true [brotli]: http://hex.pm/packages/brotli [ezstd]: https://hex.pm/packages/ezstd """ @doc step: :response def decompress_body(request_response) def decompress_body({request, %{body: body} = response}) when request.into != nil or body == "" or not is_binary(body) do {request, response} end def decompress_body({request, response}) do if request.options[:raw] do {request, response} else codecs = compression_algorithms(Req.Response.get_header(response, "content-encoding")) case decompress_body(codecs, response.body, []) do %Req.DecompressError{} = exception -> {request, exception} {decompressed_body, unknown_codecs} -> response = put_in(response.body, decompressed_body) response = if unknown_codecs == [] do response |> Req.Response.delete_header("content-encoding") |> Req.Response.delete_header("content-length") else Req.Response.put_header( response, "content-encoding", Enum.join(unknown_codecs, ", ") ) end {request, response} end end end defp decompress_body([gzip | rest], body, acc) when gzip in ["gzip", "x-gzip"] do try do decompress_body(rest, :zlib.gunzip(body), acc) rescue e in ErlangError -> if e.original == :data_error do %Req.DecompressError{format: :gzip, data: body} else reraise e, __STACKTRACE__ end end end defp decompress_body(["br" | rest], body, acc) do if brotli_loaded?() do case :brotli.decode(body) do {:ok, decompressed} -> decompress_body(rest, decompressed, acc) :error -> %Req.DecompressError{format: :br, data: body} end else Logger.debug(":brotli library not loaded, skipping brotli decompression") decompress_body(rest, body, ["br" | acc]) end end defp decompress_body(["zstd" | rest], body, acc) do if ezstd_loaded?() do case :ezstd.decompress(body) do decompressed when is_binary(decompressed) -> decompress_body(rest, decompressed, acc) {:error, reason} -> %Req.DecompressError{format: :zstd, data: body, reason: reason} end else Logger.debug(":ezstd library not loaded, skipping zstd decompression") decompress_body(rest, body, ["zstd" | acc]) end end defp decompress_body(["identity" | rest], body, acc) do decompress_body(rest, body, acc) end defp decompress_body([codec | rest], body, acc) do Logger.debug("algorithm #{inspect(codec)} is not supported") decompress_body(rest, body, [codec | acc]) end defp decompress_body([], body, acc) do {body, acc} end defp compression_algorithms(values) do values |> Enum.flat_map(fn value -> value |> String.downcase() |> String.split(",", trim: true) |> Enum.map(&String.trim/1) end) |> Enum.reverse() end defmacrop nimble_csv_loaded? do Code.ensure_loaded?(NimbleCSV) end @doc false def output(request_response) def output({request, response}) do output({request, response}, request.options[:output]) end defp output(request_response, nil) do request_response end defp output({request, response}, :remote_name) do path = Path.basename(request.url.path || "") output({request, response}, path) end defp output(_request_response, "") do raise "cannot write to file \"\"" end defp output({request, response}, path) do File.write!(path, response.body) response = %{response | body: ""} {request, response} end @doc """ Decodes response body based on the detected format. Supported formats: | Format | Decoder | | ------------ | ----------------------------------------------------------------- | | `json` | `Jason.decode/2` | | `tar`, `tgz` | `:erl_tar.extract/2` | | `zip` | `:zip.unzip/2` | | `gzip` | `:zlib.gunzip/1` | | `zst` | `:ezstd.decompress/1` (if [ezstd] is installed) | | `csv` | `NimbleCSV.RFC4180.parse_string/2` (if [nimble_csv] is installed) | The format is determined based on the `content-type` header of the response. For example, if the `content-type` is `application/json`, the response body is decoded as JSON. The built-in decoders also understand format extensions, such as decoding as JSON for a content-type of `application/vnd.api+json`. To do this, Req falls back to `MIME.extensions/1`; check the documentation for that function for more information. This step is disabled on response body streaming. If response body is not a binary, in other words it has been transformed by another step, it is left as is. ## Request Options * `:decode_body` - if set to `false`, disables automatic response body decoding. Defaults to `true`. * `:decode_json` - options to pass to `Jason.decode/2`, defaults to `[]`. * `:raw` - if set to `true`, disables response body decoding. Defaults to `false`. Note: setting `raw: true` also disables response body decompression in the `decompress_body/1` step. ## Examples Decode JSON: iex> response = Req.get!("https://httpbin.org/json") ...> response.body["slideshow"]["title"] "Sample Slide Show" Decode gzip: iex> response = Req.get!("https://httpbin.org/gzip") ...> response.body["gzipped"] true [nimble_csv]: https://hex.pm/packages/nimble_csv [ezstd]: https://hex.pm/packages/ezstd """ @doc step: :response def decode_body(request_response) def decode_body({request, %{body: body} = response}) when request.async != nil or body == "" or not is_binary(body) do {request, response} end def decode_body({request, response}) do # TODO: remove on Req 1.0 output? = request.options[:output] not in [nil, false] if request.options[:raw] == true or request.options[:decode_body] == false or output? or Req.Response.get_header(response, "content-encoding") != [] do {request, response} else decode_body({request, response}, format(request, response)) end end defp decode_body({request, response}, format) when format in ~w(json json-api) do options = Req.Request.get_option(request, :decode_json, []) case Jason.decode(response.body, options) do {:ok, decoded} -> {request, put_in(response.body, decoded)} {:error, e} -> {request, e} end end defp decode_body({request, response}, "tar") do case :erl_tar.extract({:binary, response.body}, [:memory]) do {:ok, files} -> {request, put_in(response.body, files)} {:error, reason} -> {request, %Req.ArchiveError{format: :tar, data: response.body, reason: reason}} end end defp decode_body({request, response}, "tgz") do case :erl_tar.extract({:binary, response.body}, [:memory, :compressed]) do {:ok, files} -> {request, put_in(response.body, files)} {:error, reason} -> {request, %Req.ArchiveError{format: :tar, data: response.body, reason: reason}} end end defp decode_body({request, response}, "zip") do case :zip.extract(response.body, [:memory]) do {:ok, files} -> {request, put_in(response.body, files)} {:error, _} -> {request, %Req.ArchiveError{format: :zip, data: response.body}} end end defp decode_body({request, response}, "gz") do {request, update_in(response.body, &:zlib.gunzip/1)} end defp decode_body({request, response}, "zst") do if ezstd_loaded?() do case :ezstd.decompress(response.body) do decompressed when is_binary(decompressed) -> {request, put_in(response.body, decompressed)} {:error, reason} -> err = %RuntimeError{message: "Could not decompress Zstandard data: #{inspect(reason)}"} {request, err} end else {request, response} end end defp decode_body({request, response}, "csv") do if nimble_csv_loaded?() do options = [skip_headers: false] {request, update_in(response.body, &NimbleCSV.RFC4180.parse_string(&1, options))} else {request, response} end end defp decode_body({request, response}, _format) do {request, response} end defp format(request, response) do path = request.url.path || "" case Req.Response.get_header(response, "content-type") do [content_type | _] -> case extensions(content_type, path) do [ext | _] -> ext [] -> nil end [] -> case extensions("application/octet-stream", path) do [ext | _] -> ext [] -> nil end end end defp extensions("application/octet-stream" <> _, path) do if tgz?(path) do ["tgz"] else path |> MIME.from_path() |> MIME.extensions() end end defp extensions("application/" <> subtype, path) when subtype in ~w(gzip x-gzip) do if tgz?(path) do ["tgz"] else ["gz"] end end defp extensions(content_type, _path) do MIME.extensions(content_type) end defp tgz?(path) do case Path.extname(path) do ".tgz" -> true ".gz" -> String.ends_with?(path, ".tar.gz") _ -> false end end @doc """ Follows redirects. The original request method may be changed to GET depending on the status code: | Code | Method handling | | ------------- | ------------------ | | 301, 302, 303 | Changed to GET | | 307, 308 | Method not changed | ## Request Options * `:redirect` - if set to `false`, disables automatic response redirects. Defaults to `true`. * `:redirect_trusted` - by default, authorization credentials are only sent on redirects with the same host, scheme and port. If `:redirect_trusted` is set to `true`, credentials will be sent to any host. * `:redirect_log_level` - the log level to emit redirect logs at. Can also be set to `false` to disable logging these messages. Defaults to `:debug`. * `:max_redirects` - the maximum number of redirects, defaults to `10`. If the limit is reached, the pipeline is halted and a `Req.TooManyRedirectsError` exception is returned. ## Examples iex> Req.get!("http://api.github.com").status # 23:24:11.670 [debug] redirecting to https://api.github.com/ 200 iex> Req.get!("https://httpbin.org/redirect/4", max_redirects: 3) # 23:07:59.570 [debug] redirecting to /relative-redirect/3 # 23:08:00.068 [debug] redirecting to /relative-redirect/2 # 23:08:00.206 [debug] redirecting to /relative-redirect/1 ** (RuntimeError) too many redirects (3) iex> Req.get!("http://api.github.com", redirect_log_level: false) 200 iex> Req.get!("http://api.github.com", redirect_log_level: :error) # 23:24:11.670 [error] redirecting to https://api.github.com/ 200 """ @doc step: :response def redirect(request_response) def redirect({request, response}) do redirect? = case Req.Request.fetch_option(request, :follow_redirects) do {:ok, redirect?} -> IO.warn(":follow_redirects option has been renamed to :redirect") redirect? :error -> Req.Request.get_option(request, :redirect, true) end with true <- redirect? && response.status in [301, 302, 303, 307, 308], [location | _] <- Req.Response.get_header(response, "location") do max_redirects = Req.Request.get_option(request, :max_redirects, 10) redirect_count = Req.Request.get_private(request, :req_redirect_count, 0) if redirect_count < max_redirects do with %Req.Response.Async{} <- response.body do Req.cancel_async_response(response) end request = request |> build_redirect_request(response, location) |> Req.Request.put_private(:req_redirect_count, redirect_count + 1) {request, response_or_exception} = Req.Request.run_request(request) Req.Request.halt(request, response_or_exception) else Req.Request.halt(request, %Req.TooManyRedirectsError{max_redirects: max_redirects}) end else _ -> {request, response} end end defp build_redirect_request(request, response, location) do log_level = Req.Request.get_option(request, :redirect_log_level, :debug) log_redirect(log_level, location) redirect_trusted = case Req.Request.fetch_option(request, :location_trusted) do {:ok, trusted} -> IO.warn(":location_trusted option has been renamed to :redirect_trusted") trusted :error -> request.options[:redirect_trusted] end location_url = request.url |> URI.merge(URI.parse(location)) |> normalize_redirect_uri() request # assume put_params step already run so remove :params option so it's not applied again |> Req.Request.delete_option(:params) |> remove_credentials_if_untrusted(redirect_trusted, location_url) |> put_redirect_method(response.status) |> Map.replace!(:url, location_url) end defp log_redirect(false, _location), do: :ok defp log_redirect(level, location) do Logger.log(level, ["redirecting to ", location]) end defp normalize_redirect_uri(%URI{scheme: "http", port: nil} = uri), do: %{uri | port: 80} defp normalize_redirect_uri(%URI{scheme: "https", port: nil} = uri), do: %{uri | port: 443} defp normalize_redirect_uri(%URI{} = uri), do: uri # https://www.rfc-editor.org/rfc/rfc9110#name-301-moved-permanently and 302: # # > Note: For historical reasons, a user agent MAY change the request method from # > POST to GET for the subsequent request. # # And my understanding is essentially same applies for 303. # Also see https://everything.curl.dev/http/redirects defp put_redirect_method(%{method: :post} = request, status) when status in 301..303 do %{request | method: :get} end defp put_redirect_method(request, _status) do request end defp remove_credentials_if_untrusted(request, true, _), do: request defp remove_credentials_if_untrusted(request, _, location_url) do if {location_url.host, location_url.scheme, location_url.port} == {request.url.host, request.url.scheme, request.url.port} do request else request |> Req.Request.delete_header("authorization") |> Req.Request.delete_option(:auth) end end @doc false @deprecated "Use Req.Steps.redirect/1 instead" def follow_redirects(request_response) do follow_redirects(request_response) end @doc """ Handles HTTP 4xx/5xx error responses. ## Request Options * `:http_errors` - how to handle HTTP 4xx/5xx error responses. Can be one of the following: * `:return` (default) - return the response * `:raise` - raise an error ## Examples iex> Req.get!("https://httpbin.org/status/404").status 404 iex> Req.get!("https://httpbin.org/status/404", http_errors: :raise) ** (RuntimeError) The requested URL returned error: 404 Response body: "" """ @doc step: :response def handle_http_errors(request_response) def handle_http_errors({request, response}) when response.status >= 400 do case Map.get(request.options, :http_errors, :return) do :return -> {request, response} :raise -> raise """ The requested URL returned error: #{response.status} Response body: #{inspect(response.body)}\ """ end end def handle_http_errors({request, response}) do {request, response} end ## Error steps @doc """ Retries a request in face of errors. This function can be used as either or both response and error step. ## Request Options * `:retry` - can be one of the following: * `:safe_transient` (default) - retry safe (GET/HEAD) requests on one of: * HTTP 408/429/500/502/503/504 responses * `Req.TransportError` with `reason: :timeout | :econnrefused | :closed` * `Req.HTTPError` with `protocol: :http2, reason: :unprocessed` * `:transient` - same as `:safe_transient` except retries all HTTP methods (POST, DELETE, etc.) * `fun` - a 2-arity function that accepts a `Req.Request` and either a `Req.Response` or an exception struct and returns one of the following: * `true` - retry with the default delay controller by default delay option described below. * `{:delay, milliseconds}` - retry with the given delay. * `false/nil` - don't retry. * `false` - don't retry. * `:retry_delay` - if not set, which is the default, the retry delay is determined by the value of the `Retry-After` header on HTTP 429/503 responses. If the header is not set, the default delay follows a simple exponential backoff: 1s, 2s, 4s, 8s, ... `:retry_delay` can be set to a function that receives the retry count (starting at 0) and returns the delay, the number of milliseconds to sleep before making another attempt. * `:retry_log_level` - the log level to emit retry logs at. Can also be set to `false` to disable logging these messages. Defaults to `:warning`. * `:max_retries` - maximum number of retry attempts, defaults to `3` (for a total of `4` requests to the server, including the initial one.) ## Examples With default options: iex> Req.get!("https://httpbin.org/status/500,200").status # 19:02:08.463 [warning] retry: got response with status 500, will retry in 2000ms, 2 attempts left # 19:02:10.710 [warning] retry: got response with status 500, will retry in 4000ms, 1 attempt left 200 Delay with jitter: iex> delay = fn n -> trunc(Integer.pow(2, n) * 1000 * (1 - 0.1 * :rand.uniform())) end iex> Req.get!("https://httpbin.org/status/500,200", retry_delay: delay).status # 08:43:19.101 [warning] retry: got response with status 500, will retry in 941ms, 2 attempts left # 08:43:22.958 [warning] retry: got response with status 500, will retry in 1877s, 1 attempt left 200 """ @doc step: :error def retry(request_response_or_error) def retry({request, response_or_exception}) do retry = case Map.get(request.options, :retry, :safe_transient) do :safe_transient -> request.method in [:get, :head] and transient?(response_or_exception) :transient -> transient?(response_or_exception) false -> false fun when is_function(fun) -> apply_retry(fun, request, response_or_exception) :safe -> IO.warn("setting `retry: :safe` is deprecated in favour of `retry: :safe_transient`") request.method in [:get, :head] and transient?(response_or_exception) :never -> IO.warn("setting `retry: :never` is deprecated in favour of `retry: false`") false other -> raise ArgumentError, "expected :retry to be :safe_transient, :transient, false, or a 2-arity function, " <> "got: #{inspect(other)}" end case retry do {:delay, delay} -> if !Req.Request.get_option(request, :retry_delay) do retry(request, response_or_exception, delay) else raise ArgumentError, "expected :retry_delay not to be set when the :retry function is returning `{:delay, milliseconds}`" end true -> retry(request, response_or_exception) retry when retry in [false, nil] -> {request, response_or_exception} end end defp apply_retry(fun, request, response_or_exception) defp apply_retry(fun, _request, response_or_exception) when is_function(fun, 1) do IO.warn("`retry: fun/1` has been deprecated in favor of `retry: fun/2`") fun.(response_or_exception) end defp apply_retry(fun, request, response_or_exception) when is_function(fun, 2) do fun.(request, response_or_exception) end defp transient?(%Req.Response{status: status}) when status in [408, 429, 500, 502, 503, 504] do true end defp transient?(%Req.Response{}) do false end defp transient?(%Req.TransportError{reason: reason}) when reason in [:timeout, :econnrefused, :closed] do true end defp transient?(%Req.HTTPError{protocol: :http2, reason: :unprocessed}) do true end defp transient?(%{__exception__: true}) do false end defp retry(request, response_or_exception, delay_or_nil \\ nil) defp retry(request, response_or_exception, nil) do do_retry(request, response_or_exception, &get_retry_delay/3) end defp retry(request, response_or_exception, delay) when is_integer(delay) do do_retry(request, response_or_exception, fn request, _, _ -> {request, delay} end) end defp do_retry(request, response_or_exception, delay_getter) do retry_count = Req.Request.get_private(request, :req_retry_count, 0) {request, delay} = delay_getter.(request, response_or_exception, retry_count) max_retries = Req.Request.get_option(request, :max_retries, 3) log_level = Req.Request.get_option(request, :retry_log_level, :warning) if retry_count < max_retries do log_retry(response_or_exception, retry_count, max_retries, delay, log_level) Process.sleep(delay) request = Req.Request.put_private(request, :req_retry_count, retry_count + 1) {request, response_or_exception} = Req.Request.run_request(%{request | halted: false}) Req.Request.halt(request, response_or_exception) else {request, response_or_exception} end end defp get_retry_delay(request, %Req.Response{status: status} = response, retry_count) when status in [429, 503] do if delay = Req.Response.get_retry_after(response) do {request, delay} else calculate_retry_delay(request, retry_count) end end defp get_retry_delay(request, _response, retry_count) do calculate_retry_delay(request, retry_count) end defp calculate_retry_delay(request, retry_count) do case Req.Request.get_option(request, :retry_delay, &exp_backoff/1) do delay when is_integer(delay) -> {request, delay} fun when is_function(fun, 1) -> case fun.(retry_count) do delay when is_integer(delay) and delay >= 0 -> {request, delay} other -> raise ArgumentError, "expected :retry_delay function to return non-negative integer, got: #{inspect(other)}" end end end defp exp_backoff(n) do Integer.pow(2, n) * 1000 end defp log_retry(_, _, _, _, false), do: :ok defp log_retry(response_or_exception, retry_count, max_retries, delay, level) do retries_left = case max_retries - retry_count do 1 -> "1 attempt" n -> "#{n} attempts" end message = ["will retry in #{delay}ms, ", retries_left, " left"] case response_or_exception do %{__exception__: true} = exception -> Logger.log(level, [ "retry: got exception, ", message ]) Logger.log(level, [ "** (#{inspect(exception.__struct__)}) ", Exception.message(exception) ]) response -> Logger.log(level, ["retry: got response with status #{response.status}, ", message]) end end ## Utilities defp cache_path(cache_dir, request) do cache_key = Enum.join( [ request.url.host, Atom.to_string(request.method), :crypto.hash(:sha256, :erlang.term_to_binary(request.url)) |> Base.encode16(case: :lower) ], "-" ) Path.join(cache_dir, cache_key) end defp write_cache(path, response) do File.mkdir_p!(Path.dirname(path)) File.write!(path, :erlang.term_to_binary(response)) end defp load_cache(path) do path |> File.read!() |> :erlang.binary_to_term() end end