AdaptiveBackfill (adaptive_backfill v0.2.1)

View Source

DSL for creating adaptive backfills with health checks.

Usage

defmodule MyBackfill do
  use AdaptiveBackfill

  single_operation :process_user do
    mode :sync
    health_checks [&check_db/0, &check_memory/0]

    handle fn health_check ->
      case health_check.() do
        :ok -> :done
        {:halt, reason} -> {:halt, reason}
      end
    end
  end

  batch_operation :process_batch, initial_state: 0 do
    mode :async
    health_checks [&check_db/0]

    handle_batch fn state ->
      if state < 100, do: {:ok, state + 1}, else: :done
    end
  end
end

# Run the backfills
MyBackfill.process_user()
MyBackfill.process_batch()

Summary

Functions

Defines a batch operation backfill.

Sets the batch size (informational, for user's handle_batch logic).

Sets the delay between batches in milliseconds.

Sets the handle function for single operations.

Sets the handle_batch function for batch operations.

Sets the health checks.

Sets the operation mode.

Sets the on_complete callback.

Sets the on_error callback (receives error and state).

Sets the on_success callback (called after each successful batch).

Run a backfill with options struct (non-DSL API).

Defines a single operation backfill.

Sets the telemetry event prefix (list of atoms).

Sets the timeout for each batch in milliseconds.

Functions

batch_operation(name, opts \\ [], list)

(macro)

Defines a batch operation backfill.

Options

  • :initial_state - The starting state for the batch processor
  • :mode - Either :sync or :async (default: :sync)
  • :health_checks - List of health check functions
  • :handle_batch - The batch handler function
  • :on_complete - Optional completion callback
  • :on_success - Optional success callback (called after each successful batch)
  • :on_error - Optional error callback (receives error and state)
  • :delay_between_batches - Milliseconds to wait between batches
  • :timeout - Milliseconds before timing out a batch
  • :batch_size - Number of items per batch (informational)
  • :telemetry_prefix - List of atoms for telemetry event prefix

Example

batch_operation :my_batch, initial_state: 0 do
  mode :async
  health_checks [&check_health/0]
  handle_batch fn state -> {:ok, state + 1} end
  delay_between_batches 1000
  timeout 30_000
  on_success fn state -> Logger.info("Batch succeeded") end
  on_error fn error, state -> Logger.error("Batch failed") end
  telemetry_prefix [:my_app, :backfill]
end

batch_size(size)

(macro)

Sets the batch size (informational, for user's handle_batch logic).

delay_between_batches(ms)

(macro)

Sets the delay between batches in milliseconds.

handle(func)

(macro)

Sets the handle function for single operations.

handle_batch(func)

(macro)

Sets the handle_batch function for batch operations.

health_checks(checks)

(macro)

Sets the health checks.

mode(value)

(macro)

Sets the operation mode.

on_complete(func)

(macro)

Sets the on_complete callback.

on_error(func)

(macro)

Sets the on_error callback (receives error and state).

on_success(func)

(macro)

Sets the on_success callback (called after each successful batch).

run(opts)

Run a backfill with options struct (non-DSL API).

single_operation(name, list)

(macro)

Defines a single operation backfill.

Options

  • :mode - Either :sync or :async (default: :sync)
  • :health_checks - List of health check functions
  • :handle - The operation handler function
  • :on_complete - Optional completion callback
  • :on_success - Optional success callback
  • :on_error - Optional error callback
  • :timeout - Optional timeout in milliseconds
  • :telemetry_prefix - Optional telemetry event prefix (list of atoms)

Example

single_operation :my_operation do
  mode :sync
  health_checks [&check_health/0]
  timeout 30_000
  telemetry_prefix [:my_app, :operation]

  handle fn health_check -> :done end

  on_success fn result ->
    Logger.info("Operation succeeded")
  end

  on_error fn error ->
    Logger.error("Operation failed")
  end
end

telemetry_prefix(prefix)

(macro)

Sets the telemetry event prefix (list of atoms).

timeout(ms)

(macro)

Sets the timeout for each batch in milliseconds.