Skip to content

Commit

Permalink
add time groupping
Browse files Browse the repository at this point in the history
  • Loading branch information
joaquinco committed Sep 18, 2024
1 parent 64f07d6 commit bd9f657
Show file tree
Hide file tree
Showing 6 changed files with 222 additions and 105 deletions.
61 changes: 41 additions & 20 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,34 +91,38 @@ defmodule YourApp.Router do
options: # ...
]
]

#...
end
```

## Notification Trigger
## Throttling notifications

By default, `BoomNotifier` will send a notification every time an exception is
raised.

However, there are different strategies to decide when to send the
notifications using the `:notification_trigger` option with one of the
following values: `:always` and `:exponential`.

### Always
There are two throttling mechanisms you can use to reduce notification rate. Counting
based and time based throttling. The former allows notifications to be delivered
on exponential error counting and the latter based on time throttling, that is
if a predefined amount of time has passed from the last error.

This option is the default one. It will trigger a notification for every
exception.
### Count Based Throttle

```elixir
defmodule YourApp.Router do
use Phoenix.Router

use BoomNotifier,
notification_trigger: :always,
notifiers: [
# ...
]
# ...
],
groupping: :count,
count: :exponential,
time_limit: :timer.minutes(30)
end
```

### Exponential
#### Exponential

It uses a formula of `log2(errors_count)` to determine whether to send a
notification, based on the accumulated error count for each specific
Expand All @@ -132,7 +136,7 @@ defmodule YourApp.Router do
use Phoenix.Router

use BoomNotifier,
notification_trigger: :exponential,
count: :exponential,
notifiers: [
# ...
]
Expand All @@ -143,32 +147,49 @@ defmodule YourApp.Router do
use Phoenix.Router

use BoomNotifier,
notification_trigger: [exponential: [limit: 100]]
count: [exponential: [limit: 100]]
notifiers: [
# ...
]
```

### Notification trigger time limit
### Time Based Throttle

```elixir
defmodule YourApp.Router do
use Phoenix.Router

use BoomNotifier,
notifiers: [
# ...
],
groupping: :time,
throttle: :timer.minutes(1)
end
```

### Time Limit

If you've defined a triggering strategy which holds off notification delivering you can define a time limit value
which will be used to deliver the notification after a time limit milliseconds have passed from the last error. The
time counter is reset on new errors and only applies for cases where notifications are not sent.
Both groupping strategies allow you to define a time limit that tells boom to deliver a notification if the amount
of time has passed from the last time a notification was sent or error groupping started.

If specified, a notification will be triggered even though the groupping strategy does not met the criteria. For
time based throttling this is usefull if errors keep appearing before the throttle time and for count based throttle
to reset (and notify) the grupping after a certain time period.

```elixir
defmodule YourApp.Router do
use Phoenix.Router

use BoomNotifier,
notification_trigger: [:exponential],
count: [:exponential],
time_limit: :timer.minutes(30),
notifier: CustomNotifier

# ...
end
```


## Custom data or Metadata

By default, `BoomNotifier` will **not** include any custom data from your
Expand Down
25 changes: 18 additions & 7 deletions lib/boom_notifier/error_storage.ex
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ defmodule BoomNotifier.ErrorStorage do
]

@type t :: %__MODULE__{}
@type error_strategy :: :always | :exponential | [exponential: [limit: non_neg_integer()]]
@type error_strategy ::
:always | :none | :exponential | [exponential: [limit: non_neg_integer()]]

use Agent, start: {__MODULE__, :start_link, []}

Expand All @@ -32,8 +33,8 @@ defmodule BoomNotifier.ErrorStorage do
occurrences is increased and it also updates the first and last time it
happened.
"""
@spec store_error(ErrorInfo.t()) :: :ok
def store_error(error_info) do
@spec accumulate(ErrorInfo.t()) :: :ok
def accumulate(error_info) do
%{key: error_hash_key} = error_info
timestamp = error_info.timestamp || DateTime.utc_now()

Expand Down Expand Up @@ -65,8 +66,8 @@ defmodule BoomNotifier.ErrorStorage do
@doc """
Given an error info, it returns the aggregated info stored in the agent.
"""
@spec get_error_stats(ErrorInfo.t()) :: %__MODULE__{}
def get_error_stats(error_info) do
@spec get_stats(ErrorInfo.t()) :: %__MODULE__{}
def get_stats(error_info) do
%{key: error_hash_key} = error_info

Agent.get(:boom_notifier, fn state -> state end)
Expand Down Expand Up @@ -94,7 +95,7 @@ defmodule BoomNotifier.ErrorStorage do
Reset the accumulated_occurrences for the given error info to zero. It also
increments the max storage capacity based on the notification strategy.
"""
@spec reset_accumulated_errors(error_strategy, ErrorInfo.t()) :: :ok
@spec reset_accumulated_errors(error_strategy | nil, ErrorInfo.t()) :: :ok
def reset_accumulated_errors(:exponential, error_info) do
%{key: error_hash_key} = error_info

Expand All @@ -119,7 +120,7 @@ defmodule BoomNotifier.ErrorStorage do
)
end

def reset_accumulated_errors(:always, error_info) do
def reset_accumulated_errors(value, error_info) when value in [nil, :none, :always] do
%{key: error_hash_key} = error_info

Agent.update(
Expand All @@ -138,6 +139,16 @@ defmodule BoomNotifier.ErrorStorage do
|> Map.replace!(:last_occurrence, nil)
end

def eleapsed(nil), do: 0

def eleapsed(%__MODULE__{} = error_info) do
DateTime.diff(
error_info.last_occurrence,
error_info.first_occurrence,
:millisecond
)
end

@spec do_send_notification?(ErrorInfo.t() | nil) :: boolean()
defp do_send_notification?(nil), do: false

Expand Down
71 changes: 47 additions & 24 deletions lib/boom_notifier/notification_sender.ex
Original file line number Diff line number Diff line change
Expand Up @@ -14,49 +14,49 @@ defmodule BoomNotifier.NotificationSender do
GenServer.start_link(__MODULE__, :ok, name: __MODULE__)
end

def async_notify(notifier, occurrences, options) do
GenServer.cast(__MODULE__, {:notify, notifier, occurrences, options})
end

def async_trigger_notify(settings, error_info) do
GenServer.cast(__MODULE__, {:trigger_notify, settings, error_info})
end

def notify(notifier, occurrences, options) do
spawn_link(fn ->
notifier.notify(occurrences, options)
end)
end

def notify_all(settings, error_info) do
notification_trigger = Keyword.get(settings, :notification_trigger, :always)
occurrences = Map.put(error_info, :occurrences, ErrorStorage.get_error_stats(error_info))

ErrorStorage.reset_accumulated_errors(notification_trigger, error_info)
def trigger_notify(settings, error_info) do
ErrorStorage.accumulate(error_info)

BoomNotifier.walkthrough_notifiers(
do_trigger_notify(
Keyword.get(settings, :groupping, :count),
settings,
fn notifier, options -> notify(notifier, occurrences, options) end
error_info
)
end

def trigger_notify(settings, error_info) do
timeout = Keyword.get(settings, :time_limit)

ErrorStorage.store_error(error_info)
defp do_trigger_notify(:count, settings, error_info) do
time_limit = Keyword.get(settings, :time_limit)

if ErrorStorage.send_notification?(error_info) do
notify_all(settings, error_info)
:ok
else
if timeout do
{:schedule, timeout}
if time_limit do
{:schedule, time_limit}
else
:ok
:ignored
end
end
end

defp do_trigger_notify(:time, settings, error_info) do
throttle = Keyword.get(settings, :throttle, 100)
time_limit = Keyword.get(settings, :time_limit)

stats = ErrorStorage.get_stats(error_info)

if ErrorStorage.eleapsed(stats) >= time_limit do
notify_all(settings, error_info)
:ok
else
{:schedule, throttle}
end
end

# Server callbacks

@impl true
Expand All @@ -79,6 +79,9 @@ defmodule BoomNotifier.NotificationSender do
cancel_timer(timer)

case trigger_notify(settings, error_info) do
:ignored ->
{:noreply, state}

:ok ->
{:noreply, state}

Expand Down Expand Up @@ -116,6 +119,26 @@ defmodule BoomNotifier.NotificationSender do
{:noreply, state |> Map.delete(error_info.key)}
end

# Private methods

defp notify(notifier, occurrences, options) do
spawn_link(fn ->
notifier.notify(occurrences, options)
end)
end

defp notify_all(settings, error_info) do
count_strategy = Keyword.get(settings, :count)

occurrences = Map.put(error_info, :occurrences, ErrorStorage.get_stats(error_info))
ErrorStorage.reset_accumulated_errors(count_strategy, error_info)

BoomNotifier.walkthrough_notifiers(
settings,
fn notifier, options -> notify(notifier, occurrences, options) end
)
end

defp cancel_timer(nil), do: nil
defp cancel_timer(timer), do: Process.cancel_timer(timer)
end
2 changes: 1 addition & 1 deletion test/example_app/lib/example_app_web/router.ex
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ defmodule ExampleAppWeb.Router do
use ExampleAppWeb, :router

use BoomNotifier,
notification_trigger: [exponential: [limit: 8]],
count: [exponential: [limit: 8]],
custom_data: [:assigns, :logger],
ignore_exceptions: [IgnoreExceptionError],
notifiers: [
Expand Down
Loading

0 comments on commit bd9f657

Please sign in to comment.