-- SPDX-FileCopyrightText: 2026 Alexandra de Wit
--
-- SPDX-License-Identifier: MIT

{- | The resilience harness around the advisory package read: a per-attempt timeout, bounded retry
with backoff, and a per-rule circuit breaker, attached by 'Ecluse.Core.Rules.prepare'.

Returned values settle success. Exhausted faults count once per run. Cancellation propagates
and releases an owned recovery probe without counting a source failure.
-}
module Ecluse.Core.Rules.Effectful (
    -- * The resilience policy
    Resilience (..),
    EffectfulConfig (..),
    defaultEffectfulConfig,
    newBreaker,

    -- * Running a read through it
    runResilient,
    ReadFault (..),
) where

import Control.Retry (retrying)
import Data.Time (NominalDiffTime, UTCTime)
import UnliftIO (timeout, tryAny)
import UnliftIO.Exception (mask)

import Ecluse.Core.Breaker (
    Attempt,
    Breaker,
    BreakerReporter,
    admit,
    initialBreaker,
    recordFailure,
    recordSuccess,
    release,
    reportBreakerChange,
 )
import Ecluse.Core.InFlight (guardInFlight)
import Ecluse.Core.Rules.Types
import Ecluse.Core.Supervision (delayListPolicy)
import Ecluse.Core.Text (displayExceptionT)

-- | The resilience policy around one advisory rule's reads. Each rule holds its own breaker state.
data Resilience = Resilience
    { Resilience -> EffectfulConfig
resConfig :: EffectfulConfig
    -- ^ The per-attempt timeout, retry budget\/backoff, and breaker threshold\/cooldown.
    , Resilience -> TVar Breaker
resBreaker :: TVar Breaker
    -- ^ This rule's circuit-breaker state, shared across requests.
    , Resilience -> BreakerReporter
resBreakerReporter :: BreakerReporter
    -- ^ Observes coarse transitions for @ecluse.rule.breaker.state@.
    , Resilience -> IO UTCTime
resClock :: IO UTCTime
    -- ^ Fresh clock readings for admission and failure commit, separate from 'ctxNow'.
    }

-- | Why the harness gave a read up. Each rule relying on it resolves this under its own alignment.
data ReadFault = ReadFault
    { ReadFault -> Transience
rfTransience :: Transience
    -- ^ Whether a retry may succeed, with the configured @Retry-After@ hint.
    , ReadFault -> Inability
rfReason :: Inability
    -- ^ The client-facing cause a decision carries.
    , ReadFault -> Inability
rfDetail :: Inability
    -- ^ The fault detail an operator reads in the outage report, never in a client message.
    }
    deriving stock (ReadFault -> ReadFault -> Bool
(ReadFault -> ReadFault -> Bool)
-> (ReadFault -> ReadFault -> Bool) -> Eq ReadFault
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: ReadFault -> ReadFault -> Bool
== :: ReadFault -> ReadFault -> Bool
$c/= :: ReadFault -> ReadFault -> Bool
/= :: ReadFault -> ReadFault -> Bool
Eq, Int -> ReadFault -> ShowS
[ReadFault] -> ShowS
ReadFault -> String
(Int -> ReadFault -> ShowS)
-> (ReadFault -> String)
-> ([ReadFault] -> ShowS)
-> Show ReadFault
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> ReadFault -> ShowS
showsPrec :: Int -> ReadFault -> ShowS
$cshow :: ReadFault -> String
show :: ReadFault -> String
$cshowList :: [ReadFault] -> ShowS
showList :: [ReadFault] -> ShowS
Show)

-- | Run one read under its 'Resilience' policy: the read's value, or why the harness gave it up.
runResilient :: Resilience -> IO a -> IO (Either ReadFault a)
runResilient :: forall a. Resilience -> IO a -> IO (Either ReadFault a)
runResilient Resilience
res IO a
act = ((forall a. IO a -> IO a) -> IO (Either ReadFault a))
-> IO (Either ReadFault a)
forall (m :: * -> *) b.
MonadUnliftIO m =>
((forall a. m a -> m a) -> m b) -> m b
mask (((forall a. IO a -> IO a) -> IO (Either ReadFault a))
 -> IO (Either ReadFault a))
-> ((forall a. IO a -> IO a) -> IO (Either ReadFault a))
-> IO (Either ReadFault a)
forall a b. (a -> b) -> a -> b
$ \forall a. IO a -> IO a
restore -> do
    now <- Resilience -> IO UTCTime
resClock Resilience
res
    (ticket, old, new) <- atomically (admitProbe res now)
    case ticket of
        Maybe Attempt
Nothing -> Either ReadFault a -> IO (Either ReadFault a)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (ReadFault -> Either ReadFault a
forall a b. a -> Either a b
Left (Transience -> Inability -> Inability -> ReadFault
ReadFault (EffectfulConfig -> Transience
transientCause (Resilience -> EffectfulConfig
resConfig Resilience
res)) Inability
SourceBreakerOpen Inability
SourceBreakerOpen))
        Just Attempt
admitted ->
            (IO (Either ReadFault a) -> IO (Either ReadFault a))
-> (SomeException -> IO ())
-> IO ()
-> IO (Either ReadFault a)
-> IO (Either ReadFault a)
forall a.
(IO a -> IO a) -> (SomeException -> IO ()) -> IO () -> IO a -> IO a
guardInFlight IO (Either ReadFault a) -> IO (Either ReadFault a)
forall a. IO a -> IO a
restore (IO () -> SomeException -> IO ()
forall a b. a -> b -> a
const IO ()
forall (f :: * -> *). Applicative f => f ()
pass) (Resilience -> Attempt -> IO ()
releaseProbe Resilience
res Attempt
admitted) (IO (Either ReadFault a) -> IO (Either ReadFault a))
-> IO (Either ReadFault a) -> IO (Either ReadFault a)
forall a b. (a -> b) -> a -> b
$ do
                BreakerReporter -> Breaker -> Breaker -> IO ()
reportBreakerChange (Resilience -> BreakerReporter
resBreakerReporter Resilience
res) Breaker
old Breaker
new
                result <- Resilience -> IO a -> IO (Either (Transience, Inability) a)
forall a.
Resilience -> IO a -> IO (Either (Transience, Inability) a)
attemptWithRetry Resilience
res IO a
act
                settledNow <- resClock res
                settleOutcome res admitted settledNow result

settleOutcome :: Resilience -> Attempt -> UTCTime -> Either (Transience, Inability) a -> IO (Either ReadFault a)
settleOutcome :: forall a.
Resilience
-> Attempt
-> UTCTime
-> Either (Transience, Inability) a
-> IO (Either ReadFault a)
settleOutcome Resilience
res Attempt
ticket UTCTime
now = \case
    Right a
value -> do
        Resilience -> (Breaker -> Breaker) -> IO ()
commitBreaker Resilience
res (Attempt -> Breaker -> Breaker
recordSuccess Attempt
ticket)
        Either ReadFault a -> IO (Either ReadFault a)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (a -> Either ReadFault a
forall a b. b -> Either a b
Right a
value)
    Left (Transience
transience, Inability
detail) -> do
        Resilience -> (Breaker -> Breaker) -> IO ()
commitBreaker Resilience
res (EffectfulConfig -> UTCTime -> Attempt -> Breaker -> Breaker
tripOnFailure (Resilience -> EffectfulConfig
resConfig Resilience
res) UTCTime
now Attempt
ticket)
        Either ReadFault a -> IO (Either ReadFault a)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (ReadFault -> Either ReadFault a
forall a b. a -> Either a b
Left (Transience -> Inability -> Inability -> ReadFault
ReadFault Transience
transience Inability
EvaluationFailed Inability
detail))

releaseProbe :: Resilience -> Attempt -> IO ()
releaseProbe :: Resilience -> Attempt -> IO ()
releaseProbe Resilience
res Attempt
ticket = STM () -> IO ()
forall (m :: * -> *) a. MonadIO m => STM a -> m a
atomically (TVar Breaker -> (Breaker -> Breaker) -> STM ()
forall a. TVar a -> (a -> a) -> STM ()
modifyTVar' (Resilience -> TVar Breaker
resBreaker Resilience
res) (Attempt -> Breaker -> Breaker
release Attempt
ticket))

-- Attempt the read under the per-attempt timeout until the retry budget is spent. Only a fault retries.
attemptWithRetry :: Resilience -> IO a -> IO (Either (Transience, Inability) a)
attemptWithRetry :: forall a.
Resilience -> IO a -> IO (Either (Transience, Inability) a)
attemptWithRetry Resilience
res IO a
act =
    RetryPolicyM IO
-> (RetryStatus -> Either (Transience, Inability) a -> IO Bool)
-> (RetryStatus -> IO (Either (Transience, Inability) a))
-> IO (Either (Transience, Inability) a)
forall (m :: * -> *) b.
MonadIO m =>
RetryPolicyM m
-> (RetryStatus -> b -> m Bool) -> (RetryStatus -> m b) -> m b
retrying ([Int] -> RetryPolicyM IO
forall (m :: * -> *). Monad m => [Int] -> RetryPolicyM m
delayListPolicy (EffectfulConfig -> [Int]
ecBackoff (Resilience -> EffectfulConfig
resConfig Resilience
res))) RetryStatus -> Either (Transience, Inability) a -> IO Bool
forall {f :: * -> *} {p} {a} {b}.
Applicative f =>
p -> Either a b -> f Bool
shouldRetry (\RetryStatus
_ -> Resilience -> IO a -> IO (Either (Transience, Inability) a)
forall a.
Resilience -> IO a -> IO (Either (Transience, Inability) a)
attemptOnce Resilience
res IO a
act)
  where
    shouldRetry :: p -> Either a b -> f Bool
shouldRetry p
_ = Bool -> f Bool
forall a. a -> f a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Bool -> f Bool) -> (Either a b -> Bool) -> Either a b -> f Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Either a b -> Bool
forall a b. Either a b -> Bool
isLeft

-- One attempt under the timeout. Only a throw or a timeout retries and feeds the breaker.
attemptOnce :: Resilience -> IO a -> IO (Either (Transience, Inability) a)
attemptOnce :: forall a.
Resilience -> IO a -> IO (Either (Transience, Inability) a)
attemptOnce Resilience
res IO a
act = do
    result <- IO (Maybe a) -> IO (Either SomeException (Maybe a))
forall (m :: * -> *) a.
MonadUnliftIO m =>
m a -> m (Either SomeException a)
tryAny (Int -> IO a -> IO (Maybe a)
forall (m :: * -> *) a.
MonadUnliftIO m =>
Int -> m a -> m (Maybe a)
timeout (EffectfulConfig -> Int
ecTimeout (Resilience -> EffectfulConfig
resConfig Resilience
res)) IO a
act)
    pure $ case result of
        Left SomeException
e -> (Transience, Inability) -> Either (Transience, Inability) a
forall a b. a -> Either a b
Left (Transience
transient, Text -> Inability
RuleThrew (SomeException -> Text
forall e. Exception e => e -> Text
displayExceptionT SomeException
e))
        Right Maybe a
Nothing -> (Transience, Inability) -> Either (Transience, Inability) a
forall a b. a -> Either a b
Left (Transience
transient, Inability
AttemptTimedOut)
        Right (Just a
value) -> a -> Either (Transience, Inability) a
forall a b. b -> Either a b
Right a
value
  where
    transient :: Transience
transient = EffectfulConfig -> Transience
transientCause (Resilience -> EffectfulConfig
resConfig Resilience
res)

transientCause :: EffectfulConfig -> Transience
transientCause :: EffectfulConfig -> Transience
transientCause EffectfulConfig
cfg = Maybe RetryAfter -> Transience
WillResolve (EffectfulConfig -> Maybe RetryAfter
ecRetryAfter EffectfulConfig
cfg)

admitProbe :: Resilience -> UTCTime -> STM (Maybe Attempt, Breaker, Breaker)
admitProbe :: Resilience -> UTCTime -> STM (Maybe Attempt, Breaker, Breaker)
admitProbe Resilience
res UTCTime
now = do
    old <- TVar Breaker -> STM Breaker
forall a. TVar a -> STM a
readTVar (Resilience -> TVar Breaker
resBreaker Resilience
res)
    let (ticket, new) = admit now old
    writeTVar (resBreaker res) new
    pure (ticket, old, new)

-- Reads the breaker before and after in one transaction, so the report reflects exactly the
-- transition that committed.
commitBreaker :: Resilience -> (Breaker -> Breaker) -> IO ()
commitBreaker :: Resilience -> (Breaker -> Breaker) -> IO ()
commitBreaker Resilience
res Breaker -> Breaker
step = do
    (old, new) <- STM (Breaker, Breaker) -> IO (Breaker, Breaker)
forall (m :: * -> *) a. MonadIO m => STM a -> m a
atomically (STM (Breaker, Breaker) -> IO (Breaker, Breaker))
-> STM (Breaker, Breaker) -> IO (Breaker, Breaker)
forall a b. (a -> b) -> a -> b
$ do
        st <- TVar Breaker -> STM Breaker
forall a. TVar a -> STM a
readTVar (Resilience -> TVar Breaker
resBreaker Resilience
res)
        let st' = Breaker -> Breaker
step Breaker
st
        writeTVar (resBreaker res) st'
        pure (st, st')
    reportBreakerChange (resBreakerReporter res) old new

tripOnFailure :: EffectfulConfig -> UTCTime -> Attempt -> Breaker -> Breaker
tripOnFailure :: EffectfulConfig -> UTCTime -> Attempt -> Breaker -> Breaker
tripOnFailure EffectfulConfig
cfg = Int -> NominalDiffTime -> UTCTime -> Attempt -> Breaker -> Breaker
recordFailure (EffectfulConfig -> Int
ecBreakerThreshold EffectfulConfig
cfg) (EffectfulConfig -> NominalDiffTime
ecBreakerCooldown EffectfulConfig
cfg)

-- | Per-rule limits. Breaker timing uses the fresh 'resClock', separate from request snapshots.
data EffectfulConfig = EffectfulConfig
    { EffectfulConfig -> Int
ecTimeout :: Int
    -- ^ Per-attempt timeout in microseconds. Expiry is a retryable fault.
    , EffectfulConfig -> [Int]
ecBackoff :: [Int]
    -- ^ Microsecond delay per retry. An empty list disables retries.
    , EffectfulConfig -> Int
ecBreakerThreshold :: Int
    -- ^ Consecutive exhausted reads that trip the breaker, one read per request.
    , EffectfulConfig -> NominalDiffTime
ecBreakerCooldown :: NominalDiffTime
    -- ^ Cooldown in seconds before one owned recovery run.
    , EffectfulConfig -> Maybe RetryAfter
ecRetryAfter :: Maybe RetryAfter
    -- ^ Optional client @Retry-After@ hint for a faulted evaluation.
    }

-- | Timeout 2s, retries after 100ms and 250ms, trip after five failed runs, cooldown 30s.
defaultEffectfulConfig :: EffectfulConfig
defaultEffectfulConfig :: EffectfulConfig
defaultEffectfulConfig =
    EffectfulConfig
        { ecTimeout :: Int
ecTimeout = Int
2_000_000
        , ecBackoff :: [Int]
ecBackoff = [Int
100_000, Int
250_000]
        , ecBreakerThreshold :: Int
ecBreakerThreshold = Int
5
        , ecBreakerCooldown :: NominalDiffTime
ecBreakerCooldown = NominalDiffTime
30
        , ecRetryAfter :: Maybe RetryAfter
ecRetryAfter = Maybe RetryAfter
forall a. Maybe a
Nothing
        }

-- | A fresh, healthy breaker (no failures recorded) in a new 'TVar'.
newBreaker :: IO (TVar Breaker)
newBreaker :: IO (TVar Breaker)
newBreaker = Breaker -> IO (TVar Breaker)
forall (m :: * -> *) a. MonadIO m => a -> m (TVar a)
newTVarIO Breaker
initialBreaker