module Ecluse.Core.Rules.Effectful (
Resilience (..),
EffectfulConfig (..),
defaultEffectfulConfig,
newBreaker,
runResilient,
ReadFault (..),
) where
import Control.Retry (retrying)
import Data.Time (NominalDiffTime, UTCTime)
import UnliftIO (timeout, tryAny)
import UnliftIO.Exception (mask)
import Ecluse.Core.Breaker (
Attempt,
Breaker,
BreakerReporter,
admit,
initialBreaker,
recordFailure,
recordSuccess,
release,
reportBreakerChange,
)
import Ecluse.Core.InFlight (guardInFlight)
import Ecluse.Core.Rules.Types
import Ecluse.Core.Supervision (delayListPolicy)
import Ecluse.Core.Text (displayExceptionT)
data Resilience = Resilience
{ Resilience -> EffectfulConfig
resConfig :: EffectfulConfig
, Resilience -> TVar Breaker
resBreaker :: TVar Breaker
, Resilience -> BreakerReporter
resBreakerReporter :: BreakerReporter
, Resilience -> IO UTCTime
resClock :: IO UTCTime
}
data ReadFault = ReadFault
{ ReadFault -> Transience
rfTransience :: Transience
, ReadFault -> Inability
rfReason :: Inability
, ReadFault -> Inability
rfDetail :: Inability
}
deriving stock (ReadFault -> ReadFault -> Bool
(ReadFault -> ReadFault -> Bool)
-> (ReadFault -> ReadFault -> Bool) -> Eq ReadFault
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: ReadFault -> ReadFault -> Bool
== :: ReadFault -> ReadFault -> Bool
$c/= :: ReadFault -> ReadFault -> Bool
/= :: ReadFault -> ReadFault -> Bool
Eq, Int -> ReadFault -> ShowS
[ReadFault] -> ShowS
ReadFault -> String
(Int -> ReadFault -> ShowS)
-> (ReadFault -> String)
-> ([ReadFault] -> ShowS)
-> Show ReadFault
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> ReadFault -> ShowS
showsPrec :: Int -> ReadFault -> ShowS
$cshow :: ReadFault -> String
show :: ReadFault -> String
$cshowList :: [ReadFault] -> ShowS
showList :: [ReadFault] -> ShowS
Show)
runResilient :: Resilience -> IO a -> IO (Either ReadFault a)
runResilient :: forall a. Resilience -> IO a -> IO (Either ReadFault a)
runResilient Resilience
res IO a
act = ((forall a. IO a -> IO a) -> IO (Either ReadFault a))
-> IO (Either ReadFault a)
forall (m :: * -> *) b.
MonadUnliftIO m =>
((forall a. m a -> m a) -> m b) -> m b
mask (((forall a. IO a -> IO a) -> IO (Either ReadFault a))
-> IO (Either ReadFault a))
-> ((forall a. IO a -> IO a) -> IO (Either ReadFault a))
-> IO (Either ReadFault a)
forall a b. (a -> b) -> a -> b
$ \forall a. IO a -> IO a
restore -> do
now <- Resilience -> IO UTCTime
resClock Resilience
res
(ticket, old, new) <- atomically (admitProbe res now)
case ticket of
Maybe Attempt
Nothing -> Either ReadFault a -> IO (Either ReadFault a)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (ReadFault -> Either ReadFault a
forall a b. a -> Either a b
Left (Transience -> Inability -> Inability -> ReadFault
ReadFault (EffectfulConfig -> Transience
transientCause (Resilience -> EffectfulConfig
resConfig Resilience
res)) Inability
SourceBreakerOpen Inability
SourceBreakerOpen))
Just Attempt
admitted ->
(IO (Either ReadFault a) -> IO (Either ReadFault a))
-> (SomeException -> IO ())
-> IO ()
-> IO (Either ReadFault a)
-> IO (Either ReadFault a)
forall a.
(IO a -> IO a) -> (SomeException -> IO ()) -> IO () -> IO a -> IO a
guardInFlight IO (Either ReadFault a) -> IO (Either ReadFault a)
forall a. IO a -> IO a
restore (IO () -> SomeException -> IO ()
forall a b. a -> b -> a
const IO ()
forall (f :: * -> *). Applicative f => f ()
pass) (Resilience -> Attempt -> IO ()
releaseProbe Resilience
res Attempt
admitted) (IO (Either ReadFault a) -> IO (Either ReadFault a))
-> IO (Either ReadFault a) -> IO (Either ReadFault a)
forall a b. (a -> b) -> a -> b
$ do
BreakerReporter -> Breaker -> Breaker -> IO ()
reportBreakerChange (Resilience -> BreakerReporter
resBreakerReporter Resilience
res) Breaker
old Breaker
new
result <- Resilience -> IO a -> IO (Either (Transience, Inability) a)
forall a.
Resilience -> IO a -> IO (Either (Transience, Inability) a)
attemptWithRetry Resilience
res IO a
act
settledNow <- resClock res
settleOutcome res admitted settledNow result
settleOutcome :: Resilience -> Attempt -> UTCTime -> Either (Transience, Inability) a -> IO (Either ReadFault a)
settleOutcome :: forall a.
Resilience
-> Attempt
-> UTCTime
-> Either (Transience, Inability) a
-> IO (Either ReadFault a)
settleOutcome Resilience
res Attempt
ticket UTCTime
now = \case
Right a
value -> do
Resilience -> (Breaker -> Breaker) -> IO ()
commitBreaker Resilience
res (Attempt -> Breaker -> Breaker
recordSuccess Attempt
ticket)
Either ReadFault a -> IO (Either ReadFault a)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (a -> Either ReadFault a
forall a b. b -> Either a b
Right a
value)
Left (Transience
transience, Inability
detail) -> do
Resilience -> (Breaker -> Breaker) -> IO ()
commitBreaker Resilience
res (EffectfulConfig -> UTCTime -> Attempt -> Breaker -> Breaker
tripOnFailure (Resilience -> EffectfulConfig
resConfig Resilience
res) UTCTime
now Attempt
ticket)
Either ReadFault a -> IO (Either ReadFault a)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (ReadFault -> Either ReadFault a
forall a b. a -> Either a b
Left (Transience -> Inability -> Inability -> ReadFault
ReadFault Transience
transience Inability
EvaluationFailed Inability
detail))
releaseProbe :: Resilience -> Attempt -> IO ()
releaseProbe :: Resilience -> Attempt -> IO ()
releaseProbe Resilience
res Attempt
ticket = STM () -> IO ()
forall (m :: * -> *) a. MonadIO m => STM a -> m a
atomically (TVar Breaker -> (Breaker -> Breaker) -> STM ()
forall a. TVar a -> (a -> a) -> STM ()
modifyTVar' (Resilience -> TVar Breaker
resBreaker Resilience
res) (Attempt -> Breaker -> Breaker
release Attempt
ticket))
attemptWithRetry :: Resilience -> IO a -> IO (Either (Transience, Inability) a)
attemptWithRetry :: forall a.
Resilience -> IO a -> IO (Either (Transience, Inability) a)
attemptWithRetry Resilience
res IO a
act =
RetryPolicyM IO
-> (RetryStatus -> Either (Transience, Inability) a -> IO Bool)
-> (RetryStatus -> IO (Either (Transience, Inability) a))
-> IO (Either (Transience, Inability) a)
forall (m :: * -> *) b.
MonadIO m =>
RetryPolicyM m
-> (RetryStatus -> b -> m Bool) -> (RetryStatus -> m b) -> m b
retrying ([Int] -> RetryPolicyM IO
forall (m :: * -> *). Monad m => [Int] -> RetryPolicyM m
delayListPolicy (EffectfulConfig -> [Int]
ecBackoff (Resilience -> EffectfulConfig
resConfig Resilience
res))) RetryStatus -> Either (Transience, Inability) a -> IO Bool
forall {f :: * -> *} {p} {a} {b}.
Applicative f =>
p -> Either a b -> f Bool
shouldRetry (\RetryStatus
_ -> Resilience -> IO a -> IO (Either (Transience, Inability) a)
forall a.
Resilience -> IO a -> IO (Either (Transience, Inability) a)
attemptOnce Resilience
res IO a
act)
where
shouldRetry :: p -> Either a b -> f Bool
shouldRetry p
_ = Bool -> f Bool
forall a. a -> f a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Bool -> f Bool) -> (Either a b -> Bool) -> Either a b -> f Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Either a b -> Bool
forall a b. Either a b -> Bool
isLeft
attemptOnce :: Resilience -> IO a -> IO (Either (Transience, Inability) a)
attemptOnce :: forall a.
Resilience -> IO a -> IO (Either (Transience, Inability) a)
attemptOnce Resilience
res IO a
act = do
result <- IO (Maybe a) -> IO (Either SomeException (Maybe a))
forall (m :: * -> *) a.
MonadUnliftIO m =>
m a -> m (Either SomeException a)
tryAny (Int -> IO a -> IO (Maybe a)
forall (m :: * -> *) a.
MonadUnliftIO m =>
Int -> m a -> m (Maybe a)
timeout (EffectfulConfig -> Int
ecTimeout (Resilience -> EffectfulConfig
resConfig Resilience
res)) IO a
act)
pure $ case result of
Left SomeException
e -> (Transience, Inability) -> Either (Transience, Inability) a
forall a b. a -> Either a b
Left (Transience
transient, Text -> Inability
RuleThrew (SomeException -> Text
forall e. Exception e => e -> Text
displayExceptionT SomeException
e))
Right Maybe a
Nothing -> (Transience, Inability) -> Either (Transience, Inability) a
forall a b. a -> Either a b
Left (Transience
transient, Inability
AttemptTimedOut)
Right (Just a
value) -> a -> Either (Transience, Inability) a
forall a b. b -> Either a b
Right a
value
where
transient :: Transience
transient = EffectfulConfig -> Transience
transientCause (Resilience -> EffectfulConfig
resConfig Resilience
res)
transientCause :: EffectfulConfig -> Transience
transientCause :: EffectfulConfig -> Transience
transientCause EffectfulConfig
cfg = Maybe RetryAfter -> Transience
WillResolve (EffectfulConfig -> Maybe RetryAfter
ecRetryAfter EffectfulConfig
cfg)
admitProbe :: Resilience -> UTCTime -> STM (Maybe Attempt, Breaker, Breaker)
admitProbe :: Resilience -> UTCTime -> STM (Maybe Attempt, Breaker, Breaker)
admitProbe Resilience
res UTCTime
now = do
old <- TVar Breaker -> STM Breaker
forall a. TVar a -> STM a
readTVar (Resilience -> TVar Breaker
resBreaker Resilience
res)
let (ticket, new) = admit now old
writeTVar (resBreaker res) new
pure (ticket, old, new)
commitBreaker :: Resilience -> (Breaker -> Breaker) -> IO ()
commitBreaker :: Resilience -> (Breaker -> Breaker) -> IO ()
commitBreaker Resilience
res Breaker -> Breaker
step = do
(old, new) <- STM (Breaker, Breaker) -> IO (Breaker, Breaker)
forall (m :: * -> *) a. MonadIO m => STM a -> m a
atomically (STM (Breaker, Breaker) -> IO (Breaker, Breaker))
-> STM (Breaker, Breaker) -> IO (Breaker, Breaker)
forall a b. (a -> b) -> a -> b
$ do
st <- TVar Breaker -> STM Breaker
forall a. TVar a -> STM a
readTVar (Resilience -> TVar Breaker
resBreaker Resilience
res)
let st' = Breaker -> Breaker
step Breaker
st
writeTVar (resBreaker res) st'
pure (st, st')
reportBreakerChange (resBreakerReporter res) old new
tripOnFailure :: EffectfulConfig -> UTCTime -> Attempt -> Breaker -> Breaker
tripOnFailure :: EffectfulConfig -> UTCTime -> Attempt -> Breaker -> Breaker
tripOnFailure EffectfulConfig
cfg = Int -> NominalDiffTime -> UTCTime -> Attempt -> Breaker -> Breaker
recordFailure (EffectfulConfig -> Int
ecBreakerThreshold EffectfulConfig
cfg) (EffectfulConfig -> NominalDiffTime
ecBreakerCooldown EffectfulConfig
cfg)
data EffectfulConfig = EffectfulConfig
{ EffectfulConfig -> Int
ecTimeout :: Int
, EffectfulConfig -> [Int]
ecBackoff :: [Int]
, EffectfulConfig -> Int
ecBreakerThreshold :: Int
, EffectfulConfig -> NominalDiffTime
ecBreakerCooldown :: NominalDiffTime
, EffectfulConfig -> Maybe RetryAfter
ecRetryAfter :: Maybe RetryAfter
}
defaultEffectfulConfig :: EffectfulConfig
defaultEffectfulConfig :: EffectfulConfig
defaultEffectfulConfig =
EffectfulConfig
{ ecTimeout :: Int
ecTimeout = Int
2_000_000
, ecBackoff :: [Int]
ecBackoff = [Int
100_000, Int
250_000]
, ecBreakerThreshold :: Int
ecBreakerThreshold = Int
5
, ecBreakerCooldown :: NominalDiffTime
ecBreakerCooldown = NominalDiffTime
30
, ecRetryAfter :: Maybe RetryAfter
ecRetryAfter = Maybe RetryAfter
forall a. Maybe a
Nothing
}
newBreaker :: IO (TVar Breaker)
newBreaker :: IO (TVar Breaker)
newBreaker = Breaker -> IO (TVar Breaker)
forall (m :: * -> *) a. MonadIO m => a -> m (TVar a)
newTVarIO Breaker
initialBreaker