module Ecluse.Core.Supervision.Internal (
superviseLoop,
superviseLoopWithDelay,
superviseBackground,
superviseUnder,
SupervisionPolicy (..),
transientPolicy,
FaultDisposition (..),
BackoffSchedule (..),
backoffMicros,
backgroundLoopBackoff,
delayListPolicy,
) where
import Control.Retry (RetryPolicyM, RetryStatus (rsIterNumber), retryPolicy)
import Katip (KatipContext, LogEnv, Namespace (Namespace), Severity (ErrorS, WarningS), SimpleLogPayload, logFM, ls, runKatipContextT)
import UnliftIO (MonadUnliftIO)
import UnliftIO.Concurrent (threadDelay)
import UnliftIO.Exception (throwIO, tryAny)
import Ecluse.Core.Fault.Http (displayFaultT)
data FaultDisposition
=
Transient
|
Permanent
deriving stock (FaultDisposition -> FaultDisposition -> Bool
(FaultDisposition -> FaultDisposition -> Bool)
-> (FaultDisposition -> FaultDisposition -> Bool)
-> Eq FaultDisposition
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: FaultDisposition -> FaultDisposition -> Bool
== :: FaultDisposition -> FaultDisposition -> Bool
$c/= :: FaultDisposition -> FaultDisposition -> Bool
/= :: FaultDisposition -> FaultDisposition -> Bool
Eq, Int -> FaultDisposition -> ShowS
[FaultDisposition] -> ShowS
FaultDisposition -> String
(Int -> FaultDisposition -> ShowS)
-> (FaultDisposition -> String)
-> ([FaultDisposition] -> ShowS)
-> Show FaultDisposition
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> FaultDisposition -> ShowS
showsPrec :: Int -> FaultDisposition -> ShowS
$cshow :: FaultDisposition -> String
show :: FaultDisposition -> String
$cshowList :: [FaultDisposition] -> ShowS
showList :: [FaultDisposition] -> ShowS
Show)
data BackoffSchedule = BackoffSchedule
{ BackoffSchedule -> Int
bsBaseMicros :: Int
, BackoffSchedule -> Int
bsCapMicros :: Int
}
deriving stock (BackoffSchedule -> BackoffSchedule -> Bool
(BackoffSchedule -> BackoffSchedule -> Bool)
-> (BackoffSchedule -> BackoffSchedule -> Bool)
-> Eq BackoffSchedule
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: BackoffSchedule -> BackoffSchedule -> Bool
== :: BackoffSchedule -> BackoffSchedule -> Bool
$c/= :: BackoffSchedule -> BackoffSchedule -> Bool
/= :: BackoffSchedule -> BackoffSchedule -> Bool
Eq, Int -> BackoffSchedule -> ShowS
[BackoffSchedule] -> ShowS
BackoffSchedule -> String
(Int -> BackoffSchedule -> ShowS)
-> (BackoffSchedule -> String)
-> ([BackoffSchedule] -> ShowS)
-> Show BackoffSchedule
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> BackoffSchedule -> ShowS
showsPrec :: Int -> BackoffSchedule -> ShowS
$cshow :: BackoffSchedule -> String
show :: BackoffSchedule -> String
$cshowList :: [BackoffSchedule] -> ShowS
showList :: [BackoffSchedule] -> ShowS
Show)
backoffMicros :: BackoffSchedule -> Int -> Int
backoffMicros :: BackoffSchedule -> Int -> Int
backoffMicros BackoffSchedule
schedule Int
consecutiveFailures =
Int -> Int -> Int
forall a. Ord a => a -> a -> a
min (BackoffSchedule -> Int
bsCapMicros BackoffSchedule
schedule) (BackoffSchedule -> Int
bsBaseMicros BackoffSchedule
schedule Int -> Int -> Int
forall a. Num a => a -> a -> a
* (Int
2 Int -> Int -> Int
forall a b. (Num a, Integral b) => a -> b -> a
^ Int -> Int -> Int
forall a. Ord a => a -> a -> a
min Int
consecutiveFailures Int
backoffShiftClamp))
backoffShiftClamp :: Int
backoffShiftClamp :: Int
backoffShiftClamp = Int
12
data SupervisionPolicy = SupervisionPolicy
{ SupervisionPolicy -> Text
spLabel :: Text
, SupervisionPolicy -> SomeException -> FaultDisposition
spClassify :: SomeException -> FaultDisposition
, SupervisionPolicy -> BackoffSchedule
spBackoff :: BackoffSchedule
}
transientPolicy :: Text -> BackoffSchedule -> SupervisionPolicy
transientPolicy :: Text -> BackoffSchedule -> SupervisionPolicy
transientPolicy Text
label BackoffSchedule
schedule =
SupervisionPolicy
{ spLabel :: Text
spLabel = Text
label
, spClassify :: SomeException -> FaultDisposition
spClassify = FaultDisposition -> SomeException -> FaultDisposition
forall a b. a -> b -> a
const FaultDisposition
Transient
, spBackoff :: BackoffSchedule
spBackoff = BackoffSchedule
schedule
}
superviseLoop :: (MonadUnliftIO m, KatipContext m) => SupervisionPolicy -> m () -> m Void
superviseLoop :: forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
SupervisionPolicy -> m () -> m Void
superviseLoop = (Int -> m ()) -> SupervisionPolicy -> m () -> m Void
forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
(Int -> m ()) -> SupervisionPolicy -> m () -> m Void
superviseLoopWithDelay Int -> m ()
forall (m :: * -> *). MonadIO m => Int -> m ()
threadDelay
superviseLoopWithDelay :: (MonadUnliftIO m, KatipContext m) => (Int -> m ()) -> SupervisionPolicy -> m () -> m Void
superviseLoopWithDelay :: forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
(Int -> m ()) -> SupervisionPolicy -> m () -> m Void
superviseLoopWithDelay Int -> m ()
delayAction SupervisionPolicy
policy m ()
step = Int -> m Void
forall {b}. Int -> m b
go Int
0
where
go :: Int -> m b
go Int
consecutiveFaults =
m () -> m (Either SomeException ())
forall (m :: * -> *) a.
MonadUnliftIO m =>
m a -> m (Either SomeException a)
tryAny m ()
step m (Either SomeException ())
-> (Either SomeException () -> m b) -> m b
forall a b. m a -> (a -> m b) -> m b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= \case
Right () -> Int -> m b
go Int
0
Left SomeException
fault -> case SupervisionPolicy -> SomeException -> FaultDisposition
spClassify SupervisionPolicy
policy SomeException
fault of
FaultDisposition
Permanent -> do
Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (SupervisionPolicy -> Text
spLabel SupervisionPolicy
policy Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": permanent fault, failing up: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> SomeException -> Text
displayFaultT SomeException
fault))
SomeException -> m b
forall (m :: * -> *) e a. (MonadIO m, Exception e) => e -> m a
throwIO SomeException
fault
FaultDisposition
Transient -> (Int -> m ()) -> SupervisionPolicy -> Int -> SomeException -> m ()
forall (m :: * -> *).
KatipContext m =>
(Int -> m ()) -> SupervisionPolicy -> Int -> SomeException -> m ()
warnAndBackOff Int -> m ()
delayAction SupervisionPolicy
policy Int
consecutiveFaults SomeException
fault m () -> m b -> m b
forall a b. m a -> m b -> m b
forall (m :: * -> *) a b. Monad m => m a -> m b -> m b
>> Int -> m b
go (Int
consecutiveFaults Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1)
superviseBackground :: Text -> LogEnv -> IO () -> IO ()
superviseBackground :: Text -> LogEnv -> IO () -> IO ()
superviseBackground Text
label = SupervisionPolicy -> LogEnv -> IO () -> IO ()
superviseUnder (Text -> BackoffSchedule -> SupervisionPolicy
transientPolicy Text
label BackoffSchedule
backgroundLoopBackoff)
superviseUnder :: SupervisionPolicy -> LogEnv -> IO () -> IO ()
superviseUnder :: SupervisionPolicy -> LogEnv -> IO () -> IO ()
superviseUnder SupervisionPolicy
policy LogEnv
logEnv IO ()
task =
IO Void -> IO ()
forall (f :: * -> *) a. Functor f => f a -> f ()
void (IO Void -> IO ())
-> (KatipContextT IO Void -> IO Void)
-> KatipContextT IO Void
-> IO ()
forall b c a. (b -> c) -> (a -> b) -> a -> c
. LogEnv
-> SimpleLogPayload
-> Namespace
-> KatipContextT IO Void
-> IO Void
forall c (m :: * -> *) a.
LogItem c =>
LogEnv -> c -> Namespace -> KatipContextT m a -> m a
runKatipContextT LogEnv
logEnv (SimpleLogPayload
forall a. Monoid a => a
mempty :: SimpleLogPayload) ([Text] -> Namespace
Namespace [SupervisionPolicy -> Text
spLabel SupervisionPolicy
policy]) (KatipContextT IO Void -> IO ()) -> KatipContextT IO Void -> IO ()
forall a b. (a -> b) -> a -> b
$
SupervisionPolicy -> KatipContextT IO () -> KatipContextT IO Void
forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
SupervisionPolicy -> m () -> m Void
superviseLoop SupervisionPolicy
policy (IO () -> KatipContextT IO ()
forall a. IO a -> KatipContextT IO a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO IO ()
task)
warnAndBackOff :: (KatipContext m) => (Int -> m ()) -> SupervisionPolicy -> Int -> SomeException -> m ()
warnAndBackOff :: forall (m :: * -> *).
KatipContext m =>
(Int -> m ()) -> SupervisionPolicy -> Int -> SomeException -> m ()
warnAndBackOff Int -> m ()
delayAction SupervisionPolicy
policy Int
consecutiveFaults SomeException
fault = do
let delay :: Int
delay = BackoffSchedule -> Int -> Int
backoffMicros (SupervisionPolicy -> BackoffSchedule
spBackoff SupervisionPolicy
policy) Int
consecutiveFaults
Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
WarningS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (SupervisionPolicy -> Text
spLabel SupervisionPolicy
policy Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": iteration faulted (retrying in " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Int -> Text
forall b a. (Show a, IsString b) => a -> b
show Int
delay Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"µs): " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> SomeException -> Text
displayFaultT SomeException
fault))
Int -> m ()
delayAction Int
delay
backgroundLoopBackoff :: BackoffSchedule
backgroundLoopBackoff :: BackoffSchedule
backgroundLoopBackoff = BackoffSchedule{bsBaseMicros :: Int
bsBaseMicros = Int
1_000_000, bsCapMicros :: Int
bsCapMicros = Int
30_000_000}
delayListPolicy :: (Monad m) => [Int] -> RetryPolicyM m
delayListPolicy :: forall (m :: * -> *). Monad m => [Int] -> RetryPolicyM m
delayListPolicy [Int]
delays = (RetryStatus -> Maybe Int) -> RetryPolicyM m
forall (m :: * -> *).
Monad m =>
(RetryStatus -> Maybe Int) -> RetryPolicyM m
retryPolicy (\RetryStatus
rs -> [Int]
delays [Int] -> Int -> Maybe Int
forall a. [a] -> Int -> Maybe a
!!? RetryStatus -> Int
rsIterNumber RetryStatus
rs)