module Ecluse.Runtime.Cve.Sync.Internal (
CveFetch (..),
FetchedObject (..),
DbEtag (..),
OsvDbFetchFault (..),
OsvDbCapExceeded (..),
S3CveSource,
newS3CveSource,
s3CveFetchFor,
s3CveFetch,
cappedAt,
SyncEnv (..),
SyncOutcome (..),
syncStep,
SyncSchedule (..),
SyncHooks (..),
runCveSync,
bootBackoffDelays,
absentReportInterval,
) where
import Data.List (lookup)
import Data.Text qualified as T
import Data.Time (UTCTime)
import Data.Time.Format.ISO8601 (iso8601ParseM)
import Katip (KatipContext, Severity (DebugS, ErrorS, InfoS), logFM, ls)
import UnliftIO (MonadUnliftIO, withRunInIO)
import UnliftIO.Concurrent (threadDelay)
import Ecluse.Core.Cve.Slot (AdvisorySource (..), currentAdvisoryEtag, currentAdvisorySource)
import Ecluse.Core.Cve.Types (renderOsvDbFetchFault)
import Ecluse.Core.Ecosystem (Ecosystem)
import Ecluse.Core.Osv.Provenance (AdvisoryProvenance (apEpssScoreDate, apOsvNewestModified, apOsvSource))
import Ecluse.Core.Osv.Schema (MetaKey (MetaBuiltAt, MetaRowCount), renderMetaKey)
import Ecluse.Core.Security.Authority (dialledAuthorityLabel)
import Ecluse.Core.Telemetry.Metrics (
AdvisorySyncResult (AdvisoryFetchFailed, AdvisoryNonePublished, AdvisoryRefused, AdvisorySwapped, AdvisoryUnchanged),
)
import Ecluse.Core.Telemetry.Record (AdvisorySyncMetricsPort (asmpSyncAttempt, asmpSyncDuration), timedSeconds)
import Ecluse.Core.Telemetry.Span (AdvisorySyncTracingPort (astpSyncAttemptSpan))
import Ecluse.Core.Text (readDecimalText)
import Ecluse.Core.Text.Iso8601 (renderIso8601Utc)
import Ecluse.Runtime.Cve.Sync.S3 (OsvDbCapExceeded (..), S3CveSource, cappedAt, newS3CveSource, s3CveFetch, s3CveFetchFor)
import Ecluse.Runtime.Cve.Sync.Step (CveFetch (..), DbEtag (..), FetchedObject (..), OsvDbFetchFault (..), SyncEnv (..), SyncOutcome (..), syncStep)
data SyncSchedule = SyncSchedule
{ SyncSchedule -> [Int]
schedBootBackoff :: [Int]
, SyncSchedule -> Int
schedPollDelay :: Int
, SyncSchedule -> Int
schedAbsentReport :: Int
}
bootBackoffDelays :: [Int]
bootBackoffDelays :: [Int]
bootBackoffDelays = [Int
1_000_000, Int
2_000_000, Int
4_000_000, Int
8_000_000, Int
16_000_000]
absentReportInterval :: Int
absentReportInterval :: Int
absentReportInterval = Int
900_000_000
data SyncHooks = SyncHooks
{ SyncHooks -> IO ()
hookFirstSync :: IO ()
, SyncHooks -> IO ()
hookPushAge :: IO ()
}
runCveSync ::
(MonadUnliftIO m, KatipContext m) =>
AdvisorySyncMetricsPort ->
AdvisorySyncTracingPort ->
SyncEnv ->
SyncSchedule ->
SyncHooks ->
m ()
runCveSync :: forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
AdvisorySyncMetricsPort
-> AdvisorySyncTracingPort
-> SyncEnv
-> SyncSchedule
-> SyncHooks
-> m ()
runCveSync AdvisorySyncMetricsPort
metrics AdvisorySyncTracingPort
tracing SyncEnv
env SyncSchedule
schedule SyncHooks
hooks =
SyncLoop -> Pacing -> Int -> [Int] -> m (Pacing, Maybe DbEtag)
forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
SyncLoop -> Pacing -> Int -> [Int] -> m (Pacing, Maybe DbEtag)
burstCycle SyncLoop
loop Pacing
initialPacing Int
0 (SyncSchedule -> [Int]
schedBootBackoff SyncSchedule
schedule) m (Pacing, Maybe DbEtag)
-> ((Pacing, Maybe DbEtag) -> m ()) -> m ()
forall a b. m a -> (a -> m b) -> m b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= (Pacing -> Maybe DbEtag -> m ()) -> (Pacing, Maybe DbEtag) -> m ()
forall a b c. (a -> b -> c) -> (a, b) -> c
uncurry (SyncLoop -> Pacing -> Maybe DbEtag -> m ()
forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
SyncLoop -> Pacing -> Maybe DbEtag -> m ()
pollCycle SyncLoop
loop)
where
loop :: SyncLoop
loop =
SyncLoop
{ slMetrics :: AdvisorySyncMetricsPort
slMetrics = AdvisorySyncMetricsPort
metrics
, slTracing :: AdvisorySyncTracingPort
slTracing = AdvisorySyncTracingPort
tracing
, slEnv :: SyncEnv
slEnv = SyncEnv
env
, slSchedule :: SyncSchedule
slSchedule = SyncSchedule
schedule
, slHooks :: SyncHooks
slHooks = SyncHooks
hooks
, slEcosystem :: Text
slEcosystem = Ecosystem -> Text
forall b a. (Show a, IsString b) => a -> b
show (SyncEnv -> Ecosystem
syncEcosystem SyncEnv
env)
}
data SyncLoop = SyncLoop
{ SyncLoop -> AdvisorySyncMetricsPort
slMetrics :: AdvisorySyncMetricsPort
, SyncLoop -> AdvisorySyncTracingPort
slTracing :: AdvisorySyncTracingPort
, SyncLoop -> SyncEnv
slEnv :: SyncEnv
, SyncLoop -> SyncSchedule
slSchedule :: SyncSchedule
, SyncLoop -> SyncHooks
slHooks :: SyncHooks
, SyncLoop -> Text
slEcosystem :: Text
}
data Pacing = Pacing
{ Pacing -> Int
pacUnloaded :: Int
, Pacing -> Maybe Int
pacFetchFailure :: Maybe Int
}
initialPacing :: Pacing
initialPacing :: Pacing
initialPacing = Pacing{pacUnloaded :: Int
pacUnloaded = Int
0, pacFetchFailure :: Maybe Int
pacFetchFailure = Maybe Int
forall a. Maybe a
Nothing}
loopStep :: (MonadUnliftIO m, KatipContext m) => SyncLoop -> Maybe DbEtag -> m Stepped
loopStep :: forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
SyncLoop -> Maybe DbEtag -> m Stepped
loopStep SyncLoop
loop Maybe DbEtag
lastSeen = do
stepped <-
AdvisorySyncMetricsPort
-> AdvisorySyncTracingPort
-> SyncEnv
-> Text
-> IO ()
-> Maybe DbEtag
-> m Stepped
forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
AdvisorySyncMetricsPort
-> AdvisorySyncTracingPort
-> SyncEnv
-> Text
-> IO ()
-> Maybe DbEtag
-> m Stepped
observedStep
(SyncLoop -> AdvisorySyncMetricsPort
slMetrics SyncLoop
loop)
(SyncLoop -> AdvisorySyncTracingPort
slTracing SyncLoop
loop)
(SyncLoop -> SyncEnv
slEnv SyncLoop
loop)
(SyncLoop -> Text
slEcosystem SyncLoop
loop)
(SyncHooks -> IO ()
hookFirstSync (SyncLoop -> SyncHooks
slHooks SyncLoop
loop))
Maybe DbEtag
lastSeen
liftIO (hookPushAge (slHooks loop))
pure stepped
burstCycle :: (MonadUnliftIO m, KatipContext m) => SyncLoop -> Pacing -> Int -> [Int] -> m (Pacing, Maybe DbEtag)
burstCycle :: forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
SyncLoop -> Pacing -> Int -> [Int] -> m (Pacing, Maybe DbEtag)
burstCycle SyncLoop
loop Pacing
pacing Int
delta [Int]
delays = do
stepped <- SyncLoop -> Maybe DbEtag -> m Stepped
forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
SyncLoop -> Maybe DbEtag -> m Stepped
loopStep SyncLoop
loop Maybe DbEtag
forall a. Maybe a
Nothing
pacing' <- reportFetch loop delta stepped pacing
case delays of
[Int]
_ | Stepped -> Bool
stSettled Stepped
stepped -> (Pacing, Maybe DbEtag) -> m (Pacing, Maybe DbEtag)
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Pacing
pacing', Stepped -> Maybe DbEtag
stSeen Stepped
stepped)
[] -> (Pacing
pacing', Stepped -> Maybe DbEtag
stSeen Stepped
stepped) (Pacing, Maybe DbEtag) -> m () -> m (Pacing, Maybe DbEtag)
forall a b. a -> m b -> m a
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ SyncLoop -> AdvisorySyncResult -> m ()
forall (m :: * -> *).
KatipContext m =>
SyncLoop -> AdvisorySyncResult -> m ()
reportLoopUnloaded SyncLoop
loop (Stepped -> AdvisorySyncResult
stResult Stepped
stepped)
Int
delay : [Int]
rest -> Int -> m ()
forall (m :: * -> *). MonadIO m => Int -> m ()
threadDelay Int
delay m () -> m (Pacing, Maybe DbEtag) -> m (Pacing, Maybe DbEtag)
forall a b. m a -> m b -> m b
forall (m :: * -> *) a b. Monad m => m a -> m b -> m b
>> SyncLoop -> Pacing -> Int -> [Int] -> m (Pacing, Maybe DbEtag)
forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
SyncLoop -> Pacing -> Int -> [Int] -> m (Pacing, Maybe DbEtag)
burstCycle SyncLoop
loop Pacing
pacing' Int
delay [Int]
rest
pollCycle :: (MonadUnliftIO m, KatipContext m) => SyncLoop -> Pacing -> Maybe DbEtag -> m ()
pollCycle :: forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
SyncLoop -> Pacing -> Maybe DbEtag -> m ()
pollCycle SyncLoop
loop Pacing
pacing Maybe DbEtag
lastSeen = do
Int -> m ()
forall (m :: * -> *). MonadIO m => Int -> m ()
threadDelay Int
pollDelay
stepped <- SyncLoop -> Maybe DbEtag -> m Stepped
forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
SyncLoop -> Maybe DbEtag -> m Stepped
loopStep SyncLoop
loop Maybe DbEtag
lastSeen
pacing' <- reportFetch loop pollDelay stepped pacing
unloaded <- repeatUnloaded loop (stResult stepped) (pacUnloaded pacing' + pollDelay)
pollCycle loop pacing'{pacUnloaded = unloaded} (stSeen stepped)
where
pollDelay :: Int
pollDelay = SyncSchedule -> Int
schedPollDelay (SyncLoop -> SyncSchedule
slSchedule SyncLoop
loop)
reportFetch :: (KatipContext m) => SyncLoop -> Int -> Stepped -> Pacing -> m Pacing
reportFetch :: forall (m :: * -> *).
KatipContext m =>
SyncLoop -> Int -> Stepped -> Pacing -> m Pacing
reportFetch SyncLoop
loop Int
delta Stepped
stepped Pacing
pacing = do
let (Maybe Int
fetching, Maybe FetchHealth
report) =
Int
-> Int
-> Maybe OsvDbFetchFault
-> Maybe Int
-> (Maybe Int, Maybe FetchHealth)
paceFetchFailure (SyncSchedule -> Int
schedAbsentReport (SyncLoop -> SyncSchedule
slSchedule SyncLoop
loop)) Int
delta (Stepped -> Maybe OsvDbFetchFault
stFault Stepped
stepped) (Pacing -> Maybe Int
pacFetchFailure Pacing
pacing)
(FetchHealth -> m ()) -> Maybe FetchHealth -> m ()
forall (t :: * -> *) (f :: * -> *) a b.
(Foldable t, Applicative f) =>
(a -> f b) -> t a -> f ()
traverse_ (Text -> FetchHealth -> m ()
forall (m :: * -> *). KatipContext m => Text -> FetchHealth -> m ()
reportFetchHealth (SyncLoop -> Text
slEcosystem SyncLoop
loop)) Maybe FetchHealth
report
Pacing -> m Pacing
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure Pacing
pacing{pacFetchFailure = fetching}
repeatUnloaded :: (KatipContext m) => SyncLoop -> AdvisorySyncResult -> Int -> m Int
repeatUnloaded :: forall (m :: * -> *).
KatipContext m =>
SyncLoop -> AdvisorySyncResult -> Int -> m Int
repeatUnloaded SyncLoop
loop AdvisorySyncResult
result Int
elapsed =
IO (Maybe DbEtag) -> m (Maybe DbEtag)
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (CveSlot -> IO (Maybe DbEtag)
currentAdvisoryEtag (SyncEnv -> CveSlot
syncSlot (SyncLoop -> SyncEnv
slEnv SyncLoop
loop))) m (Maybe DbEtag) -> (Maybe DbEtag -> m Int) -> m Int
forall a b. m a -> (a -> m b) -> m b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= \case
Just DbEtag
_ -> Int -> m Int
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure Int
0
Maybe DbEtag
Nothing
| Int
elapsed Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
< SyncSchedule -> Int
schedAbsentReport (SyncLoop -> SyncSchedule
slSchedule SyncLoop
loop) -> Int -> m Int
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure Int
elapsed
| Bool
otherwise -> Int
0 Int -> m () -> m Int
forall a b. a -> m b -> m a
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ SyncLoop -> AdvisorySyncResult -> m ()
forall (m :: * -> *).
KatipContext m =>
SyncLoop -> AdvisorySyncResult -> m ()
reportLoopUnloaded SyncLoop
loop AdvisorySyncResult
result
reportLoopUnloaded :: (KatipContext m) => SyncLoop -> AdvisorySyncResult -> m ()
reportLoopUnloaded :: forall (m :: * -> *).
KatipContext m =>
SyncLoop -> AdvisorySyncResult -> m ()
reportLoopUnloaded SyncLoop
loop = Text -> Text -> AdvisorySyncResult -> m ()
forall (m :: * -> *).
KatipContext m =>
Text -> Text -> AdvisorySyncResult -> m ()
reportUnloaded (SyncLoop -> Text
slEcosystem SyncLoop
loop) (SyncEnv -> Text
syncStoreRef (SyncLoop -> SyncEnv
slEnv SyncLoop
loop))
data FetchHealth
= FetchFailing OsvDbFetchFault
| FetchStillFailing OsvDbFetchFault
| FetchRecovered
paceFetchFailure :: Int -> Int -> Maybe OsvDbFetchFault -> Maybe Int -> (Maybe Int, Maybe FetchHealth)
paceFetchFailure :: Int
-> Int
-> Maybe OsvDbFetchFault
-> Maybe Int
-> (Maybe Int, Maybe FetchHealth)
paceFetchFailure Int
interval Int
delta Maybe OsvDbFetchFault
fault Maybe Int
failing = case (Maybe OsvDbFetchFault
fault, Maybe Int
failing) of
(Maybe OsvDbFetchFault
Nothing, Maybe Int
Nothing) -> (Maybe Int
forall a. Maybe a
Nothing, Maybe FetchHealth
forall a. Maybe a
Nothing)
(Maybe OsvDbFetchFault
Nothing, Just Int
_) -> (Maybe Int
forall a. Maybe a
Nothing, FetchHealth -> Maybe FetchHealth
forall a. a -> Maybe a
Just FetchHealth
FetchRecovered)
(Just OsvDbFetchFault
f, Maybe Int
Nothing) -> (Int -> Maybe Int
forall a. a -> Maybe a
Just Int
0, FetchHealth -> Maybe FetchHealth
forall a. a -> Maybe a
Just (OsvDbFetchFault -> FetchHealth
FetchFailing OsvDbFetchFault
f))
(Just OsvDbFetchFault
f, Just Int
elapsed)
| Int
elapsed Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
delta Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
>= Int
interval -> (Int -> Maybe Int
forall a. a -> Maybe a
Just Int
0, FetchHealth -> Maybe FetchHealth
forall a. a -> Maybe a
Just (OsvDbFetchFault -> FetchHealth
FetchStillFailing OsvDbFetchFault
f))
| Bool
otherwise -> (Int -> Maybe Int
forall a. a -> Maybe a
Just (Int
elapsed Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
delta), Maybe FetchHealth
forall a. Maybe a
Nothing)
reportFetchHealth :: (KatipContext m) => Text -> FetchHealth -> m ()
reportFetchHealth :: forall (m :: * -> *). KatipContext m => Text -> FetchHealth -> m ()
reportFetchHealth Text
eco = \case
FetchFailing OsvDbFetchFault
fault -> Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"cve-sync[" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
eco Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"]: sync fetch failed: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> OsvDbFetchFault -> Text
renderOsvDbFetchFault OsvDbFetchFault
fault))
FetchStillFailing OsvDbFetchFault
fault -> Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"cve-sync[" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
eco Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"]: sync fetch still failing: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> OsvDbFetchFault -> Text
renderOsvDbFetchFault OsvDbFetchFault
fault))
FetchHealth
FetchRecovered -> Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
InfoS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"cve-sync[" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
eco Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"]: sync fetch recovered"))
reportUnloaded :: (KatipContext m) => Text -> Text -> AdvisorySyncResult -> m ()
reportUnloaded :: forall (m :: * -> *).
KatipContext m =>
Text -> Text -> AdvisorySyncResult -> m ()
reportUnloaded Text
eco Text
store AdvisorySyncResult
result =
Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"cve-sync[" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
eco Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"]: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text -> AdvisorySyncResult -> Text
unloadedCause Text
store AdvisorySyncResult
result))
unloadedCause :: Text -> AdvisorySyncResult -> Text
unloadedCause :: Text -> AdvisorySyncResult -> Text
unloadedCause Text
store = \case
AdvisorySyncResult
AdvisoryNonePublished ->
Text
"no advisory artifact has ever been published to " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
store Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
", and ecluse pilot is what compiles and publishes one. This ecosystem stays not-ready and its advisory denies refuse until an artifact lands."
AdvisorySyncResult
AdvisoryRefused -> Text
refusedArtifact
AdvisorySyncResult
AdvisoryUnchanged -> Text
refusedArtifact
AdvisorySyncResult
AdvisoryFetchFailed ->
Text
"no advisory database could be fetched from " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
store Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": this ecosystem stays not-ready and denies by default until one loads. Continuing to poll; investigate the bucket, object, or IAM if this persists."
AdvisorySyncResult
AdvisorySwapped ->
Text
"no advisory database is loaded from " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
store Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
", so this ecosystem stays not-ready and denies by default until one is."
where
refusedArtifact :: Text
refusedArtifact =
Text
"the advisory artifact at " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
store Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" was refused by verification, so nothing is loaded and this ecosystem stays not-ready. The refusal line names what failed, and ecluse pilot must publish an artifact that verifies."
data Stepped = Stepped
{ Stepped -> AdvisorySyncResult
stResult :: AdvisorySyncResult
, Stepped -> Bool
stSettled :: Bool
, Stepped -> Maybe DbEtag
stSeen :: Maybe DbEtag
, Stepped -> Maybe OsvDbFetchFault
stFault :: Maybe OsvDbFetchFault
}
steppedOf :: AdvisorySyncResult -> Bool -> Maybe DbEtag -> Stepped
steppedOf :: AdvisorySyncResult -> Bool -> Maybe DbEtag -> Stepped
steppedOf AdvisorySyncResult
result Bool
settled Maybe DbEtag
seen =
Stepped{stResult :: AdvisorySyncResult
stResult = AdvisorySyncResult
result, stSettled :: Bool
stSettled = Bool
settled, stSeen :: Maybe DbEtag
stSeen = Maybe DbEtag
seen, stFault :: Maybe OsvDbFetchFault
stFault = Maybe OsvDbFetchFault
forall a. Maybe a
Nothing}
observedStep ::
(MonadUnliftIO m, KatipContext m) =>
AdvisorySyncMetricsPort ->
AdvisorySyncTracingPort ->
SyncEnv ->
Text ->
IO () ->
Maybe DbEtag ->
m Stepped
observedStep :: forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
AdvisorySyncMetricsPort
-> AdvisorySyncTracingPort
-> SyncEnv
-> Text
-> IO ()
-> Maybe DbEtag
-> m Stepped
observedStep AdvisorySyncMetricsPort
metrics AdvisorySyncTracingPort
tracing SyncEnv
env Text
eco IO ()
notifyFirstSync Maybe DbEtag
lastSeen =
((forall a. m a -> IO a) -> IO Stepped) -> m Stepped
forall b. ((forall a. m a -> IO a) -> IO b) -> m b
forall (m :: * -> *) b.
MonadUnliftIO m =>
((forall a. m a -> IO a) -> IO b) -> m b
withRunInIO (((forall a. m a -> IO a) -> IO Stepped) -> m Stepped)
-> ((forall a. m a -> IO a) -> IO Stepped) -> m Stepped
forall a b. (a -> b) -> a -> b
$ \forall a. m a -> IO a
runInIO ->
AdvisorySyncTracingPort
-> forall a.
Ecosystem
-> (a -> (AdvisorySyncResult, Maybe OsvDbFetchFault))
-> IO a
-> IO a
astpSyncAttemptSpan
AdvisorySyncTracingPort
tracing
Ecosystem
ecosystem
(\Stepped
stepped -> (Stepped -> AdvisorySyncResult
stResult Stepped
stepped, Stepped -> Maybe OsvDbFetchFault
stFault Stepped
stepped))
(AdvisorySyncMetricsPort -> Ecosystem -> IO Stepped -> IO Stepped
meteredStep AdvisorySyncMetricsPort
metrics Ecosystem
ecosystem (m Stepped -> IO Stepped
forall a. m a -> IO a
runInIO (SyncEnv -> Text -> IO () -> Maybe DbEtag -> m Stepped
forall (m :: * -> *).
KatipContext m =>
SyncEnv -> Text -> IO () -> Maybe DbEtag -> m Stepped
attemptStep SyncEnv
env Text
eco IO ()
notifyFirstSync Maybe DbEtag
lastSeen)))
where
ecosystem :: Ecosystem
ecosystem = SyncEnv -> Ecosystem
syncEcosystem SyncEnv
env
meteredStep :: AdvisorySyncMetricsPort -> Ecosystem -> IO Stepped -> IO Stepped
meteredStep :: AdvisorySyncMetricsPort -> Ecosystem -> IO Stepped -> IO Stepped
meteredStep AdvisorySyncMetricsPort
metrics Ecosystem
ecosystem IO Stepped
act = do
(attempted, seconds) <- IO Stepped -> IO (Stepped, Double)
forall (m :: * -> *) a. MonadIO m => m a -> m (a, Double)
timedSeconds IO Stepped
act
asmpSyncAttempt metrics ecosystem (stResult attempted)
asmpSyncDuration metrics ecosystem (stResult attempted) seconds
pure attempted
attemptStep :: (KatipContext m) => SyncEnv -> Text -> IO () -> Maybe DbEtag -> m Stepped
attemptStep :: forall (m :: * -> *).
KatipContext m =>
SyncEnv -> Text -> IO () -> Maybe DbEtag -> m Stepped
attemptStep SyncEnv
env Text
eco IO ()
notifyFirstSync Maybe DbEtag
lastSeen =
IO SyncOutcome -> m SyncOutcome
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (SyncEnv -> Maybe DbEtag -> IO SyncOutcome
syncStep SyncEnv
env Maybe DbEtag
lastSeen) m SyncOutcome -> (SyncOutcome -> m Stepped) -> m Stepped
forall a b. m a -> (a -> m b) -> m b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= \case
SyncFetchFaulted OsvDbFetchFault
fault ->
Stepped -> m Stepped
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (AdvisorySyncResult -> Bool -> Maybe DbEtag -> Stepped
steppedOf AdvisorySyncResult
AdvisoryFetchFailed Bool
False Maybe DbEtag
lastSeen){stFault = Just fault}
SyncSwapped DbEtag
etag [(Text, Text)]
meta -> do
Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
InfoS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"cve-sync[" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
eco Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"]: advisory database swapped in: etag=" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> DbEtag -> Text
forall b a. (Show a, IsString b) => a -> b
show DbEtag
etag Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" meta=" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> (Maybe UTCTime, Maybe Word64) -> Text
forall b a. (Show a, IsString b) => a -> b
show ([(Text, Text)] -> (Maybe UTCTime, Maybe Word64)
metadataSummary [(Text, Text)]
meta)))
source <- IO (Maybe AdvisorySource) -> m (Maybe AdvisorySource)
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (CveSlot -> IO (Maybe AdvisorySource)
currentAdvisorySource (SyncEnv -> CveSlot
syncSlot SyncEnv
env))
logFM InfoS (ls ("cve-sync[" <> eco <> "]: serving artifact source: " <> maybe unrecordedValue renderAdvisorySource source))
whenNothing_ (asPushedAt =<< source) (undatedArtifact eco etag)
liftIO notifyFirstSync
pure (steppedOf AdvisorySwapped True (Just etag))
SyncOutcome
SyncUnchanged -> do
Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
DebugS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"cve-sync[" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
eco Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"]: advisory database unchanged"))
Stepped -> m Stepped
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (AdvisorySyncResult -> Bool -> Maybe DbEtag -> Stepped
steppedOf AdvisorySyncResult
AdvisoryUnchanged Bool
True Maybe DbEtag
lastSeen)
SyncOutcome
SyncAbsent -> do
Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
DebugS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"cve-sync[" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
eco Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"]: no advisory database published yet"))
Stepped -> m Stepped
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (AdvisorySyncResult -> Bool -> Maybe DbEtag -> Stepped
steppedOf AdvisorySyncResult
AdvisoryNonePublished Bool
False Maybe DbEtag
lastSeen)
SyncRejected DbEtag
etag CveDbRejected
rejection -> do
Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"cve-sync[" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
eco Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"]: downloaded artifact refused (keeping last good): " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> CveDbRejected -> Text
forall b a. (Show a, IsString b) => a -> b
show CveDbRejected
rejection))
Stepped -> m Stepped
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (AdvisorySyncResult -> Bool -> Maybe DbEtag -> Stepped
steppedOf AdvisorySyncResult
AdvisoryRefused Bool
True (DbEtag -> Maybe DbEtag
forall a. a -> Maybe a
Just DbEtag
etag))
undatedArtifact :: (KatipContext m) => Text -> DbEtag -> m ()
undatedArtifact :: forall (m :: * -> *). KatipContext m => Text -> DbEtag -> m ()
undatedArtifact Text
eco DbEtag
etag =
Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM
Severity
ErrorS
( Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls
( Text
"cve-sync["
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
eco
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"]: the object store reported no publication time for the artifact it served (etag="
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> DbEtag -> Text
forall b a. (Show a, IsString b) => a -> b
show DbEtag
etag
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"), so its age cannot be established and CVE-based denial refuses until a push carries one"
)
)
renderAdvisorySource :: AdvisorySource -> Text
renderAdvisorySource :: AdvisorySource -> Text
renderAdvisorySource AdvisorySource
source =
Text
"pushed_at="
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Maybe UTCTime -> Text
renderStamp (AdvisorySource -> Maybe UTCTime
asPushedAt AdvisorySource
source)
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" osv_source="
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text -> (Text -> Text) -> Maybe Text -> Text
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Text
unrecordedValue Text -> Text
dialledAuthorityLabel (AdvisoryProvenance -> Maybe Text
apOsvSource AdvisoryProvenance
prov)
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" osv_newest_modified="
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Maybe UTCTime -> Text
renderStamp (AdvisoryProvenance -> Maybe UTCTime
apOsvNewestModified AdvisoryProvenance
prov)
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" epss_score_date="
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Maybe UTCTime -> Text
renderStamp (AdvisoryProvenance -> Maybe UTCTime
apEpssScoreDate AdvisoryProvenance
prov)
where
prov :: AdvisoryProvenance
prov = AdvisorySource -> AdvisoryProvenance
asProvenance AdvisorySource
source
renderStamp :: Maybe UTCTime -> Text
renderStamp :: Maybe UTCTime -> Text
renderStamp = Text -> (UTCTime -> Text) -> Maybe UTCTime -> Text
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Text
unrecordedValue UTCTime -> Text
renderIso8601Utc
unrecordedValue :: Text
unrecordedValue :: Text
unrecordedValue = Text
"<unrecorded>"
metadataSummary :: [(Text, Text)] -> (Maybe UTCTime, Maybe Word64)
metadataSummary :: [(Text, Text)] -> (Maybe UTCTime, Maybe Word64)
metadataSummary [(Text, Text)]
meta =
( MetaKey -> Int -> Maybe Text
boundedValue MetaKey
MetaBuiltAt Int
64 Maybe Text -> (Text -> Maybe UTCTime) -> Maybe UTCTime
forall a b. Maybe a -> (a -> Maybe b) -> Maybe b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= String -> Maybe UTCTime
forall (m :: * -> *) t. (MonadFail m, ISO8601 t) => String -> m t
iso8601ParseM (String -> Maybe UTCTime)
-> (Text -> String) -> Text -> Maybe UTCTime
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Text -> String
forall a. ToString a => a -> String
toString
, MetaKey -> Int -> Maybe Text
boundedValue MetaKey
MetaRowCount Int
20 Maybe Text -> (Text -> Maybe Word64) -> Maybe Word64
forall a b. Maybe a -> (a -> Maybe b) -> Maybe b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= Text -> Maybe Word64
parseMetadataCount
)
where
boundedValue :: MetaKey -> Int -> Maybe Text
boundedValue MetaKey
key Int
limit = do
value <- Text -> [(Text, Text)] -> Maybe Text
forall a b. Eq a => a -> [(a, b)] -> Maybe b
lookup (MetaKey -> Text
renderMetaKey MetaKey
key) [(Text, Text)]
meta
guard (T.compareLength value limit /= GT)
pure value
parseMetadataCount :: Text -> Maybe Word64
parseMetadataCount :: Text -> Maybe Word64
parseMetadataCount Text
value = do
count <- Text -> Maybe Integer
forall a. Integral a => Text -> Maybe a
readDecimalText Text
value :: Maybe Integer
guard (count <= toInteger (maxBound :: Word64))
pure (fromInteger count)