-- SPDX-FileCopyrightText: 2026 Alexandra de Wit
--
-- SPDX-License-Identifier: MIT

{- | Compile OSV advisories and EPSS scores into the artifact consumed by CVE sync. Every pass
attempts the EPSS feed, and the ecosystem's 'EpssRequirement' decides whether a failed feed stops
publication or leaves the artifact recording unavailable enrichment.

A pass compiles into an exclusive temporary file, the candidate, which replaces the canonical file
only once its caller promotes it, so a refused publication leaves the previous artifact in place.
-}
module Ecluse.Core.Osv.Compile (
    CompileSources,
    mkCompileSources,
    csOsvExportUrl,
    csEpssFeedUrl,
    csOsvRecorded,
    csEpssRecorded,

    -- * The candidate a publication decides on
    OsvCandidate,
    candidatePath,
    candidateDestination,
    candidateFacts,
    withOsvCandidate,
    stampCandidateNonce,
    promoteCandidate,

    -- * Feed downloads
    newFeedManagerWithIdentity,

    -- * Refusals and rows
    PilotEpssRequired (..),
    PilotOsvFetchFailed (..),
    osvToRow,
) where

import Conduit
import Control.Monad.Catch (MonadMask)
import Data.Conduit.List qualified as CL
import Data.Time (UTCTime, getCurrentTime)
import Data.Time.Format.ISO8601 (iso8601Show)
import Database.SQLite.Simple
import Katip (KatipContext, Severity (..), SimpleLogPayload, katipAddContext, logFM, ls, sl)
import Network.HTTP.Client (Manager)
import Network.HTTP.Client.TLS (newTlsManagerWith)
import System.Directory (createDirectoryIfMissing, removeFile, renameFile)
import System.FilePath ((</>))
import System.IO (hClose, openTempFile)
import System.IO.Error (catchIOError)
import UnliftIO.Exception (bracket, throwIO, try)

import Ecluse.Core.BuildIdentity (identityManagerSettings, productVersion)
import Ecluse.Core.Fault (HttpFault, renderHttpFault)
import Ecluse.Core.Fault.Http (httpFault, typedTlsManagerSettings)
import Ecluse.Core.Osv.Advisory (ExtractedOsv (..))
import Ecluse.Core.Osv.Ecosystem (OsvEcosystem (osvExportDirectory, osvWireName))
import Ecluse.Core.Osv.Epss (
    EpssEnrichment (EpssEnriched, EpssUnavailable),
    EpssFeed (efDigest, efLastModified, efModelVersion, efScoreDate, efScores),
    EpssFeedFailure,
    acquireEpssFeed,
    enrichedFeed,
    enrichmentStatus,
    maxEpssFeedBytes,
    mkEpssScores,
    renderEpssFeedFailure,
    resolveEnrichment,
 )
import Ecluse.Core.Osv.Provenance (
    AdvisoryProvenance (..),
    QuietTime,
    RecordedSource,
    SourceAge,
    SourcePathRecording,
    provenanceRows,
    recordSource,
    renderRecordedSource,
    renderSourceAge,
    sourceAges,
    sourceQuiet,
 )
import Ecluse.Core.Osv.Publication (
    CandidateFacts (..),
    PublicationNonce,
    SourceMarker (..),
    newPublicationNonce,
    renderPublicationNonce,
    sourceDigestHex,
    sourceDigestOf,
    sourceIdentityOf,
    usableLastModified,
 )
import Ecluse.Core.Osv.Retry (defaultOsvRetryPolicy, withOsvRetry)
import Ecluse.Core.Osv.Schema (
    EpssRequirement (EpssOptional, EpssRequired),
    EpssStatus (EnrichmentAvailable, EnrichmentUnavailable),
    MetaKey (..),
    metaTableDdl,
    osvDbFileName,
    osvSchemaEpoch,
    rangesTableDdl,
    renderEpssStatus,
    renderMetaKey,
 )
import Ecluse.Core.Osv.Stream (
    IngestStats (..),
    OsvAttempt (..),
    OsvIngest,
    PilotIngestAborted (..),
    defaultIngestLimits,
    newOsvIngest,
    readIngestStats,
    readOsvAttempt,
    resetIngestStats,
    resetOsvAttempt,
    streamOsvUrl,
    systemicDrop,
 )
import Ecluse.Core.Osv.Types (UpperBound (FixedBefore, LastAffected, Unbounded))
import Ecluse.Core.Security.Authority (credentialFreeUrl, dialledAuthorityLabel)
import Ecluse.Core.Snapshot (ContentDigest)
import Ecluse.Core.Telemetry.Metrics (
    AdvisoryCompileResult (CompileAborted, CompileCompleted),
    AdvisoryDropCause (DropMalformed, DropOversize),
 )
import Ecluse.Core.Telemetry.Record (AdvisoryCompileMetricsPort (acmpCompileAccepted, acmpCompileDropped, acmpCompileRun))
import Ecluse.Core.Telemetry.Span (withOptionalSpan)
import OpenTelemetry.Trace.Core (Span, SpanKind (Internal), SpanStatus (Error), TracerProvider, addAttribute, setStatus)

-- | The two upstreams one compile pass reads: the ecosystem's advisories, and the scores it joins onto them.
data CompileSources = CompileSources String String RecordedSource RecordedSource
    deriving stock (CompileSources -> CompileSources -> Bool
(CompileSources -> CompileSources -> Bool)
-> (CompileSources -> CompileSources -> Bool) -> Eq CompileSources
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: CompileSources -> CompileSources -> Bool
== :: CompileSources -> CompileSources -> Bool
$c/= :: CompileSources -> CompileSources -> Bool
/= :: CompileSources -> CompileSources -> Bool
Eq, Int -> CompileSources -> ShowS
[CompileSources] -> ShowS
CompileSources -> String
(Int -> CompileSources -> ShowS)
-> (CompileSources -> String)
-> ([CompileSources] -> ShowS)
-> Show CompileSources
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> CompileSources -> ShowS
showsPrec :: Int -> CompileSources -> ShowS
$cshow :: CompileSources -> String
show :: CompileSources -> String
$cshowList :: [CompileSources] -> ShowS
showList :: [CompileSources] -> ShowS
Show)

-- | The sources one pass reads. Each recorded form derives from its fetched URL, under the one choice given.
mkCompileSources :: SourcePathRecording -> String -> String -> CompileSources
mkCompileSources :: SourcePathRecording -> String -> String -> CompileSources
mkCompileSources SourcePathRecording
recording String
osvUrl String
epssUrl =
    String
-> String -> RecordedSource -> RecordedSource -> CompileSources
CompileSources String
osvUrl String
epssUrl (SourcePathRecording -> Text -> RecordedSource
recordSource SourcePathRecording
recording (String -> Text
forall a. ToText a => a -> Text
toText String
osvUrl)) (SourcePathRecording -> Text -> RecordedSource
recordSource SourcePathRecording
recording (String -> Text
forall a. ToText a => a -> Text
toText String
epssUrl))

-- | The ecosystem's OSV export archive ('Ecluse.Core.Osv.Advisory.osvExportUrl').
csOsvExportUrl :: CompileSources -> String
csOsvExportUrl :: CompileSources -> String
csOsvExportUrl (CompileSources String
url String
_ RecordedSource
_ RecordedSource
_) = String
url

-- | The EPSS daily feed, from the configured @advisories.epssFeedUrl@.
csEpssFeedUrl :: CompileSources -> String
csEpssFeedUrl :: CompileSources -> String
csEpssFeedUrl (CompileSources String
_ String
url RecordedSource
_ RecordedSource
_) = String
url

-- | The export as its artifact records it.
csOsvRecorded :: CompileSources -> RecordedSource
csOsvRecorded :: CompileSources -> RecordedSource
csOsvRecorded (CompileSources String
_ String
_ RecordedSource
recorded RecordedSource
_) = RecordedSource
recorded

-- | The feed as its artifact records it.
csEpssRecorded :: CompileSources -> RecordedSource
csEpssRecorded :: CompileSources -> RecordedSource
csEpssRecorded (CompileSources String
_ String
_ RecordedSource
_ RecordedSource
recorded) = RecordedSource
recorded

-- | A compiled artifact in its exclusive temporary file, with the facts its publication compares.
data OsvCandidate = OsvCandidate
    { OsvCandidate -> String
candidatePath :: FilePath
    , OsvCandidate -> String
candidateDestination :: FilePath
    , OsvCandidate -> CandidateFacts
candidateFacts :: CandidateFacts
    }

-- | Compile one ecosystem into a candidate for @use@, which may promote it. A refused compile throws before @use@ runs.
withOsvCandidate :: (MonadResource m, MonadMask m, MonadUnliftIO m, KatipContext m) => AdvisoryCompileMetricsPort -> Maybe TracerProvider -> Manager -> FilePath -> OsvEcosystem -> EpssRequirement -> CompileSources -> QuietTime -> (OsvCandidate -> m a) -> m a
withOsvCandidate :: forall (m :: * -> *) a.
(MonadResource m, MonadMask m, MonadUnliftIO m, KatipContext m) =>
AdvisoryCompileMetricsPort
-> Maybe TracerProvider
-> Manager
-> String
-> OsvEcosystem
-> EpssRequirement
-> CompileSources
-> QuietTime
-> (OsvCandidate -> m a)
-> m a
withOsvCandidate AdvisoryCompileMetricsPort
metrics Maybe TracerProvider
mTracerProvider Manager
manager String
outDir OsvEcosystem
eco EpssRequirement
requirement CompileSources
sources QuietTime
quietTime OsvCandidate -> m a
use = do
    let dbFile :: String
dbFile = String
outDir String -> ShowS
</> Text -> String
osvDbFileName (OsvEcosystem -> Text
osvWireName OsvEcosystem
eco)
    Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
InfoS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"Compiling OSV data for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> OsvEcosystem -> Text
osvWireName OsvEcosystem
eco Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" to " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> String -> Text
forall a. ToText a => a -> Text
toText String
dbFile Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
", EPSS enrichment " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> EpssRequirement -> Text
renderRequirement EpssRequirement
requirement))

    IO () -> m ()
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO () -> m ()) -> IO () -> m ()
forall a b. (a -> b) -> a -> b
$ Bool -> String -> IO ()
createDirectoryIfMissing Bool
True String
outDir

    m String -> (String -> m ()) -> (String -> m a) -> m a
forall (m :: * -> *) a b c.
MonadUnliftIO m =>
m a -> (a -> m b) -> (a -> m c) -> m c
bracket (IO String -> m String
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO String -> m String) -> IO String -> m String
forall a b. (a -> b) -> a -> b
$ String -> IO String
newCandidate String
outDir) (IO () -> m ()
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO () -> m ()) -> (String -> IO ()) -> String -> m ()
forall b c a. (b -> c) -> (a -> b) -> a -> c
. String -> IO ()
removeCandidate) ((String -> m a) -> m a) -> (String -> m a) -> m a
forall a b. (a -> b) -> a -> b
$ \String
candidate -> do
        facts <- CompileRun -> String -> m CandidateFacts
forall (m :: * -> *).
(MonadResource m, MonadMask m, MonadUnliftIO m, KatipContext m) =>
CompileRun -> String -> m CandidateFacts
compileCandidate CompileRun
run String
candidate
        use OsvCandidate{candidatePath = candidate, candidateDestination = dbFile, candidateFacts = facts}
  where
    run :: CompileRun
run =
        CompileRun
            { crMetrics :: AdvisoryCompileMetricsPort
crMetrics = AdvisoryCompileMetricsPort
metrics
            , crTracerProvider :: Maybe TracerProvider
crTracerProvider = Maybe TracerProvider
mTracerProvider
            , crManager :: Manager
crManager = Manager
manager
            , crEcosystem :: OsvEcosystem
crEcosystem = OsvEcosystem
eco
            , crEpss :: EpssRequirement
crEpss = EpssRequirement
requirement
            , crSources :: CompileSources
crSources = CompileSources
sources
            , crQuietTime :: QuietTime
crQuietTime = QuietTime
quietTime
            }

-- | Reuse a precomputed outbound identity for both advisory feeds.
newFeedManagerWithIdentity :: ByteString -> IO Manager
newFeedManagerWithIdentity :: ByteString -> IO Manager
newFeedManagerWithIdentity ByteString
renderedAgent = ManagerSettings -> IO Manager
forall (m :: * -> *). MonadIO m => ManagerSettings -> m Manager
newTlsManagerWith (ByteString -> ManagerSettings -> ManagerSettings
identityManagerSettings ByteString
renderedAgent ManagerSettings
typedTlsManagerSettings)

-- | Write this attempt's nonce into the candidate. The nonce is fixed width, so the file keeps its size.
stampCandidateNonce :: (MonadIO m) => OsvCandidate -> PublicationNonce -> m ()
stampCandidateNonce :: forall (m :: * -> *).
MonadIO m =>
OsvCandidate -> PublicationNonce -> m ()
stampCandidateNonce OsvCandidate
candidate PublicationNonce
nonce =
    IO () -> m ()
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO () -> m ()) -> IO () -> m ()
forall a b. (a -> b) -> a -> b
$ String -> (Connection -> IO ()) -> IO ()
forall a. String -> (Connection -> IO a) -> IO a
withConnection (OsvCandidate -> String
candidatePath OsvCandidate
candidate) ((Connection -> IO ()) -> IO ()) -> (Connection -> IO ()) -> IO ()
forall a b. (a -> b) -> a -> b
$ \Connection
conn ->
        Connection -> Query -> (Text, Text) -> IO ()
forall q. ToRow q => Connection -> Query -> q -> IO ()
execute Connection
conn Query
"UPDATE meta SET value = ? WHERE key = ?" (PublicationNonce -> Text
renderPublicationNonce PublicationNonce
nonce, MetaKey -> Text
renderMetaKey MetaKey
MetaPublicationNonce)

-- | Replace the canonical artifact with the candidate, and return its path.
promoteCandidate :: (MonadIO m) => OsvCandidate -> m FilePath
promoteCandidate :: forall (m :: * -> *). MonadIO m => OsvCandidate -> m String
promoteCandidate OsvCandidate
candidate = do
    IO () -> m ()
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (String -> String -> IO ()
renameFile (OsvCandidate -> String
candidatePath OsvCandidate
candidate) (OsvCandidate -> String
candidateDestination OsvCandidate
candidate))
    String -> m String
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (OsvCandidate -> String
candidateDestination OsvCandidate
candidate)

-- What stays fixed across one pass, so each step below takes one parameter rather than six.
data CompileRun = CompileRun
    { CompileRun -> AdvisoryCompileMetricsPort
crMetrics :: AdvisoryCompileMetricsPort
    , CompileRun -> Maybe TracerProvider
crTracerProvider :: Maybe TracerProvider
    , CompileRun -> Manager
crManager :: Manager
    , CompileRun -> OsvEcosystem
crEcosystem :: OsvEcosystem
    , CompileRun -> EpssRequirement
crEpss :: EpssRequirement
    , CompileRun -> CompileSources
crSources :: CompileSources
    , CompileRun -> QuietTime
crQuietTime :: QuietTime
    }

-- | A required EPSS feed failed, so the compile published nothing. It names the feed by @host:port@, as the URL can carry a credential.
data PilotEpssRequired = PilotEpssRequired
    { PilotEpssRequired -> Text
perEcosystem :: Text
    , PilotEpssRequired -> Text
perFeed :: Text
    -- ^ The feed's @host:port@.
    , PilotEpssRequired -> EpssFeedFailure
perFailure :: EpssFeedFailure
    }
    deriving stock (PilotEpssRequired -> PilotEpssRequired -> Bool
(PilotEpssRequired -> PilotEpssRequired -> Bool)
-> (PilotEpssRequired -> PilotEpssRequired -> Bool)
-> Eq PilotEpssRequired
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: PilotEpssRequired -> PilotEpssRequired -> Bool
== :: PilotEpssRequired -> PilotEpssRequired -> Bool
$c/= :: PilotEpssRequired -> PilotEpssRequired -> Bool
/= :: PilotEpssRequired -> PilotEpssRequired -> Bool
Eq, Int -> PilotEpssRequired -> ShowS
[PilotEpssRequired] -> ShowS
PilotEpssRequired -> String
(Int -> PilotEpssRequired -> ShowS)
-> (PilotEpssRequired -> String)
-> ([PilotEpssRequired] -> ShowS)
-> Show PilotEpssRequired
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> PilotEpssRequired -> ShowS
showsPrec :: Int -> PilotEpssRequired -> ShowS
$cshow :: PilotEpssRequired -> String
show :: PilotEpssRequired -> String
$cshowList :: [PilotEpssRequired] -> ShowS
showList :: [PilotEpssRequired] -> ShowS
Show)

instance Exception PilotEpssRequired where
    displayException :: PilotEpssRequired -> String
displayException = Text -> String
forall a. ToString a => a -> String
toString (Text -> String)
-> (PilotEpssRequired -> Text) -> PilotEpssRequired -> String
forall b c a. (b -> c) -> (a -> b) -> a -> c
. PilotEpssRequired -> Text
renderEpssRequired

renderEpssRequired :: PilotEpssRequired -> Text
renderEpssRequired :: PilotEpssRequired -> Text
renderEpssRequired PilotEpssRequired
refusal =
    PilotEpssRequired -> Text
perEcosystem PilotEpssRequired
refusal
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" requires EPSS enrichment, and the feed failed: "
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text -> EpssFeedFailure -> Text
renderEpssFeedFailure (PilotEpssRequired -> Text
perFeed PilotEpssRequired
refusal) (PilotEpssRequired -> EpssFeedFailure
perFailure PilotEpssRequired
refusal)

-- | The OSV export fetch failed, after any retries. It names the feed by @host:port@, as a redirect can reach a URL that carries a credential.
data PilotOsvFetchFailed = PilotOsvFetchFailed
    { PilotOsvFetchFailed -> Text
pofFeed :: Text
    -- ^ The export's @host:port@.
    , PilotOsvFetchFailed -> HttpFault
pofFault :: HttpFault
    }
    deriving stock (PilotOsvFetchFailed -> PilotOsvFetchFailed -> Bool
(PilotOsvFetchFailed -> PilotOsvFetchFailed -> Bool)
-> (PilotOsvFetchFailed -> PilotOsvFetchFailed -> Bool)
-> Eq PilotOsvFetchFailed
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: PilotOsvFetchFailed -> PilotOsvFetchFailed -> Bool
== :: PilotOsvFetchFailed -> PilotOsvFetchFailed -> Bool
$c/= :: PilotOsvFetchFailed -> PilotOsvFetchFailed -> Bool
/= :: PilotOsvFetchFailed -> PilotOsvFetchFailed -> Bool
Eq, Int -> PilotOsvFetchFailed -> ShowS
[PilotOsvFetchFailed] -> ShowS
PilotOsvFetchFailed -> String
(Int -> PilotOsvFetchFailed -> ShowS)
-> (PilotOsvFetchFailed -> String)
-> ([PilotOsvFetchFailed] -> ShowS)
-> Show PilotOsvFetchFailed
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> PilotOsvFetchFailed -> ShowS
showsPrec :: Int -> PilotOsvFetchFailed -> ShowS
$cshow :: PilotOsvFetchFailed -> String
show :: PilotOsvFetchFailed -> String
$cshowList :: [PilotOsvFetchFailed] -> ShowS
showList :: [PilotOsvFetchFailed] -> ShowS
Show)

instance Exception PilotOsvFetchFailed where
    displayException :: PilotOsvFetchFailed -> String
displayException PilotOsvFetchFailed
failed = Text -> String
forall a. ToString a => a -> String
toString (Text
"OSV export fetch failed at " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text -> HttpFault -> Text
renderHttpFault (PilotOsvFetchFailed -> Text
pofFeed PilotOsvFetchFailed
failed) (PilotOsvFetchFailed -> HttpFault
pofFault PilotOsvFetchFailed
failed))

-- Fill one candidate file and return what its publication compares.
compileCandidate :: (MonadResource m, MonadMask m, MonadUnliftIO m, KatipContext m) => CompileRun -> FilePath -> m CandidateFacts
compileCandidate :: forall (m :: * -> *).
(MonadResource m, MonadMask m, MonadUnliftIO m, KatipContext m) =>
CompileRun -> String -> m CandidateFacts
compileCandidate CompileRun
run String
dbFile =
    Maybe TracerProvider
-> SpanKind
-> Text
-> (Maybe Span -> m CandidateFacts)
-> m CandidateFacts
forall (m :: * -> *) a.
MonadUnliftIO m =>
Maybe TracerProvider
-> SpanKind -> Text -> (Maybe Span -> m a) -> m a
withOptionalSpan (CompileRun -> Maybe TracerProvider
crTracerProvider CompileRun
run) SpanKind
Internal Text
"ecluse.pilot.osv.compile" ((Maybe Span -> m CandidateFacts) -> m CandidateFacts)
-> (Maybe Span -> m CandidateFacts) -> m CandidateFacts
forall a b. (a -> b) -> a -> b
$ \Maybe Span
mSpan -> do
        Maybe Span -> (Span -> m ()) -> m ()
forall (t :: * -> *) (m :: * -> *) a b.
(Foldable t, Monad m) =>
t a -> (a -> m b) -> m ()
forM_ Maybe Span
mSpan (CompileRun -> Span -> m ()
forall (m :: * -> *). MonadIO m => CompileRun -> Span -> m ()
describeCompile CompileRun
run)

        -- Every record's date is judged against this one instant, so a long pass
        -- cannot let a later record pass a check an earlier one failed.
        now <- IO UTCTime -> m UTCTime
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO IO UTCTime
getCurrentTime

        -- The join needs the whole score table before the first advisory row lands.
        enrichment <- enrichOrRefuse run mSpan
        epssMarker <- traverse (\EpssFeed
feed -> String -> Maybe UTCTime -> ContentDigest -> m SourceMarker
forall (m :: * -> *).
MonadIO m =>
String -> Maybe UTCTime -> ContentDigest -> m SourceMarker
answeredMarker (CompileSources -> String
csEpssFeedUrl CompileSources
sources) (EpssFeed -> Maybe UTCTime
efLastModified EpssFeed
feed) (EpssFeed -> ContentDigest
efDigest EpssFeed
feed)) (enrichedFeed enrichment)
        ingest <- newOsvIngest defaultIngestLimits (crEcosystem run) (maybe (mkEpssScores []) efScores (enrichedFeed enrichment)) now

        bracket (liftIO $ open dbFile) (liftIO . close) $ \Connection
conn -> do
            IO () -> m ()
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO () -> m ()) -> IO () -> m ()
forall a b. (a -> b) -> a -> b
$ Connection -> IO ()
initSchema Connection
conn
            osvDigest <- CompileRun -> OsvIngest -> Connection -> m ContentDigest
forall (m :: * -> *).
(MonadResource m, MonadMask m, MonadUnliftIO m, KatipContext m) =>
CompileRun -> OsvIngest -> Connection -> m ContentDigest
ingestAdvisories CompileRun
run OsvIngest
ingest Connection
conn
            stats <- readIngestStats ingest
            attempt <- readOsvAttempt ingest
            osvMarker <- answeredMarker (csOsvExportUrl sources) (oaLastModified attempt) osvDigest
            concludeCompile (crMetrics run) mSpan conn (conclusionOf run now enrichment attempt stats osvMarker epssMarker)
  where
    sources :: CompileSources
sources = CompileRun -> CompileSources
crSources CompileRun
run

-- A retried fetch can answer minutes after the pass began, so a source's date is bounded by a clock read once it answered.
answeredMarker :: (MonadIO m) => String -> Maybe UTCTime -> ContentDigest -> m SourceMarker
answeredMarker :: forall (m :: * -> *).
MonadIO m =>
String -> Maybe UTCTime -> ContentDigest -> m SourceMarker
answeredMarker String
url Maybe UTCTime
declared ContentDigest
digest = do
    answeredAt <- IO UTCTime -> m UTCTime
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO IO UTCTime
getCurrentTime
    pure (SourceMarker (sourceIdentityOf (credentialFreeUrl (toText url))) (usableLastModified answeredAt declared) (sourceDigestOf digest))

describeCompile :: (MonadIO m) => CompileRun -> Span -> m ()
describeCompile :: forall (m :: * -> *). MonadIO m => CompileRun -> Span -> m ()
describeCompile CompileRun
run Span
sp = do
    Span -> Text -> Text -> m ()
forall (m :: * -> *) a.
(MonadIO m, ToAttribute a) =>
Span -> Text -> a -> m ()
addAttribute Span
sp Text
"ecluse.osv.ecosystem" (OsvEcosystem -> Text
osvWireName (CompileRun -> OsvEcosystem
crEcosystem CompileRun
run))
    Span -> Text -> Text -> m ()
forall (m :: * -> *) a.
(MonadIO m, ToAttribute a) =>
Span -> Text -> a -> m ()
addAttribute Span
sp Text
"ecluse.osv.source_host" (CompileRun -> Text
osvFeedLabel CompileRun
run)

-- The attempt runs whatever the requirement, so an ecosystem that could publish without scores
-- still carries them whenever the feed is up.
enrichOrRefuse :: (MonadResource m, MonadMask m, MonadUnliftIO m, KatipContext m) => CompileRun -> Maybe Span -> m EpssEnrichment
enrichOrRefuse :: forall (m :: * -> *).
(MonadResource m, MonadMask m, MonadUnliftIO m, KatipContext m) =>
CompileRun -> Maybe Span -> m EpssEnrichment
enrichOrRefuse CompileRun
run Maybe Span
mSpan = do
    acquired <- Manager -> Int -> String -> m (Either EpssFeedFailure EpssFeed)
forall (m :: * -> *).
(MonadResource m, MonadMask m, MonadUnliftIO m, KatipContext m) =>
Manager -> Int -> String -> m (Either EpssFeedFailure EpssFeed)
acquireEpssFeed (CompileRun -> Manager
crManager CompileRun
run) Int
maxEpssFeedBytes (CompileSources -> String
csEpssFeedUrl (CompileRun -> CompileSources
crSources CompileRun
run))
    forM_ mSpan $ \Span
sp -> Span -> Text -> Text -> m ()
forall (m :: * -> *) a.
(MonadIO m, ToAttribute a) =>
Span -> Text -> a -> m ()
addAttribute Span
sp Text
"ecluse.osv.epss_status" (EpssStatus -> Text
renderEpssStatus ((EpssFeedFailure -> EpssStatus)
-> (EpssFeed -> EpssStatus)
-> Either EpssFeedFailure EpssFeed
-> EpssStatus
forall a c b. (a -> c) -> (b -> c) -> Either a b -> c
either (EpssStatus -> EpssFeedFailure -> EpssStatus
forall a b. a -> b -> a
const EpssStatus
EnrichmentUnavailable) (EpssStatus -> EpssFeed -> EpssStatus
forall a b. a -> b -> a
const EpssStatus
EnrichmentAvailable) Either EpssFeedFailure EpssFeed
acquired))
    enrichment <- either (refuseRequiredEpss run mSpan) pure (resolveEnrichment (crEpss run) acquired)
    case enrichment of
        EpssUnavailable EpssFeedFailure
failure ->
            Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
WarningS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"EPSS enrichment unavailable for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
ecosystem Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text -> EpssFeedFailure -> Text
renderEpssFeedFailure (CompileRun -> Text
epssFeedLabel CompileRun
run) EpssFeedFailure
failure Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
". No " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
ecosystem Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" rule depends on EPSS, so the artifact publishes without scores"))
        EpssEnriched EpssFeed
_ -> m ()
forall (f :: * -> *). Applicative f => f ()
pass
    pure enrichment
  where
    ecosystem :: Text
ecosystem = OsvEcosystem -> Text
osvWireName (CompileRun -> OsvEcosystem
crEcosystem CompileRun
run)

-- The throw leaves the candidate unrenamed, so nothing publishes and any previous artifact stays.
refuseRequiredEpss :: (KatipContext m) => CompileRun -> Maybe Span -> EpssFeedFailure -> m a
refuseRequiredEpss :: forall (m :: * -> *) a.
KatipContext m =>
CompileRun -> Maybe Span -> EpssFeedFailure -> m a
refuseRequiredEpss CompileRun
run Maybe Span
mSpan EpssFeedFailure
failure = do
    Maybe Span -> (Span -> m ()) -> m ()
forall (t :: * -> *) (m :: * -> *) a b.
(Foldable t, Monad m) =>
t a -> (a -> m b) -> m ()
forM_ Maybe Span
mSpan ((Span -> m ()) -> m ()) -> (Span -> m ()) -> m ()
forall a b. (a -> b) -> a -> b
$ \Span
sp -> Span -> SpanStatus -> m ()
forall (m :: * -> *). MonadIO m => Span -> SpanStatus -> m ()
setStatus Span
sp (Text -> SpanStatus
Error Text
"required EPSS enrichment unavailable, compile abandoned")
    Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"Aborting OSV compile: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> PilotEpssRequired -> Text
renderEpssRequired PilotEpssRequired
refusal))
    -- A fault for the caller: the scheduled loop retries on its cadence and a one-shot run exits non-zero.
    PilotEpssRequired -> m a
forall (m :: * -> *) e a. (MonadIO m, Exception e) => e -> m a
throwIO PilotEpssRequired
refusal
  where
    refusal :: PilotEpssRequired
refusal =
        PilotEpssRequired
            { perEcosystem :: Text
perEcosystem = OsvEcosystem -> Text
osvWireName (CompileRun -> OsvEcosystem
crEcosystem CompileRun
run)
            , perFeed :: Text
perFeed = CompileRun -> Text
epssFeedLabel CompileRun
run
            , perFailure :: EpssFeedFailure
perFailure = EpssFeedFailure
failure
            }

epssFeedLabel :: CompileRun -> Text
epssFeedLabel :: CompileRun -> Text
epssFeedLabel CompileRun
run = Text -> Text
dialledAuthorityLabel (String -> Text
forall a. ToText a => a -> Text
toText (CompileSources -> String
csEpssFeedUrl (CompileRun -> CompileSources
crSources CompileRun
run)))

osvFeedLabel :: CompileRun -> Text
osvFeedLabel :: CompileRun -> Text
osvFeedLabel CompileRun
run = Text -> Text
dialledAuthorityLabel (String -> Text
forall a. ToText a => a -> Text
toText (CompileSources -> String
csOsvExportUrl (CompileRun -> CompileSources
crSources CompileRun
run)))

renderRequirement :: EpssRequirement -> Text
renderRequirement :: EpssRequirement -> Text
renderRequirement = \case
    EpssRequirement
EpssRequired -> Text
"required"
    EpssRequirement
EpssOptional -> Text
"optional"

-- The client's exception can hold the request a redirect reached. The typed fault is thrown outside any
-- handler, so nothing can annotate it with that exception.
ingestAdvisories :: (MonadResource m, MonadMask m, MonadUnliftIO m, KatipContext m) => CompileRun -> OsvIngest -> Connection -> m ContentDigest
ingestAdvisories :: forall (m :: * -> *).
(MonadResource m, MonadMask m, MonadUnliftIO m, KatipContext m) =>
CompileRun -> OsvIngest -> Connection -> m ContentDigest
ingestAdvisories CompileRun
run OsvIngest
ingest Connection
conn =
    m ContentDigest -> m (Either HttpException ContentDigest)
forall (m :: * -> *) e a.
(MonadUnliftIO m, Exception e) =>
m a -> m (Either e a)
try (Text -> RetryPolicyM m -> m ContentDigest -> m ContentDigest
forall (m :: * -> *) a.
(MonadMask m, KatipContext m) =>
Text -> RetryPolicyM m -> m a -> m a
withOsvRetry Text
feed RetryPolicyM m
forall (m :: * -> *). MonadIO m => RetryPolicyM m
defaultOsvRetryPolicy (CompileRun -> OsvIngest -> Connection -> m ContentDigest
forall (m :: * -> *).
(MonadResource m, MonadThrow m, KatipContext m) =>
CompileRun -> OsvIngest -> Connection -> m ContentDigest
restreamAdvisories CompileRun
run OsvIngest
ingest Connection
conn))
        m (Either HttpException ContentDigest)
-> (Either HttpException ContentDigest -> m ContentDigest)
-> m ContentDigest
forall a b. m a -> (a -> m b) -> m b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= (HttpException -> m ContentDigest)
-> (ContentDigest -> m ContentDigest)
-> Either HttpException ContentDigest
-> m ContentDigest
forall a c b. (a -> c) -> (b -> c) -> Either a b -> c
either (PilotOsvFetchFailed -> m ContentDigest
forall (m :: * -> *) e a. (MonadIO m, Exception e) => e -> m a
throwIO (PilotOsvFetchFailed -> m ContentDigest)
-> (HttpException -> PilotOsvFetchFailed)
-> HttpException
-> m ContentDigest
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Text -> HttpFault -> PilotOsvFetchFailed
PilotOsvFetchFailed Text
feed (HttpFault -> PilotOsvFetchFailed)
-> (HttpException -> HttpFault)
-> HttpException
-> PilotOsvFetchFailed
forall b c a. (b -> c) -> (a -> b) -> a -> c
. HttpException -> HttpFault
httpFault) ContentDigest -> m ContentDigest
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure
  where
    feed :: Text
feed = CompileRun -> Text
osvFeedLabel CompileRun
run

-- A failed attempt leaves committed batches. NULL bounds defeat deduplication, so each attempt
-- clears the table, the tally, and the source metadata before it re-streams.
restreamAdvisories :: (MonadResource m, MonadThrow m, KatipContext m) => CompileRun -> OsvIngest -> Connection -> m ContentDigest
restreamAdvisories :: forall (m :: * -> *).
(MonadResource m, MonadThrow m, KatipContext m) =>
CompileRun -> OsvIngest -> Connection -> m ContentDigest
restreamAdvisories CompileRun
run OsvIngest
ingest Connection
conn = do
    OsvIngest -> m ()
forall (m :: * -> *). MonadIO m => OsvIngest -> m ()
resetIngestStats OsvIngest
ingest
    OsvIngest -> m ()
forall (m :: * -> *). MonadIO m => OsvIngest -> m ()
resetOsvAttempt OsvIngest
ingest
    IO () -> m ()
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO () -> m ()) -> IO () -> m ()
forall a b. (a -> b) -> a -> b
$ Connection -> Query -> IO ()
execute_ Connection
conn Query
"DELETE FROM package_vulnerability_ranges"
    ConduitT () Void m ContentDigest -> m ContentDigest
forall (m :: * -> *) r. Monad m => ConduitT () Void m r -> m r
runConduit (ConduitT () Void m ContentDigest -> m ContentDigest)
-> ConduitT () Void m ContentDigest -> m ContentDigest
forall a b. (a -> b) -> a -> b
$
        ConduitT () ExtractedOsv m ContentDigest
-> ConduitT ExtractedOsv Void m ()
-> ConduitT () Void m ContentDigest
forall (m :: * -> *) a b r c.
Monad m =>
ConduitT a b m r -> ConduitT b c m () -> ConduitT a c m r
fuseUpstream
            (Maybe TracerProvider
-> Manager
-> OsvIngest
-> String
-> ConduitT () ExtractedOsv m ContentDigest
forall (m :: * -> *) i.
(MonadResource m, MonadThrow m, KatipContext m) =>
Maybe TracerProvider
-> Manager
-> OsvIngest
-> String
-> ConduitT i ExtractedOsv m ContentDigest
streamOsvUrl (CompileRun -> Maybe TracerProvider
crTracerProvider CompileRun
run) (CompileRun -> Manager
crManager CompileRun
run) OsvIngest
ingest (CompileSources -> String
csOsvExportUrl (CompileRun -> CompileSources
crSources CompileRun
run)))
            ( (ExtractedOsv -> Bool) -> ConduitT ExtractedOsv ExtractedOsv m ()
forall (m :: * -> *) a. Monad m => (a -> Bool) -> ConduitT a a m ()
CL.filter ((Text -> Text -> Bool
forall a. Eq a => a -> a -> Bool
== OsvEcosystem -> Text
osvExportDirectory (CompileRun -> OsvEcosystem
crEcosystem CompileRun
run)) (Text -> Bool) -> (ExtractedOsv -> Text) -> ExtractedOsv -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. ExtractedOsv -> Text
extEcosystem)
                ConduitT ExtractedOsv ExtractedOsv m ()
-> ConduitT ExtractedOsv Void m ()
-> ConduitT ExtractedOsv Void m ()
forall (m :: * -> *) a b c r.
Monad m =>
ConduitT a b m () -> ConduitT b c m r -> ConduitT a c m r
.| Int -> ConduitT ExtractedOsv [ExtractedOsv] m ()
forall (m :: * -> *) a. Monad m => Int -> ConduitT a [a] m ()
CL.chunksOf Int
2000
                ConduitT ExtractedOsv [ExtractedOsv] m ()
-> ConduitT [ExtractedOsv] Void m ()
-> ConduitT ExtractedOsv Void m ()
forall (m :: * -> *) a b c r.
Monad m =>
ConduitT a b m () -> ConduitT b c m r -> ConduitT a c m r
.| Connection -> ConduitT [ExtractedOsv] Void m ()
forall (m :: * -> *) o.
MonadIO m =>
Connection -> ConduitT [ExtractedOsv] o m ()
sinkSqlite Connection
conn
            )

data CompileConclusion = CompileConclusion
    { CompileConclusion -> Text
ccEcosystem :: Text
    , CompileConclusion -> CompileSources
ccSources :: CompileSources
    , CompileConclusion -> IngestStats
ccStats :: IngestStats
    , CompileConclusion -> EpssStatus
ccEpssStatus :: EpssStatus
    , CompileConclusion -> AdvisoryProvenance
ccProvenance :: AdvisoryProvenance
    , CompileConclusion -> SourceMarker
ccOsvMarker :: SourceMarker
    , CompileConclusion -> Maybe SourceMarker
ccEpssMarker :: Maybe SourceMarker
    , CompileConclusion -> QuietTime
ccQuietTime :: QuietTime
    , CompileConclusion -> UTCTime
ccNow :: UTCTime
    }

conclusionOf :: CompileRun -> UTCTime -> EpssEnrichment -> OsvAttempt -> IngestStats -> SourceMarker -> Maybe SourceMarker -> CompileConclusion
conclusionOf :: CompileRun
-> UTCTime
-> EpssEnrichment
-> OsvAttempt
-> IngestStats
-> SourceMarker
-> Maybe SourceMarker
-> CompileConclusion
conclusionOf CompileRun
run UTCTime
now EpssEnrichment
enrichment OsvAttempt
attempt IngestStats
stats SourceMarker
osvMarker Maybe SourceMarker
epssMarker =
    CompileConclusion
        { ccEcosystem :: Text
ccEcosystem = OsvEcosystem -> Text
osvWireName (CompileRun -> OsvEcosystem
crEcosystem CompileRun
run)
        , ccSources :: CompileSources
ccSources = CompileSources
sources
        , ccStats :: IngestStats
ccStats = IngestStats
stats
        , ccEpssStatus :: EpssStatus
ccEpssStatus = EpssEnrichment -> EpssStatus
enrichmentStatus EpssEnrichment
enrichment
        , ccProvenance :: AdvisoryProvenance
ccProvenance = CompileSources
-> EpssEnrichment -> OsvAttempt -> AdvisoryProvenance
passProvenance CompileSources
sources EpssEnrichment
enrichment OsvAttempt
attempt
        , ccOsvMarker :: SourceMarker
ccOsvMarker = SourceMarker
osvMarker
        , ccEpssMarker :: Maybe SourceMarker
ccEpssMarker = Maybe SourceMarker
epssMarker
        , ccQuietTime :: QuietTime
ccQuietTime = CompileRun -> QuietTime
crQuietTime CompileRun
run
        , ccNow :: UTCTime
ccNow = UTCTime
now
        }
  where
    sources :: CompileSources
sources = CompileRun -> CompileSources
crSources CompileRun
run

newCandidate :: FilePath -> IO FilePath
newCandidate :: String -> IO String
newCandidate String
outDir = do
    (path, created) <- String -> String -> IO (String, Handle)
openTempFile String
outDir String
".osv-candidate.db"
    hClose created
    pure path

removeCandidate :: FilePath -> IO ()
removeCandidate :: String -> IO ()
removeCandidate String
path = IO () -> (IOError -> IO ()) -> IO ()
forall a. IO a -> (IOError -> IO a) -> IO a
catchIOError (String -> IO ()
removeFile String
path) (IO () -> IOError -> IO ()
forall a b. a -> b -> a
const (IO () -> IOError -> IO ()) -> IO () -> IOError -> IO ()
forall a b. (a -> b) -> a -> b
$ () -> IO ()
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ())

-- Each source in the form 'CompileSources' recorded, because the artifact reaches every consumer.
-- A feed that never arrived described nothing, so it records no EPSS source or date.
passProvenance :: CompileSources -> EpssEnrichment -> OsvAttempt -> AdvisoryProvenance
passProvenance :: CompileSources
-> EpssEnrichment -> OsvAttempt -> AdvisoryProvenance
passProvenance CompileSources
sources EpssEnrichment
enrichment OsvAttempt
attempt =
    AdvisoryProvenance
        { apOsvSource :: Maybe Text
apOsvSource = Text -> Maybe Text
forall a. a -> Maybe a
Just (RecordedSource -> Text
renderRecordedSource (CompileSources -> RecordedSource
csOsvRecorded CompileSources
sources))
        , apOsvLastModified :: Maybe UTCTime
apOsvLastModified = OsvAttempt -> Maybe UTCTime
oaLastModified OsvAttempt
attempt
        , apOsvNewestModified :: Maybe UTCTime
apOsvNewestModified = OsvAttempt -> Maybe UTCTime
oaNewestModified OsvAttempt
attempt
        , apEpssSource :: Maybe Text
apEpssSource = RecordedSource -> Text
renderRecordedSource (CompileSources -> RecordedSource
csEpssRecorded CompileSources
sources) Text -> Maybe EpssFeed -> Maybe Text
forall a b. a -> Maybe b -> Maybe a
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ Maybe EpssFeed
feed
        , apEpssLastModified :: Maybe UTCTime
apEpssLastModified = EpssFeed -> Maybe UTCTime
efLastModified (EpssFeed -> Maybe UTCTime) -> Maybe EpssFeed -> Maybe UTCTime
forall (m :: * -> *) a b. Monad m => (a -> m b) -> m a -> m b
=<< Maybe EpssFeed
feed
        , apEpssScoreDate :: Maybe UTCTime
apEpssScoreDate = EpssFeed -> Maybe UTCTime
efScoreDate (EpssFeed -> Maybe UTCTime) -> Maybe EpssFeed -> Maybe UTCTime
forall (m :: * -> *) a b. Monad m => (a -> m b) -> m a -> m b
=<< Maybe EpssFeed
feed
        , apEpssModelVersion :: Maybe Text
apEpssModelVersion = EpssFeed -> Maybe Text
efModelVersion (EpssFeed -> Maybe Text) -> Maybe EpssFeed -> Maybe Text
forall (m :: * -> *) a b. Monad m => (a -> m b) -> m a -> m b
=<< Maybe EpssFeed
feed
        }
  where
    feed :: Maybe EpssFeed
feed = EpssEnrichment -> Maybe EpssFeed
enrichedFeed EpssEnrichment
enrichment

concludeCompile :: (KatipContext m) => AdvisoryCompileMetricsPort -> Maybe Span -> Connection -> CompileConclusion -> m CandidateFacts
concludeCompile :: forall (m :: * -> *).
KatipContext m =>
AdvisoryCompileMetricsPort
-> Maybe Span
-> Connection
-> CompileConclusion
-> m CandidateFacts
concludeCompile AdvisoryCompileMetricsPort
metrics Maybe Span
mSpan Connection
conn CompileConclusion
conclusion = do
    Maybe Span -> IngestStats -> m ()
forall (m :: * -> *).
MonadIO m =>
Maybe Span -> IngestStats -> m ()
recordCompileSpan Maybe Span
mSpan IngestStats
stats
    IO () -> m ()
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (AdvisoryCompileMetricsPort -> IngestStats -> IO ()
recordTallies AdvisoryCompileMetricsPort
metrics IngestStats
stats)
    rowCount <- IO Int -> m Int
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (Connection -> Query -> IO Int
countRows Connection
conn Query
"SELECT COUNT(*) FROM package_vulnerability_ranges")
    forM_ (compileRefusal stats rowCount) (refuseCompile metrics mSpan ecosystem stats)

    scoredCount <- liftIO (countRows conn "SELECT COUNT(*) FROM package_vulnerability_ranges WHERE epss_score IS NOT NULL")
    liftIO $ writeMeta conn conclusion rowCount scoredCount
    liftIO (acmpCompileRun metrics CompileCompleted)
    forM_ mSpan $ \Span
sp -> Span -> Text -> Text -> m ()
forall (m :: * -> *) a.
(MonadIO m, ToAttribute a) =>
Span -> Text -> a -> m ()
addAttribute Span
sp Text
"ecluse.osv.row_count" (Int -> Text
forall b a. (Show a, IsString b) => a -> b
show Int
rowCount :: Text)
    katipAddContext (sl "row_count" rowCount <> sl "epss_status" epssStatus <> dropFields ecosystem stats) $
        logFM InfoS (ls ("Compiled " <> show rowCount <> " advisory ranges for " <> ecosystem <> " (" <> renderDrops stats <> "), epss_status=" <> epssStatus))
    warnOnUnusableDates ecosystem stats
    logSourceAges ecosystem (sourceAges (ccNow conclusion) (ccQuietTime conclusion) (ccProvenance conclusion))
    pure
        CandidateFacts
            { cfEcosystem = ecosystem
            , cfOsv = ccOsvMarker conclusion
            , cfEpss = ccEpssMarker conclusion
            , cfRelevantRows = toInteger rowCount
            , cfScoredRows = toInteger scoredCount
            }
  where
    ecosystem :: Text
ecosystem = CompileConclusion -> Text
ccEcosystem CompileConclusion
conclusion
    stats :: IngestStats
stats = CompileConclusion -> IngestStats
ccStats CompileConclusion
conclusion
    epssStatus :: Text
epssStatus = EpssStatus -> Text
renderEpssStatus (CompileConclusion -> EpssStatus
ccEpssStatus CompileConclusion
conclusion)

countRows :: Connection -> Query -> IO Int
countRows :: Connection -> Query -> IO Int
countRows Connection
conn Query
statement = Int -> (Only Int -> Int) -> Maybe (Only Int) -> Int
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Int
0 Only Int -> Int
forall a. Only a -> a
fromOnly (Maybe (Only Int) -> Int)
-> ([Only Int] -> Maybe (Only Int)) -> [Only Int] -> Int
forall b c a. (b -> c) -> (a -> b) -> a -> c
. [Only Int] -> Maybe (Only Int)
forall a. [a] -> Maybe a
listToMaybe ([Only Int] -> Int) -> IO [Only Int] -> IO Int
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> (Connection -> Query -> IO [Only Int]
forall r. FromRow r => Connection -> Query -> IO [r]
query_ Connection
conn Query
statement :: IO [Only Int])

recordCompileSpan :: (MonadIO m) => Maybe Span -> IngestStats -> m ()
recordCompileSpan :: forall (m :: * -> *).
MonadIO m =>
Maybe Span -> IngestStats -> m ()
recordCompileSpan Maybe Span
mSpan IngestStats
stats = Maybe Span -> (Span -> m ()) -> m ()
forall (t :: * -> *) (m :: * -> *) a b.
(Foldable t, Monad m) =>
t a -> (a -> m b) -> m ()
forM_ Maybe Span
mSpan ((Span -> m ()) -> m ()) -> (Span -> m ()) -> m ()
forall a b. (a -> b) -> a -> b
$ \Span
sp -> do
    Span -> Text -> Text -> m ()
forall (m :: * -> *) a.
(MonadIO m, ToAttribute a) =>
Span -> Text -> a -> m ()
addAttribute Span
sp Text
"ecluse.osv.accepted" (Int -> Text
forall b a. (Show a, IsString b) => a -> b
show (IngestStats -> Int
statAccepted IngestStats
stats) :: Text)
    Span -> Text -> Text -> m ()
forall (m :: * -> *) a.
(MonadIO m, ToAttribute a) =>
Span -> Text -> a -> m ()
addAttribute Span
sp Text
"ecluse.osv.dropped_oversize" (Int -> Text
forall b a. (Show a, IsString b) => a -> b
show (IngestStats -> Int
statDroppedOversize IngestStats
stats) :: Text)
    Span -> Text -> Text -> m ()
forall (m :: * -> *) a.
(MonadIO m, ToAttribute a) =>
Span -> Text -> a -> m ()
addAttribute Span
sp Text
"ecluse.osv.dropped_malformed" (Int -> Text
forall b a. (Show a, IsString b) => a -> b
show (IngestStats -> Int
statDroppedMalformed IngestStats
stats) :: Text)
    Span -> Text -> Text -> m ()
forall (m :: * -> *) a.
(MonadIO m, ToAttribute a) =>
Span -> Text -> a -> m ()
addAttribute Span
sp Text
"ecluse.osv.unorderable" (Int -> Text
forall b a. (Show a, IsString b) => a -> b
show (IngestStats -> Int
statUnorderable IngestStats
stats) :: Text)

-- The refusal throws, so nothing after it in 'concludeCompile' runs: no metadata is written and
-- the candidate file is discarded unrenamed.
refuseCompile :: (KatipContext m) => AdvisoryCompileMetricsPort -> Maybe Span -> Text -> IngestStats -> Text -> m ()
refuseCompile :: forall (m :: * -> *).
KatipContext m =>
AdvisoryCompileMetricsPort
-> Maybe Span -> Text -> IngestStats -> Text -> m ()
refuseCompile AdvisoryCompileMetricsPort
metrics Maybe Span
mSpan Text
ecosystem IngestStats
stats Text
reason = do
    Maybe Span -> (Span -> m ()) -> m ()
forall (t :: * -> *) (m :: * -> *) a b.
(Foldable t, Monad m) =>
t a -> (a -> m b) -> m ()
forM_ Maybe Span
mSpan ((Span -> m ()) -> m ()) -> (Span -> m ()) -> m ()
forall a b. (a -> b) -> a -> b
$ \Span
sp -> Span -> SpanStatus -> m ()
forall (m :: * -> *). MonadIO m => Span -> SpanStatus -> m ()
setStatus Span
sp (Text -> SpanStatus
Error (Text
reason Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
", compile abandoned"))
    IO () -> m ()
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (AdvisoryCompileMetricsPort -> AdvisoryCompileResult -> IO ()
acmpCompileRun AdvisoryCompileMetricsPort
metrics AdvisoryCompileResult
CompileAborted)
    SimpleLogPayload -> m () -> m ()
forall i (m :: * -> *) a.
(LogItem i, KatipContext m) =>
i -> m a -> m a
katipAddContext (Text -> IngestStats -> SimpleLogPayload
dropFields Text
ecosystem IngestStats
stats) (m () -> m ()) -> m () -> m ()
forall a b. (a -> b) -> a -> b
$
        Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"Aborting OSV compile for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
ecosystem Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
reason Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" (" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> IngestStats -> Text
renderDrops IngestStats
stats Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
")"))
    PilotIngestAborted -> m ()
forall (m :: * -> *) e a. (MonadIO m, Exception e) => e -> m a
throwIO (IngestStats -> PilotIngestAborted
PilotIngestAborted IngestStats
stats)

compileRefusal :: IngestStats -> Int -> Maybe Text
compileRefusal :: IngestStats -> Int -> Maybe Text
compileRefusal IngestStats
stats Int
rowCount
    | IngestStats -> Bool
systemicDrop IngestStats
stats = Text -> Maybe Text
forall a. a -> Maybe a
Just Text
"systemic advisory drop rate"
    | Int
rowCount Int -> Int -> Bool
forall a. Eq a => a -> a -> Bool
== Int
0 = Text -> Maybe Text
forall a. a -> Maybe a
Just Text
"zero relevant advisory rows"
    | Bool
otherwise = Maybe Text
forall a. Maybe a
Nothing

-- One line per pass, not per record: a source whose dates have gone wrong writes many, and
-- the rows are kept regardless.
warnOnUnusableDates :: (KatipContext m) => Text -> IngestStats -> m ()
warnOnUnusableDates :: forall (m :: * -> *). KatipContext m => Text -> IngestStats -> m ()
warnOnUnusableDates Text
ecosystem IngestStats
stats =
    Bool -> m () -> m ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when (Int
unusable Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Int
0) (m () -> m ()) -> m () -> m ()
forall a b. (a -> b) -> a -> b
$
        Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
WarningS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"Ignoring the modified date of " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Int -> Text
forall b a. (Show a, IsString b) => a -> b
show Int
unusable Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
ecosystem Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" advisory record(s), unreadable or dated after this run's clock; their ranges are kept"))
  where
    unusable :: Int
unusable = IngestStats -> Int
statUnusableModified IngestStats
stats

-- The ages the sources declared, on every pass that published. A source past its threshold is
-- an operator alarm: raise the threshold for a slow ecosystem, or change the source.
logSourceAges :: (KatipContext m) => Text -> [SourceAge] -> m ()
logSourceAges :: forall (m :: * -> *). KatipContext m => Text -> [SourceAge] -> m ()
logSourceAges Text
ecosystem [SourceAge]
ages = [SourceAge] -> (SourceAge -> m ()) -> m ()
forall (t :: * -> *) (f :: * -> *) a b.
(Foldable t, Applicative f) =>
t a -> (a -> f b) -> f ()
for_ [SourceAge]
ages ((SourceAge -> m ()) -> m ()) -> (SourceAge -> m ()) -> m ()
forall a b. (a -> b) -> a -> b
$ \SourceAge
reading -> do
    Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
InfoS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
ecosystem Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> SourceAge -> Text
renderSourceAge SourceAge
reading))
    Bool -> m () -> m ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when (SourceAge -> Bool
sourceQuiet SourceAge
reading) (m () -> m ()) -> m () -> m ()
forall a b. (a -> b) -> a -> b
$
        Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
ecosystem Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> SourceAge -> Text
renderSourceAge SourceAge
reading Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
", so the source has gone quiet"))

-- An abandoned pass records its tallies too, and a pass with no drops records a zero, so
-- the drop series exists before the first drop.
recordTallies :: AdvisoryCompileMetricsPort -> IngestStats -> IO ()
recordTallies :: AdvisoryCompileMetricsPort -> IngestStats -> IO ()
recordTallies AdvisoryCompileMetricsPort
metrics IngestStats
stats = do
    AdvisoryCompileMetricsPort -> Int -> IO ()
acmpCompileAccepted AdvisoryCompileMetricsPort
metrics (IngestStats -> Int
statAccepted IngestStats
stats)
    AdvisoryCompileMetricsPort -> AdvisoryDropCause -> Int -> IO ()
acmpCompileDropped AdvisoryCompileMetricsPort
metrics AdvisoryDropCause
DropOversize (IngestStats -> Int
statDroppedOversize IngestStats
stats)
    AdvisoryCompileMetricsPort -> AdvisoryDropCause -> Int -> IO ()
acmpCompileDropped AdvisoryCompileMetricsPort
metrics AdvisoryDropCause
DropMalformed (IngestStats -> Int
statDroppedMalformed IngestStats
stats)

renderDrops :: IngestStats -> Text
renderDrops :: IngestStats -> Text
renderDrops IngestStats
s =
    Text
"accepted "
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Int -> Text
forall b a. (Show a, IsString b) => a -> b
show (IngestStats -> Int
statAccepted IngestStats
s)
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
", dropped "
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Int -> Text
forall b a. (Show a, IsString b) => a -> b
show (IngestStats -> Int
statDroppedOversize IngestStats
s)
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" oversize / "
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Int -> Text
forall b a. (Show a, IsString b) => a -> b
show (IngestStats -> Int
statDroppedMalformed IngestStats
s)
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" malformed, kept "
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Int -> Text
forall b a. (Show a, IsString b) => a -> b
show (IngestStats -> Int
statUnorderable IngestStats
s)
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" unorderable"

dropFields :: Text -> IngestStats -> SimpleLogPayload
dropFields :: Text -> IngestStats -> SimpleLogPayload
dropFields Text
ecosystem IngestStats
s =
    Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"ecosystem" Text
ecosystem
        SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Int -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"accepted" (IngestStats -> Int
statAccepted IngestStats
s)
        SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Int -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"dropped_oversize" (IngestStats -> Int
statDroppedOversize IngestStats
s)
        SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Int -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"dropped_malformed" (IngestStats -> Int
statDroppedMalformed IngestStats
s)
        SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Int -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"unorderable" (IngestStats -> Int
statUnorderable IngestStats
s)

initSchema :: Connection -> IO ()
initSchema :: Connection -> IO ()
initSchema Connection
conn = do
    Connection -> Query -> IO ()
execute_ Connection
conn (Text -> Query
Query Text
rangesTableDdl)
    -- A unique index rather than a composite PRIMARY KEY: @STRICT@ makes primary-key
    -- columns implicitly NOT NULL, and the three bound columns are legitimately NULL.
    Connection -> Query -> IO ()
execute_ Connection
conn Query
"CREATE UNIQUE INDEX uq_ranges_segment ON package_vulnerability_ranges(package_name, cve_id, introduced_version, fixed_version, last_affected_version)"
    Connection -> Query -> IO ()
execute_ Connection
conn Query
"CREATE INDEX idx_package_name ON package_vulnerability_ranges(package_name)"
    Connection -> Query -> IO ()
execute_ Connection
conn Query
"CREATE INDEX idx_package_fixed ON package_vulnerability_ranges(package_name, fixed_version)"
    Connection -> Query -> IO ()
execute_ Connection
conn (Text -> Query
Query Text
metaTableDdl)
    Connection -> Query -> IO ()
execute_ Connection
conn (String -> Query
forall a. IsString a => String -> a
fromString (String
"PRAGMA user_version = " String -> ShowS
forall a. Semigroup a => a -> a -> a
<> Int -> String
forall b a. (Show a, IsString b) => a -> b
show Int
osvSchemaEpoch))

-- Written once, after the stream completes and the refusals pass: the row count and the
-- source provenance are only meaningful for a complete artifact.
writeMeta :: Connection -> CompileConclusion -> Int -> Int -> IO ()
writeMeta :: Connection -> CompileConclusion -> Int -> Int -> IO ()
writeMeta Connection
conn CompileConclusion
conclusion Int
rowCount Int
scoredCount = do
    builtAt <- IO UTCTime
getCurrentTime
    nonce <- newPublicationNonce
    executeMany
        conn
        "INSERT INTO meta (key, value) VALUES (?, ?)"
        ( [ (renderMetaKey MetaPilotVersion, productVersion)
          , (renderMetaKey MetaEcosystem, ccEcosystem conclusion)
          , (renderMetaKey MetaBuiltAt, toText (iso8601Show builtAt))
          , (renderMetaKey MetaSourceUrl, dialledAuthorityLabel (toText (csOsvExportUrl sources)))
          , (renderMetaKey MetaEpssStatus, renderEpssStatus status)
          , (renderMetaKey MetaRowCount, show rowCount)
          , (renderMetaKey MetaScoredRowCount, show scoredCount)
          , (renderMetaKey MetaOsvDigest, sourceDigestHex (smDigest (ccOsvMarker conclusion)))
          , (renderMetaKey MetaPublicationNonce, renderPublicationNonce nonce)
          ]
            <> [(renderMetaKey MetaEpssSourceUrl, dialledAuthorityLabel (toText (csEpssFeedUrl sources))) | status == EnrichmentAvailable]
            <> [(renderMetaKey MetaEpssDigest, sourceDigestHex (smDigest marker)) | marker <- maybeToList (ccEpssMarker conclusion)]
            <> provenanceRows (ccProvenance conclusion)
        )
  where
    sources :: CompileSources
sources = CompileConclusion -> CompileSources
ccSources CompileConclusion
conclusion
    status :: EpssStatus
status = CompileConclusion -> EpssStatus
ccEpssStatus CompileConclusion
conclusion

sinkSqlite :: (MonadIO m) => Connection -> ConduitT [ExtractedOsv] o m ()
sinkSqlite :: forall (m :: * -> *) o.
MonadIO m =>
Connection -> ConduitT [ExtractedOsv] o m ()
sinkSqlite Connection
conn = ([ExtractedOsv] -> ConduitT [ExtractedOsv] o m ())
-> ConduitT [ExtractedOsv] o m ()
forall (m :: * -> *) i o r.
Monad m =>
(i -> ConduitT i o m r) -> ConduitT i o m ()
awaitForever (([ExtractedOsv] -> ConduitT [ExtractedOsv] o m ())
 -> ConduitT [ExtractedOsv] o m ())
-> ([ExtractedOsv] -> ConduitT [ExtractedOsv] o m ())
-> ConduitT [ExtractedOsv] o m ()
forall a b. (a -> b) -> a -> b
$ \[ExtractedOsv]
batch ->
    IO () -> ConduitT [ExtractedOsv] o m ()
forall a. IO a -> ConduitT [ExtractedOsv] o m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO () -> ConduitT [ExtractedOsv] o m ())
-> IO () -> ConduitT [ExtractedOsv] o m ()
forall a b. (a -> b) -> a -> b
$
        Connection -> IO () -> IO ()
forall a. Connection -> IO a -> IO a
withTransaction Connection
conn (IO () -> IO ()) -> IO () -> IO ()
forall a b. (a -> b) -> a -> b
$
            Connection
-> Query
-> [(Text, Text, Maybe Text, Maybe Text, Maybe Text, Maybe Double,
     Maybe Double)]
-> IO ()
forall q. ToRow q => Connection -> Query -> [q] -> IO ()
executeMany
                Connection
conn
                Query
"INSERT OR IGNORE INTO package_vulnerability_ranges (package_name, cve_id, introduced_version, fixed_version, last_affected_version, severity, epss_score) VALUES (?, ?, ?, ?, ?, ?, ?)"
                ((ExtractedOsv
 -> (Text, Text, Maybe Text, Maybe Text, Maybe Text, Maybe Double,
     Maybe Double))
-> [ExtractedOsv]
-> [(Text, Text, Maybe Text, Maybe Text, Maybe Text, Maybe Double,
     Maybe Double)]
forall a b. (a -> b) -> [a] -> [b]
map ExtractedOsv
-> (Text, Text, Maybe Text, Maybe Text, Maybe Text, Maybe Double,
    Maybe Double)
osvToRow [ExtractedOsv]
batch)

-- | One extracted segment as its artifact row. The upper bound fills at most one of @fixed_version@ and @last_affected_version@.
osvToRow :: ExtractedOsv -> (Text, Text, Maybe Text, Maybe Text, Maybe Text, Maybe Double, Maybe Double)
osvToRow :: ExtractedOsv
-> (Text, Text, Maybe Text, Maybe Text, Maybe Text, Maybe Double,
    Maybe Double)
osvToRow ExtractedOsv
osv = (ExtractedOsv -> Text
extPackage ExtractedOsv
osv, ExtractedOsv -> Text
extCveId ExtractedOsv
osv, ExtractedOsv -> Maybe Text
extIntroduced ExtractedOsv
osv, Maybe Text
fixed, Maybe Text
lastAffected, ExtractedOsv -> Maybe Double
extSeverity ExtractedOsv
osv, ExtractedOsv -> Maybe Double
extEpss ExtractedOsv
osv)
  where
    (Maybe Text
fixed, Maybe Text
lastAffected) = case ExtractedOsv -> UpperBound
extUpperBound ExtractedOsv
osv of
        FixedBefore Text
f -> (Text -> Maybe Text
forall a. a -> Maybe a
Just Text
f, Maybe Text
forall a. Maybe a
Nothing)
        LastAffected Text
la -> (Maybe Text
forall a. Maybe a
Nothing, Text -> Maybe Text
forall a. a -> Maybe a
Just Text
la)
        UpperBound
Unbounded -> (Maybe Text
forall a. Maybe a
Nothing, Maybe Text
forall a. Maybe a
Nothing)