module Ecluse.Core.Worker.Job (
JobOutcome (..),
outcomeOfFetchFault,
processJob,
processBatch,
workerPublishVisibilityBudget,
) where
import Data.Map.Strict qualified as Map
import Katip (Severity (DebugS, ErrorS, InfoS, WarningS), katipAddNamespace, logFM, ls)
import UnliftIO (withRunInIO)
import Ecluse.Core.Ecosystem (ecosystemName)
import Ecluse.Core.Package (Artifact (artFilename, artSize), Hash, pkgEcosystem, renderPackageName)
import Ecluse.Core.Package.Admission (
ArtifactAdmission (
AdmissionAdmit,
AdmissionBelowFloor,
AdmissionDenied,
AdmissionFileAbsent,
AdmissionIntegrityMissing,
AdmissionUndecidable
),
admitArtifact,
)
import Ecluse.Core.Queue (MirrorJob (jobArtifactFilename, jobArtifactUrl, jobPackage, jobTraceContext, jobVersion), MirrorQueue (ack, deadLetter, extendVisibility), QueueMessage (msgJob, msgReceipt), ReceiptHandle, Seconds (Seconds), qfDetail)
import Ecluse.Core.Registry (MirrorArtifact (MirrorArtifact, maFilename, maHashes, maSize), PublishFault (PublishRejected, PublishTransport, PublishUrlUnformable))
import Ecluse.Core.Registry.Metadata (VersionEvaluation (VersionMetadataUnavailable, VersionMissing, VersionPresent))
import Ecluse.Core.Registry.Publish (MirrorPublish (mpParseVersionList, mpProbeMetadata, mpPublishArtifact))
import Ecluse.Core.Rules.Types (Decision (Blocked, Undecidable), mkEvalContext)
import Ecluse.Core.Security (hostPortAddress)
import Ecluse.Core.Security.Egress (registryUrlText)
import Ecluse.Core.Telemetry.Metrics qualified as Metric
import Ecluse.Core.Telemetry.Record (WorkerMetricsPort (..), timedSeconds)
import Ecluse.Core.Telemetry.Span (JobSpanOutcome (JobSpanOutcome), WorkerTracingPort (..))
import Ecluse.Core.Version (renderVersion)
import Ecluse.Core.Worker.Fetch (ArtifactFetchFault (ArtifactOverCap, ArtifactUnavailable), fetchArtifactBytes)
import Ecluse.Core.Worker.Integrity (IntegrityResult (..), verifyIntegrity)
import Ecluse.Core.Worker.Types
processBatch :: [QueueMessage] -> WorkerM ()
processBatch :: [QueueMessage] -> WorkerM ()
processBatch = (QueueMessage -> WorkerM ()) -> [QueueMessage] -> WorkerM ()
forall (t :: * -> *) (f :: * -> *) a b.
(Foldable t, Applicative f) =>
(a -> f b) -> t a -> f ()
traverse_ ((QueueMessage -> WorkerM ()) -> [QueueMessage] -> WorkerM ())
-> (QueueMessage -> WorkerM ()) -> [QueueMessage] -> WorkerM ()
forall a b. (a -> b) -> a -> b
$ \QueueMessage
message -> do
QueueMessage -> WorkerM ()
processMessage QueueMessage
message
WorkerM ()
recordWorkerProgress
processMessage :: QueueMessage -> WorkerM ()
processMessage :: QueueMessage -> WorkerM ()
processMessage QueueMessage
message = do
metrics <- (WorkerRuntime -> WorkerMetricsPort) -> WorkerM WorkerMetricsPort
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> WorkerMetricsPort
wrMetrics
outcome <- processJob (msgReceipt message) (msgJob message)
liftIO (wmpMirrorJobProcessed metrics (jobResultMetric outcome))
case outcome of
JobOutcome
Succeeded -> ReceiptHandle -> WorkerM ()
ackMessage (QueueMessage -> ReceiptHandle
msgReceipt QueueMessage
message)
Dropped Text
reason -> do
Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"dropping unrecoverable mirror job: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
reason))
ReceiptHandle -> WorkerM ()
ackMessage (QueueMessage -> ReceiptHandle
msgReceipt QueueMessage
message)
DeadLettered Text
reason -> do
Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"dead-lettering unmirrorable mirror job (rides the backend's dead-letter terminus): " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
reason))
ReceiptHandle -> WorkerM ()
deadLetterMessage (QueueMessage -> ReceiptHandle
msgReceipt QueueMessage
message)
Retried Text
reason ->
Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
WarningS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"leaving mirror job un-acked for retry (redelivered by a durable queue, re-mirrored on next demand by the in-memory one): " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
reason))
jobResultMetric :: JobOutcome -> Metric.MirrorResult
jobResultMetric :: JobOutcome -> MirrorResult
jobResultMetric = \case
JobOutcome
Succeeded -> MirrorResult
Metric.Published
Dropped Text
_ -> MirrorResult
Metric.Failed
DeadLettered Text
_ -> MirrorResult
Metric.Failed
Retried Text
_ -> MirrorResult
Metric.Failed
ackMessage :: ReceiptHandle -> WorkerM ()
ackMessage :: ReceiptHandle -> WorkerM ()
ackMessage ReceiptHandle
receipt = do
queue <- (WorkerRuntime -> MirrorQueue) -> WorkerM MirrorQueue
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> MirrorQueue
wrQueue
acked <- liftIO (ack queue receipt)
whenLeft_ acked $ \QueueFault
fault ->
Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
WarningS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"ack failed; the processed message will redeliver (harmless, publishing is idempotent): " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> QueueFault -> Text
qfDetail QueueFault
fault))
deadLetterMessage :: ReceiptHandle -> WorkerM ()
deadLetterMessage :: ReceiptHandle -> WorkerM ()
deadLetterMessage ReceiptHandle
receipt = do
queue <- (WorkerRuntime -> MirrorQueue) -> WorkerM MirrorQueue
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> MirrorQueue
wrQueue
outcome <- liftIO (deadLetter queue receipt)
whenLeft_ outcome $ \QueueFault
fault ->
Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
WarningS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"dead-letter realisation failed; the message redelivers and re-fails terminally (harmless): " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> QueueFault -> Text
qfDetail QueueFault
fault))
data JobOutcome
=
Succeeded
|
Dropped Text
|
DeadLettered Text
|
Retried Text
deriving stock (JobOutcome -> JobOutcome -> Bool
(JobOutcome -> JobOutcome -> Bool)
-> (JobOutcome -> JobOutcome -> Bool) -> Eq JobOutcome
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: JobOutcome -> JobOutcome -> Bool
== :: JobOutcome -> JobOutcome -> Bool
$c/= :: JobOutcome -> JobOutcome -> Bool
/= :: JobOutcome -> JobOutcome -> Bool
Eq, Int -> JobOutcome -> ShowS
[JobOutcome] -> ShowS
JobOutcome -> String
(Int -> JobOutcome -> ShowS)
-> (JobOutcome -> String)
-> ([JobOutcome] -> ShowS)
-> Show JobOutcome
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> JobOutcome -> ShowS
showsPrec :: Int -> JobOutcome -> ShowS
$cshow :: JobOutcome -> String
show :: JobOutcome -> String
$cshowList :: [JobOutcome] -> ShowS
showList :: [JobOutcome] -> ShowS
Show)
processJob :: ReceiptHandle -> MirrorJob -> WorkerM JobOutcome
processJob :: ReceiptHandle -> MirrorJob -> WorkerM JobOutcome
processJob ReceiptHandle
receipt MirrorJob
job = Namespace -> WorkerM JobOutcome -> WorkerM JobOutcome
forall (m :: * -> *) a. KatipContext m => Namespace -> m a -> m a
katipAddNamespace Namespace
"job" (WorkerM JobOutcome -> WorkerM JobOutcome)
-> WorkerM JobOutcome -> WorkerM JobOutcome
forall a b. (a -> b) -> a -> b
$ do
Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
DebugS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"starting mirror job for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job))
tracing <- (WorkerRuntime -> WorkerTracingPort) -> WorkerM WorkerTracingPort
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> WorkerTracingPort
wrTracing
runtime <- ask
withRunInIO $ \forall a. WorkerM a -> IO a
runInIO ->
WorkerTracingPort
-> forall a.
PackageName
-> Version
-> Maybe RemoteSpanContext
-> (a -> JobSpanOutcome)
-> IO a
-> IO a
wtpMirrorJobSpan WorkerTracingPort
tracing (MirrorJob -> PackageName
jobPackage MirrorJob
job) (MirrorJob -> Version
jobVersion MirrorJob
job) (MirrorJob -> Maybe RemoteSpanContext
jobTraceContext MirrorJob
job) JobOutcome -> JobSpanOutcome
jobSpanOutcome (IO JobOutcome -> IO JobOutcome) -> IO JobOutcome -> IO JobOutcome
forall a b. (a -> b) -> a -> b
$
WorkerM JobOutcome -> IO JobOutcome
forall a. WorkerM a -> IO a
runInIO (WorkerM JobOutcome -> IO JobOutcome)
-> WorkerM JobOutcome -> IO JobOutcome
forall a b. (a -> b) -> a -> b
$
WorkerRuntime
-> forall (m :: * -> *) a.
(KatipContext m, MonadIO m) =>
m a -> m a
wrInjectTraceContext WorkerRuntime
runtime (ReceiptHandle -> MirrorJob -> WorkerM JobOutcome
reevaluateThenMirror ReceiptHandle
receipt MirrorJob
job)
where
jobSpanOutcome :: JobOutcome -> JobSpanOutcome
jobSpanOutcome :: JobOutcome -> JobSpanOutcome
jobSpanOutcome = \case
JobOutcome
Succeeded -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"succeeded" Maybe Text
forall a. Maybe a
Nothing
Dropped Text
reason -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"dropped" (Text -> Maybe Text
forall a. a -> Maybe a
Just Text
reason)
DeadLettered Text
reason -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"dead-lettered" (Text -> Maybe Text
forall a. a -> Maybe a
Just Text
reason)
Retried Text
reason -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"retried" (Text -> Maybe Text
forall a. a -> Maybe a
Just Text
reason)
data ReevalOutcome
= ReevalAdmit MirrorArtifact
| ReevalDrop Text
| ReevalRetry Text
reevaluateThenMirror :: ReceiptHandle -> MirrorJob -> WorkerM JobOutcome
reevaluateThenMirror :: ReceiptHandle -> MirrorJob -> WorkerM JobOutcome
reevaluateThenMirror ReceiptHandle
receipt MirrorJob
job = do
policies <- (WorkerRuntime -> WorkerPolicies) -> WorkerM WorkerPolicies
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> WorkerPolicies
wrPolicies
case Map.lookup (pkgEcosystem (jobPackage job)) policies of
Maybe WorkerPolicy
Nothing ->
JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
Dropped (Text
"no rule policy is configured for the " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Ecosystem -> Text
ecosystemName (PackageName -> Ecosystem
pkgEcosystem (MirrorJob -> PackageName
jobPackage MirrorJob
job)) Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" ecosystem; refusing to mirror " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job))
Just WorkerPolicy
policy ->
WorkerPolicy -> MirrorJob -> WorkerM Bool
alreadyMirrored WorkerPolicy
policy MirrorJob
job WorkerM Bool -> (Bool -> WorkerM JobOutcome) -> WorkerM JobOutcome
forall a b. WorkerM a -> (a -> WorkerM b) -> WorkerM b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= \case
Bool
True -> do
Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
InfoS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"already present at the mirror target, acking without re-publish: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job))
JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure JobOutcome
Succeeded
Bool
False ->
WorkerPolicy -> MirrorJob -> WorkerM ReevalOutcome
reevaluatePolicy WorkerPolicy
policy MirrorJob
job WorkerM ReevalOutcome
-> (ReevalOutcome -> WorkerM JobOutcome) -> WorkerM JobOutcome
forall a b. WorkerM a -> (a -> WorkerM b) -> WorkerM b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= \case
ReevalAdmit MirrorArtifact
admitted -> WorkerPolicy
-> ReceiptHandle
-> MirrorJob
-> MirrorArtifact
-> WorkerM JobOutcome
mirrorArtifact WorkerPolicy
policy ReceiptHandle
receipt MirrorJob
job MirrorArtifact
admitted
ReevalDrop Text
reason -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
Dropped Text
reason)
ReevalRetry Text
reason -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
Retried Text
reason)
alreadyMirrored :: WorkerPolicy -> MirrorJob -> WorkerM Bool
alreadyMirrored :: WorkerPolicy -> MirrorJob -> WorkerM Bool
alreadyMirrored WorkerPolicy
policy MirrorJob
job = do
probed <- IO (Either FetchFault RegistryResponse)
-> WorkerM (Either FetchFault RegistryResponse)
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (MirrorPublish
-> PackageName -> IO (Either FetchFault RegistryResponse)
mpProbeMetadata (WorkerPolicy -> MirrorPublish
wpPublish WorkerPolicy
policy) (MirrorJob -> PackageName
jobPackage MirrorJob
job))
case probed of
Left FetchFault
fault -> do
Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
DebugS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"mirror presence probe did not confirm " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"; falling through to full re-evaluation: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> FetchFault -> Text
forall b a. (Show a, IsString b) => a -> b
show FetchFault
fault))
Bool -> WorkerM Bool
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure Bool
False
Right RegistryResponse
response -> case MirrorPublish -> RegistryResponse -> Either ParseError [Version]
mpParseVersionList (WorkerPolicy -> MirrorPublish
wpPublish WorkerPolicy
policy) RegistryResponse
response of
Left ParseError
_ -> Bool -> WorkerM Bool
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure Bool
False
Right [Version]
versions -> Bool -> WorkerM Bool
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (MirrorJob -> Version
jobVersion MirrorJob
job Version -> [Version] -> Bool
forall (f :: * -> *) a.
(Foldable f, DisallowElem f, Eq a) =>
a -> f a -> Bool
`elem` [Version]
versions)
reevaluatePolicy :: WorkerPolicy -> MirrorJob -> WorkerM ReevalOutcome
reevaluatePolicy :: WorkerPolicy -> MirrorJob -> WorkerM ReevalOutcome
reevaluatePolicy WorkerPolicy
policy MirrorJob
job
| Bool -> Bool
not (WorkerPolicy -> Maybe HostPort -> Bool
wpArtifactHostHonoured WorkerPolicy
policy (Text -> Maybe HostPort
hostPortAddress (RegistryUrl -> Text
registryUrlText (MirrorJob -> RegistryUrl
jobArtifactUrl MirrorJob
job)))) =
ReevalOutcome -> WorkerM ReevalOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> ReevalOutcome
ReevalDrop (Text
"the tarball-host policy refuses the artifact host of " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" (" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> RegistryUrl -> Text
registryUrlText (MirrorJob -> RegistryUrl
jobArtifactUrl MirrorJob
job) Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"); refusing to fetch or mirror it"))
| Bool
otherwise = do
evaluation <- IO VersionEvaluation -> WorkerM VersionEvaluation
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (WorkerPolicy -> PackageName -> Version -> IO VersionEvaluation
wpResolveVersion WorkerPolicy
policy (MirrorJob -> PackageName
jobPackage MirrorJob
job) (MirrorJob -> Version
jobVersion MirrorJob
job))
case evaluation of
VersionEvaluation
VersionMetadataUnavailable ->
ReevalOutcome -> WorkerM ReevalOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> ReevalOutcome
ReevalRetry (Text
"could not re-fetch metadata to re-evaluate current policy for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job))
VersionEvaluation
VersionMissing ->
ReevalOutcome -> WorkerM ReevalOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> ReevalOutcome
ReevalDrop (Text
"the public upstream no longer offers " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"; refusing to mirror a withdrawn version"))
VersionPresent PackageDetails
details -> do
ctx <- IO EvalContext -> WorkerM EvalContext
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO UTCTime -> IO (Maybe DbEtag) -> IO EvalContext
mkEvalContext (WorkerPolicy -> IO UTCTime
wpNow WorkerPolicy
policy) (Maybe DbEtag -> IO (Maybe DbEtag)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure Maybe DbEtag
forall a. Maybe a
Nothing))
admission <-
liftIO
( admitArtifact
ctx
(wpRules policy)
(wpMinIntegrity policy)
(jobArtifactFilename job)
details
)
pure (outcomeOfAdmission job admission)
outcomeOfAdmission :: MirrorJob -> ArtifactAdmission -> ReevalOutcome
outcomeOfAdmission :: MirrorJob -> ArtifactAdmission -> ReevalOutcome
outcomeOfAdmission MirrorJob
job = \case
AdmissionAdmit Artifact
artifact NonEmpty Hash
digests -> MirrorArtifact -> ReevalOutcome
ReevalAdmit (Artifact -> NonEmpty Hash -> MirrorArtifact
readmittedDescriptor Artifact
artifact NonEmpty Hash
digests)
AdmissionDenied (Blocked Text
ruleName Text
reason) ->
Text -> ReevalOutcome
ReevalDrop (Text
"current policy denies " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": blocked by " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
ruleName Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" (" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
reason Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
")")
AdmissionDenied Decision
_ ->
Text -> ReevalOutcome
ReevalDrop (Text
"current policy denies " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": no rule admits it")
AdmissionUndecidable (Undecidable Transience
_ Text
reason) ->
Text -> ReevalOutcome
ReevalRetry (Text
"current policy could not be evaluated for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
reason)
AdmissionUndecidable Decision
_ ->
Text -> ReevalOutcome
ReevalRetry (Text
"current policy could not be evaluated for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job)
ArtifactAdmission
AdmissionFileAbsent ->
Text -> ReevalOutcome
ReevalDrop (Text
"the public upstream no longer offers the admitted artifact file of " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"; refusing to mirror a withdrawn artifact")
ArtifactAdmission
AdmissionBelowFloor ->
Text -> ReevalOutcome
ReevalDrop (Text
"current admission policy refuses " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": its strongest integrity digest is below the configured public floor")
ArtifactAdmission
AdmissionIntegrityMissing ->
Text -> ReevalOutcome
ReevalDrop (Text
"current admission policy refuses " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": it no longer carries any integrity digest")
readmittedDescriptor :: Artifact -> NonEmpty Hash -> MirrorArtifact
readmittedDescriptor :: Artifact -> NonEmpty Hash -> MirrorArtifact
readmittedDescriptor Artifact
artifact NonEmpty Hash
digests =
MirrorArtifact
{ maFilename :: Text
maFilename = Artifact -> Text
artFilename Artifact
artifact
, maHashes :: NonEmpty Hash
maHashes = NonEmpty Hash
digests
, maSize :: Maybe Int
maSize = Artifact -> Maybe Int
artSize Artifact
artifact
}
outcomeOfFetchFault :: ArtifactFetchFault -> JobOutcome
outcomeOfFetchFault :: ArtifactFetchFault -> JobOutcome
outcomeOfFetchFault = \case
ArtifactOverCap Text
reason -> Text -> JobOutcome
DeadLettered Text
reason
ArtifactUnavailable Text
reason -> Text -> JobOutcome
Retried Text
reason
mirrorArtifact :: WorkerPolicy -> ReceiptHandle -> MirrorJob -> MirrorArtifact -> WorkerM JobOutcome
mirrorArtifact :: WorkerPolicy
-> ReceiptHandle
-> MirrorJob
-> MirrorArtifact
-> WorkerM JobOutcome
mirrorArtifact WorkerPolicy
policy ReceiptHandle
receipt MirrorJob
job MirrorArtifact
admitted = do
Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
DebugS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"fetching artifact bytes from " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> RegistryUrl -> Text
registryUrlText (MirrorJob -> RegistryUrl
jobArtifactUrl MirrorJob
job)))
fetched <- Limits
-> (Limits
-> Manager
-> Text
-> Maybe Secret
-> Text
-> Either UrlFormationError Request)
-> RegistryUrl
-> WorkerM (Either ArtifactFetchFault ByteString)
fetchArtifactBytes (WorkerPolicy -> Limits
wpArtifactLimits WorkerPolicy
policy) (WorkerPolicy
-> Limits
-> Manager
-> Text
-> Maybe Secret
-> Text
-> Either UrlFormationError Request
wpBuildArtifactRequest WorkerPolicy
policy) (MirrorJob -> RegistryUrl
jobArtifactUrl MirrorJob
job)
case fetched of
Left ArtifactFetchFault
fault -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (ArtifactFetchFault -> JobOutcome
outcomeOfFetchFault ArtifactFetchFault
fault)
Right ByteString
bytes ->
case NonEmpty Hash -> ByteString -> IntegrityResult
verifyIntegrity (MirrorArtifact -> NonEmpty Hash
maHashes MirrorArtifact
admitted) ByteString
bytes of
IntegrityMismatch Text
detail -> do
Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"artifact integrity mismatch, refusing to publish: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
detail))
JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
Dropped (Text
"integrity mismatch: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
detail))
IntegrityResult
IntegrityVerified -> WorkerPolicy
-> ReceiptHandle
-> MirrorJob
-> MirrorArtifact
-> ByteString
-> WorkerM JobOutcome
publishVerified WorkerPolicy
policy ReceiptHandle
receipt MirrorJob
job MirrorArtifact
admitted ByteString
bytes
publishVerified :: WorkerPolicy -> ReceiptHandle -> MirrorJob -> MirrorArtifact -> ByteString -> WorkerM JobOutcome
publishVerified :: WorkerPolicy
-> ReceiptHandle
-> MirrorJob
-> MirrorArtifact
-> ByteString
-> WorkerM JobOutcome
publishVerified WorkerPolicy
policy ReceiptHandle
receipt MirrorJob
job MirrorArtifact
admitted ByteString
bytes = do
ReceiptHandle -> WorkerM ()
holdForLongPublish ReceiptHandle
receipt
metrics <- (WorkerRuntime -> WorkerMetricsPort) -> WorkerM WorkerMetricsPort
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> WorkerMetricsPort
wrMetrics
(result, seconds) <- timedSeconds (liftIO (mpPublishArtifact (wpPublish policy) (jobPackage job) (jobVersion job) admitted bytes))
liftIO (wmpMirrorPublishDuration metrics seconds)
case result of
Right () -> do
Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
InfoS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"mirrored artifact published: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job))
JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure JobOutcome
Succeeded
Left (PublishRejected PublishError
err) -> do
ReceiptHandle -> WorkerM ()
releaseForRetry ReceiptHandle
receipt
JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
Retried (Text
"registry rejected publish: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> PublishError -> Text
forall b a. (Show a, IsString b) => a -> b
show PublishError
err))
Left (PublishTransport TransportFault
fault) -> do
ReceiptHandle -> WorkerM ()
releaseForRetry ReceiptHandle
receipt
JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
Retried (Text
"publish transport failure: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> TransportFault -> Text
forall b a. (Show a, IsString b) => a -> b
show TransportFault
fault))
Left (PublishUrlUnformable UrlFormationError
urlErr) ->
JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
Dropped (Text
"unformable publish URL: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> UrlFormationError -> Text
forall b a. (Show a, IsString b) => a -> b
show UrlFormationError
urlErr))
holdForLongPublish :: ReceiptHandle -> WorkerM ()
holdForLongPublish :: ReceiptHandle -> WorkerM ()
holdForLongPublish ReceiptHandle
receipt = do
queue <- (WorkerRuntime -> MirrorQueue) -> WorkerM MirrorQueue
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> MirrorQueue
wrQueue
_ <- liftIO (extendVisibility queue receipt workerPublishVisibilityBudget)
pass
workerPublishVisibilityBudget :: Seconds
workerPublishVisibilityBudget :: Seconds
workerPublishVisibilityBudget = Int -> Seconds
Seconds Int
300
releaseForRetry :: ReceiptHandle -> WorkerM ()
releaseForRetry :: ReceiptHandle -> WorkerM ()
releaseForRetry ReceiptHandle
receipt = do
queue <- (WorkerRuntime -> MirrorQueue) -> WorkerM MirrorQueue
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> MirrorQueue
wrQueue
_ <- liftIO (extendVisibility queue receipt (Seconds 0))
pass
renderJob :: MirrorJob -> Text
renderJob :: MirrorJob -> Text
renderJob MirrorJob
job = PackageName -> Text
renderPackageName (MirrorJob -> PackageName
jobPackage MirrorJob
job) Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"@" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Version -> Text
renderVersion (MirrorJob -> Version
jobVersion MirrorJob
job)