-- SPDX-FileCopyrightText: 2026 Alexandra de Wit
--
-- SPDX-License-Identifier: MIT

{- | Decide mirror admission, preparation, integrity, and publication outcomes.
The lease controller owns the receipt, and "Ecluse.Core.Worker.Realise" applies queue outcomes.
-}
module Ecluse.Core.Worker.Job (
    JobOutcome (..),
    RetryLeg (..),
    mirrorLatest,
    outcomeOfAdmission,
    outcomeOfFetchFault,
    processJob,
) where

import Control.Monad (foldM)
import Control.Monad.Except (throwError)
import Data.Aeson (object, (.=))
import Data.Map.Strict qualified as Map
import Data.Traversable (for)
import Katip (Severity (DebugS, ErrorS, InfoS), SimpleLogPayload, katipAddContext, katipAddNamespace, logFM, ls, sl)
import UnliftIO (withRunInIO)

import Ecluse.Core.Ecosystem (ecosystemName)
import Ecluse.Core.Fault (tfCause)
import Ecluse.Core.Mirror.State
import Ecluse.Core.Mirror.State.Format (MirrorStateFormat (msfCompanions))
import Ecluse.Core.Package (Artifact (artFilename, artHashes, artSize, artUrl), Hash, PackageDetails (pkgArtifacts, pkgVersion), PackageInfo (..), hashAlg, hashValue, pkgEcosystem, renderHashAlg, renderPackageName)
import Ecluse.Core.Package.Admission (
    ArtifactAdmission (
        AdmissionAdmit,
        AdmissionBelowFloor,
        AdmissionDenied,
        AdmissionFileAbsent,
        AdmissionIntegrityMissing,
        AdmissionUndecidable
    ),
    admissionTransience,
    admitArtifactWithDecision,
 )
import Ecluse.Core.Package.Integrity (VersionIntegrity (..), classifyDigests, partitionByFloor)
import Ecluse.Core.Queue (MirrorJob (jobPackage, jobTraceContext, jobTriggerFilename, jobVersion))
import Ecluse.Core.Registry (
    FetchFault (FetchBoundExceeded, FetchCooldown, FetchTransport, FetchUrlUnformable),
    MirrorArtifact (MirrorArtifact, maFilename, maHashes, maSize),
    PublishFault (PublishFetch, PublishIntegrityMismatch, PublishRejected, PublishSourceUnavailable),
    renderUrlFormationError,
 )
import Ecluse.Core.Registry.Adapter.Capability (AdapterArtifact (artifactByUrl))
import Ecluse.Core.Registry.Exchange.Types (ExchangeClient (ecDeadlines))
import Ecluse.Core.Registry.Metadata (
    MetadataError (..),
    VersionDoc (vdDetails),
    VersionEvaluation (VersionCooling, VersionMetadataUnavailable, VersionMissing, VersionPresent),
    versionTransience,
 )
import Ecluse.Core.Registry.MirrorUpload
import Ecluse.Core.Registry.Publish (
    MirrorCompanion (..),
    MirrorPublish (mpMirrorCompanion, mpNativeComplete, mpPrepareVersion, mpProbeInventory),
    PreparedMirrorUpload,
    PreparedMirrorVersion,
    PublishPlan (PublishPlan),
    pmvInventory,
    pmvPrepareUpload,
    publishPrepared,
 )
import Ecluse.Core.Registry.Request (sidecarUrl)
import Ecluse.Core.Rules.Outage (logSkippedCheckEvidence)
import Ecluse.Core.Rules.Render (renderDecision, renderReason)
import Ecluse.Core.Rules.Types (Decision (Admitted, Blocked, BlockedByDefault, Undecidable), Transience (WillResolve, WontResolve), identityEvidence, mkEvalContext, skippedChecks)
import Ecluse.Core.Security (Limits (maxMetadataBytes, maxMirrorArtifactBytes), authorityLabel, hostPortAddress)
import Ecluse.Core.Security.Egress (RegistryUrl, registryUrlText)
import Ecluse.Core.Server.Path (Filename, mkFilename, unFilename)
import Ecluse.Core.Telemetry.Record (WorkerMetricsPort (..), timedSeconds)
import Ecluse.Core.Telemetry.Span (JobSpanOutcome (JobSpanOutcome), WorkerTracingPort (..))
import Ecluse.Core.Version (Version, renderVersion, selectLatest)
import Ecluse.Core.Worker.Fetch (fetchArtifactBytesCharged)
import Ecluse.Core.Worker.Managed
import Ecluse.Core.Worker.Types

-- | A mirror outcome determines acknowledgement, redelivery, or dead-letter handling.
data JobOutcome
    = -- | Completion proof covers every expected file and companion, including idempotent redelivery.
      Succeeded
    | -- | A non-retryable rejection. The worker alarms and acknowledges the message.
      Dropped Text
    | -- | The source version object is unavailable. Retire the job separately from a policy denial.
      SourceUnavailable Text
    | -- | A terminal fault sent to the dead-letter handler instead of silently deleted.
      DeadLettered Text
    | -- | A transient fault leaves the message unacknowledged and retains the failed leg.
      Retried RetryLeg Text
    deriving stock (JobOutcome -> JobOutcome -> Bool
(JobOutcome -> JobOutcome -> Bool)
-> (JobOutcome -> JobOutcome -> Bool) -> Eq JobOutcome
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: JobOutcome -> JobOutcome -> Bool
== :: JobOutcome -> JobOutcome -> Bool
$c/= :: JobOutcome -> JobOutcome -> Bool
/= :: JobOutcome -> JobOutcome -> Bool
Eq, Int -> JobOutcome -> ShowS
[JobOutcome] -> ShowS
JobOutcome -> String
(Int -> JobOutcome -> ShowS)
-> (JobOutcome -> String)
-> ([JobOutcome] -> ShowS)
-> Show JobOutcome
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> JobOutcome -> ShowS
showsPrec :: Int -> JobOutcome -> ShowS
$cshow :: JobOutcome -> String
show :: JobOutcome -> String
$cshowList :: [JobOutcome] -> ShowS
showList :: [JobOutcome] -> ShowS
Show)

-- | The failed leg determines whether the worker resets queue visibility.
data RetryLeg
    = -- | A failure before publication keeps the lease until redelivery.
      BeforePublish
    | -- | A transient publication failure releases the message for immediate redelivery.
      AfterPublish
    deriving stock (RetryLeg -> RetryLeg -> Bool
(RetryLeg -> RetryLeg -> Bool)
-> (RetryLeg -> RetryLeg -> Bool) -> Eq RetryLeg
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: RetryLeg -> RetryLeg -> Bool
== :: RetryLeg -> RetryLeg -> Bool
$c/= :: RetryLeg -> RetryLeg -> Bool
/= :: RetryLeg -> RetryLeg -> Bool
Eq, Int -> RetryLeg -> ShowS
[RetryLeg] -> ShowS
RetryLeg -> String
(Int -> RetryLeg -> ShowS)
-> (RetryLeg -> String) -> ([RetryLeg] -> ShowS) -> Show RetryLeg
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> RetryLeg -> ShowS
showsPrec :: Int -> RetryLeg -> ShowS
$cshow :: RetryLeg -> String
show :: RetryLeg -> String
$cshowList :: [RetryLeg] -> ShowS
showList :: [RetryLeg] -> ShowS
Show)

-- Incomplete versions mark the span errored even if some physical uploads succeeded.
jobSpanOutcome :: JobOutcome -> JobSpanOutcome
jobSpanOutcome :: JobOutcome -> JobSpanOutcome
jobSpanOutcome = \case
    JobOutcome
Succeeded -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"succeeded" Maybe Text
forall a. Maybe a
Nothing
    Dropped Text
reason -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"dropped" (Text -> Maybe Text
forall a. a -> Maybe a
Just Text
reason)
    SourceUnavailable Text
reason -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"source-unavailable" (Text -> Maybe Text
forall a. a -> Maybe a
Just Text
reason)
    DeadLettered Text
reason -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"dead-lettered" (Text -> Maybe Text
forall a. a -> Maybe a
Just Text
reason)
    Retried RetryLeg
_ Text
reason -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"retried" (Text -> Maybe Text
forall a. a -> Maybe a
Just Text
reason)

-- | Recheck current policy before publishing because queue delays are unbounded.
processJob :: MirrorJob -> WorkerM JobOutcome
processJob :: MirrorJob -> WorkerM JobOutcome
processJob MirrorJob
job = Namespace -> WorkerM JobOutcome -> WorkerM JobOutcome
forall (m :: * -> *) a. KatipContext m => Namespace -> m a -> m a
katipAddNamespace Namespace
"job" (WorkerM JobOutcome -> WorkerM JobOutcome)
-> WorkerM JobOutcome -> WorkerM JobOutcome
forall a b. (a -> b) -> a -> b
$ do
    Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
DebugS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"starting mirror job for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job))
    tracing <- (WorkerRuntime -> WorkerTracingPort) -> WorkerM WorkerTracingPort
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> WorkerTracingPort
wrTracing
    runtime <- ask
    withRunInIO $ \forall a. WorkerM a -> IO a
runInIO ->
        WorkerTracingPort
-> forall a.
   PackageName
   -> Version
   -> Maybe RemoteSpanContext
   -> (a -> JobSpanOutcome)
   -> IO a
   -> IO a
wtpMirrorJobSpan WorkerTracingPort
tracing (MirrorJob -> PackageName
jobPackage MirrorJob
job) (MirrorJob -> Version
jobVersion MirrorJob
job) (MirrorJob -> Maybe RemoteSpanContext
jobTraceContext MirrorJob
job) JobOutcome -> JobSpanOutcome
jobSpanOutcome (IO JobOutcome -> IO JobOutcome) -> IO JobOutcome -> IO JobOutcome
forall a b. (a -> b) -> a -> b
$
            WorkerM JobOutcome -> IO JobOutcome
forall a. WorkerM a -> IO a
runInIO (WorkerM JobOutcome -> IO JobOutcome)
-> WorkerM JobOutcome -> IO JobOutcome
forall a b. (a -> b) -> a -> b
$
                WorkerRuntime
-> forall (m :: * -> *) a.
   (KatipContext m, MonadIO m) =>
   m a -> m a
wrInjectTraceContext WorkerRuntime
runtime (MirrorJob -> WorkerM JobOutcome
reevaluateThenMirror MirrorJob
job)

-- Use one ecosystem bundle throughout so a job cannot consult another ecosystem's policy or registry.
reevaluateThenMirror :: MirrorJob -> WorkerM JobOutcome
reevaluateThenMirror :: MirrorJob -> WorkerM JobOutcome
reevaluateThenMirror MirrorJob
job = do
    policies <- (WorkerRuntime -> WorkerPolicies) -> WorkerM WorkerPolicies
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> WorkerPolicies
wrPolicies
    case Map.lookup (pkgEcosystem (jobPackage job)) policies of
        -- Structurally unreachable: only an activated ecosystem enqueues jobs, and activation
        -- implies a bundle. Kept as the fail-closed drop.
        Maybe WorkerPolicy
Nothing -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
Dropped (MirrorJob -> Text
noPolicyReason MirrorJob
job))
        Just WorkerPolicy
policy
            -- An operator can declare the namespace after the enqueue, so the privilege is read
            -- ahead of the mirror probe: the public leg is never entered for a name it owns.
            | WorkerPolicy -> PackageName -> Bool
wpFirstParty WorkerPolicy
policy (MirrorJob -> PackageName
jobPackage MirrorJob
job) -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
Dropped (MirrorJob -> Text
firstPartyReason MirrorJob
job))
            | Bool
otherwise -> WorkerPolicy
-> (WorkerStateBudget -> WorkerM JobOutcome) -> WorkerM JobOutcome
withMirrorBudget WorkerPolicy
policy (\WorkerStateBudget
charge -> WorkerStateBudget
-> WorkerPolicy -> MirrorJob -> WorkerM JobOutcome
mirrorUnlessPresent WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job)

withMirrorBudget :: WorkerPolicy -> (WorkerStateBudget -> WorkerM JobOutcome) -> WorkerM JobOutcome
withMirrorBudget :: WorkerPolicy
-> (WorkerStateBudget -> WorkerM JobOutcome) -> WorkerM JobOutcome
withMirrorBudget WorkerPolicy
policy WorkerStateBudget -> WorkerM JobOutcome
action = case WorkerPolicy -> Maybe WorkerMirrorState
wpMirrorState WorkerPolicy
policy of
    Maybe WorkerMirrorState
Nothing -> WorkerStateBudget -> WorkerM JobOutcome
action (StateCharge
-> StateCharge
-> StateCharge
-> StateCharge
-> StateCharge
-> StateCharge
-> StateCharge
-> WorkerStateBudget
WorkerStateBudget StateCharge
forall {b}. b -> IO ()
ignore StateCharge
forall {b}. b -> IO ()
ignore StateCharge
forall {b}. b -> IO ()
ignore StateCharge
forall {b}. b -> IO ()
ignore StateCharge
forall {b}. b -> IO ()
ignore StateCharge
forall {b}. b -> IO ()
ignore StateCharge
forall {b}. b -> IO ()
ignore)
      where
        ignore :: b -> IO ()
ignore = IO () -> b -> IO ()
forall a b. a -> b -> a
const (() -> IO ()
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ())
    Just WorkerMirrorState
binding -> ((forall a. WorkerM a -> IO a) -> IO JobOutcome)
-> WorkerM JobOutcome
forall b. ((forall a. WorkerM a -> IO a) -> IO b) -> WorkerM b
forall (m :: * -> *) b.
MonadUnliftIO m =>
((forall a. m a -> IO a) -> IO b) -> m b
withRunInIO (((forall a. WorkerM a -> IO a) -> IO JobOutcome)
 -> WorkerM JobOutcome)
-> ((forall a. WorkerM a -> IO a) -> IO JobOutcome)
-> WorkerM JobOutcome
forall a b. (a -> b) -> a -> b
$ \forall a. WorkerM a -> IO a
run ->
        JobOutcome -> Maybe JobOutcome -> JobOutcome
forall a. a -> Maybe a -> a
fromMaybe (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
"worker memory admission refused managed mirror job") (Maybe JobOutcome -> JobOutcome)
-> IO (Maybe JobOutcome) -> IO JobOutcome
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> WorkerMirrorState
-> forall a. (WorkerStateBudget -> IO a) -> IO (Maybe a)
wmsWithinBudget WorkerMirrorState
binding (WorkerM JobOutcome -> IO JobOutcome
forall a. WorkerM a -> IO a
run (WorkerM JobOutcome -> IO JobOutcome)
-> (WorkerStateBudget -> WorkerM JobOutcome)
-> WorkerStateBudget
-> IO JobOutcome
forall b c a. (b -> c) -> (a -> b) -> a -> c
. WorkerStateBudget -> WorkerM JobOutcome
action)

noPolicyReason :: MirrorJob -> Text
noPolicyReason :: MirrorJob -> Text
noPolicyReason MirrorJob
job =
    Text
"no rule policy is configured for the "
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Ecosystem -> Text
ecosystemName (PackageName -> Ecosystem
pkgEcosystem (MirrorJob -> PackageName
jobPackage MirrorJob
job))
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" ecosystem; refusing to mirror "
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job

firstPartyReason :: MirrorJob -> Text
firstPartyReason :: MirrorJob -> Text
firstPartyReason MirrorJob
job =
    Text
"this deployment owns the namespace of "
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"; refusing to mirror public content under a first-party name"

mirrorUnlessPresent :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> WorkerM JobOutcome
mirrorUnlessPresent :: WorkerStateBudget
-> WorkerPolicy -> MirrorJob -> WorkerM JobOutcome
mirrorUnlessPresent WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job = do
    result <- ExceptT JobOutcome WorkerM Severity
-> WorkerM (Either JobOutcome Severity)
forall e (m :: * -> *) a. ExceptT e m a -> m (Either e a)
runExceptT (ExceptT JobOutcome WorkerM Severity
 -> WorkerM (Either JobOutcome Severity))
-> ExceptT JobOutcome WorkerM Severity
-> WorkerM (Either JobOutcome Severity)
forall a b. (a -> b) -> a -> b
$ do
        target <- WorkerM (Either JobOutcome PackageInfo)
-> ExceptT JobOutcome WorkerM PackageInfo
forall e (m :: * -> *) a. m (Either e a) -> ExceptT e m a
ExceptT (WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> WorkerM (Either JobOutcome PackageInfo)
probeInventory WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job)
        if isNothing (wpMirrorState policy) && triggerPresent (\Text
name -> (Artifact -> Bool) -> [Artifact] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
any ((Text -> Text -> Bool
forall a. Eq a => a -> a -> Bool
== Text
name) (Text -> Bool) -> (Artifact -> Text) -> Artifact -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Artifact -> Text
artFilename) (MirrorJob -> PackageInfo -> [Artifact]
targetFiles MirrorJob
job PackageInfo
target)) && mpNativeComplete (wpPublish policy) (jobPackage job) (jobVersion job) target
            then pure DebugS
            else do
                previous <- readRecord charge policy job Nothing
                case previous >>= recordVersion (jobVersion job) of
                    Just MirrorVersion
version
                        | (Text -> Bool) -> Bool
triggerPresent (Text -> Map Text ExpectedMirrorFile -> Bool
forall k a. Ord k => k -> Map k a -> Bool
`Map.member` MirrorVersion -> Map Text ExpectedMirrorFile
versionFiles MirrorVersion
version)
                        , MirrorVersion -> [Artifact] -> MirrorCompletion
checkMirrorCompletion MirrorVersion
version (MirrorJob -> PackageInfo -> [Artifact]
targetFiles MirrorJob
job PackageInfo
target) MirrorCompletion -> MirrorCompletion -> Bool
forall a. Eq a => a -> a -> Bool
== MirrorCompletion
MirrorComplete ->
                            Severity
DebugS Severity
-> ExceptT JobOutcome WorkerM ()
-> ExceptT JobOutcome WorkerM Severity
forall a b.
a -> ExceptT JobOutcome WorkerM b -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ WorkerStateBudget
-> WorkerPolicy -> MirrorVersion -> ExceptT JobOutcome WorkerM ()
verifyRecordedCompanions WorkerStateBudget
charge WorkerPolicy
policy MirrorVersion
version
                    Maybe MirrorVersion
_ -> Severity
InfoS Severity
-> ExceptT JobOutcome WorkerM ()
-> ExceptT JobOutcome WorkerM Severity
forall a b.
a -> ExceptT JobOutcome WorkerM b -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> Int
-> PackageInfo
-> Maybe ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ()
completePass WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job Int
3 PackageInfo
target Maybe ProjectMirrorRecord
previous
    case result of
        Left JobOutcome
failure -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure JobOutcome
failure
        Right Severity
severity -> JobOutcome
Succeeded JobOutcome -> WorkerM () -> WorkerM JobOutcome
forall a b. a -> WorkerM b -> WorkerM a
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
severity (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"mirror version complete: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job))
  where
    triggerPresent :: (Text -> Bool) -> Bool
triggerPresent Text -> Bool
contains = Bool -> (Filename -> Bool) -> Maybe Filename -> Bool
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Bool
True (Text -> Bool
contains (Text -> Bool) -> (Filename -> Text) -> Filename -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Filename -> Text
unFilename) (MirrorJob -> Maybe Filename
jobTriggerFilename MirrorJob
job)

-- A pass rereads native inventory before state, so concurrent additions cannot hide behind an old proof.
completePass :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> Int -> PackageInfo -> Maybe ProjectMirrorRecord -> ExceptT JobOutcome WorkerM ()
completePass :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> Int
-> PackageInfo
-> Maybe ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ()
completePass WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job Int
attempts PackageInfo
target Maybe ProjectMirrorRecord
previous = do
    admitted <- WorkerM (Either JobOutcome Readmitted)
-> ExceptT JobOutcome WorkerM Readmitted
forall e (m :: * -> *) a. m (Either e a) -> ExceptT e m a
ExceptT (WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> WorkerM (Either JobOutcome Readmitted)
reevaluatePolicy WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job)
    record <- stateAction (persistInventory charge (wpMirrorFormat policy) (wpArtifactLimits policy) (wpMirrorState policy) (pmvInventory (raPrepared admitted)) previous)
    copied <- copyRecorded charge policy job target admitted record
    finalTarget <- ExceptT (probeInventory charge policy job)
    latest <- readRecord charge policy job (Just copied)
    current <- maybe (throwError (Retried AfterPublish "mirror record disappeared before completion")) pure latest
    version <- requiredVersion job current
    let copiedNames = Map Text ExpectedMirrorFile
-> (MirrorVersion -> Map Text ExpectedMirrorFile)
-> Maybe MirrorVersion
-> Map Text ExpectedMirrorFile
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Map Text ExpectedMirrorFile
forall a. Monoid a => a
mempty MirrorVersion -> Map Text ExpectedMirrorFile
versionFiles (Version -> ProjectMirrorRecord -> Maybe MirrorVersion
recordVersion (MirrorJob -> Version
jobVersion MirrorJob
job) ProjectMirrorRecord
record)
    case checkMirrorCompletion version (targetFiles job finalTarget) of
        MirrorCompletion
MirrorComplete -> WorkerStateBudget
-> WorkerPolicy -> MirrorVersion -> ExceptT JobOutcome WorkerM ()
verifyRecordedCompanions WorkerStateBudget
charge WorkerPolicy
policy MirrorVersion
version
        MirrorIncomplete NonEmpty MirrorCompletionFault
faults
            | (MirrorCompletionFault -> Bool)
-> NonEmpty MirrorCompletionFault -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
any MirrorCompletionFault -> Bool
conflicting NonEmpty MirrorCompletionFault
faults -> JobOutcome -> ExceptT JobOutcome WorkerM ()
forall a. JobOutcome -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. MonadError e m => e -> m a
throwError (Text -> JobOutcome
Dropped (Text
"mirror target identity conflicts: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> NonEmpty MirrorCompletionFault -> Text
forall b a. (Show a, IsString b) => a -> b
show NonEmpty MirrorCompletionFault
faults))
            | Int
attempts Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Int
1 Bool -> Bool -> Bool
&& (MirrorCompletionFault -> Bool)
-> NonEmpty MirrorCompletionFault -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
any (Map Text ExpectedMirrorFile -> MirrorCompletionFault -> Bool
forall {a}. Map Text a -> MirrorCompletionFault -> Bool
newlyRequired Map Text ExpectedMirrorFile
copiedNames) NonEmpty MirrorCompletionFault
faults -> WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> Int
-> PackageInfo
-> Maybe ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ()
completePass WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job (Int
attempts Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1) PackageInfo
finalTarget (ProjectMirrorRecord -> Maybe ProjectMirrorRecord
forall a. a -> Maybe a
Just ProjectMirrorRecord
current)
            | Bool
otherwise -> JobOutcome -> ExceptT JobOutcome WorkerM ()
forall a. JobOutcome -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. MonadError e m => e -> m a
throwError (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
AfterPublish (Text
"mirror remains incomplete: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> NonEmpty MirrorCompletionFault -> Text
forall b a. (Show a, IsString b) => a -> b
show NonEmpty MirrorCompletionFault
faults))
  where
    conflicting :: MirrorCompletionFault -> Bool
conflicting (ConflictingMirrorFile Text
_) = Bool
True
    conflicting MirrorCompletionFault
_ = Bool
False
    newlyRequired :: Map Text a -> MirrorCompletionFault -> Bool
newlyRequired Map Text a
known (MissingMirrorFile Text
name) = Text -> Map Text a -> Bool
forall k a. Ord k => k -> Map k a -> Bool
Map.notMember Text
name Map Text a
known
    newlyRequired Map Text a
known (PendingMirrorCompanion Text
name) = Text -> Map Text a -> Bool
forall k a. Ord k => k -> Map k a -> Bool
Map.notMember Text
name Map Text a
known
    newlyRequired Map Text a
_ MirrorCompletionFault
_ = Bool
False

readRecord :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> Maybe ProjectMirrorRecord -> ExceptT JobOutcome WorkerM (Maybe ProjectMirrorRecord)
readRecord :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> Maybe ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM (Maybe ProjectMirrorRecord)
readRecord WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job Maybe ProjectMirrorRecord
fallback = case WorkerPolicy -> Maybe WorkerMirrorState
wpMirrorState WorkerPolicy
policy of
    Maybe WorkerMirrorState
Nothing -> Maybe ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM (Maybe ProjectMirrorRecord)
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure Maybe ProjectMirrorRecord
fallback
    Just WorkerMirrorState
binding -> IO (Either MirrorStateFault (Maybe ProjectMirrorRecord))
-> ExceptT JobOutcome WorkerM (Maybe ProjectMirrorRecord)
forall a.
IO (Either MirrorStateFault a) -> ExceptT JobOutcome WorkerM a
stateAction (WorkerStateBudget
-> WorkerMirrorState
-> PackageName
-> IO (Either MirrorStateFault (Maybe ProjectMirrorRecord))
readMirrorRecord WorkerStateBudget
charge WorkerMirrorState
binding (MirrorJob -> PackageName
jobPackage MirrorJob
job))

verifyRecordedCompanions :: WorkerStateBudget -> WorkerPolicy -> MirrorVersion -> ExceptT JobOutcome WorkerM ()
verifyRecordedCompanions :: WorkerStateBudget
-> WorkerPolicy -> MirrorVersion -> ExceptT JobOutcome WorkerM ()
verifyRecordedCompanions WorkerStateBudget
charge WorkerPolicy
policy MirrorVersion
version = WorkerM (Either JobOutcome ()) -> ExceptT JobOutcome WorkerM ()
forall e (m :: * -> *) a. m (Either e a) -> ExceptT e m a
ExceptT (WorkerM (Either JobOutcome ()) -> ExceptT JobOutcome WorkerM ())
-> WorkerM (Either JobOutcome ()) -> ExceptT JobOutcome WorkerM ()
forall a b. (a -> b) -> a -> b
$ ((forall a. WorkerM a -> IO a) -> IO (Either JobOutcome ()))
-> WorkerM (Either JobOutcome ())
forall b. ((forall a. WorkerM a -> IO a) -> IO b) -> WorkerM b
forall (m :: * -> *) b.
MonadUnliftIO m =>
((forall a. m a -> IO a) -> IO b) -> m b
withRunInIO (((forall a. WorkerM a -> IO a) -> IO (Either JobOutcome ()))
 -> WorkerM (Either JobOutcome ()))
-> ((forall a. WorkerM a -> IO a) -> IO (Either JobOutcome ()))
-> WorkerM (Either JobOutcome ())
forall a b. (a -> b) -> a -> b
$ \forall a. WorkerM a -> IO a
run ->
    (MirrorStateFault -> JobOutcome)
-> Either MirrorStateFault () -> Either JobOutcome ()
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first MirrorStateFault -> JobOutcome
stateOutcome (Either MirrorStateFault () -> Either JobOutcome ())
-> IO (Either MirrorStateFault ()) -> IO (Either JobOutcome ())
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> WorkerStateBudget
-> IO ()
-> Maybe WorkerMirrorState
-> MirrorVersion
-> IO (Either MirrorStateFault ())
verifyCompanions WorkerStateBudget
charge (WorkerM () -> IO ()
forall a. WorkerM a -> IO a
run WorkerM ()
recordWorkerProgress) (WorkerPolicy -> Maybe WorkerMirrorState
wpMirrorState WorkerPolicy
policy) MirrorVersion
version

stateAction :: IO (Either MirrorStateFault a) -> ExceptT JobOutcome WorkerM a
stateAction :: forall a.
IO (Either MirrorStateFault a) -> ExceptT JobOutcome WorkerM a
stateAction IO (Either MirrorStateFault a)
action = WorkerM (Either JobOutcome a) -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. m (Either e a) -> ExceptT e m a
ExceptT ((Either MirrorStateFault a -> Either JobOutcome a)
-> WorkerM (Either MirrorStateFault a)
-> WorkerM (Either JobOutcome a)
forall a b. (a -> b) -> WorkerM a -> WorkerM b
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
fmap ((MirrorStateFault -> JobOutcome)
-> Either MirrorStateFault a -> Either JobOutcome a
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first MirrorStateFault -> JobOutcome
stateOutcome) (IO (Either MirrorStateFault a)
-> WorkerM (Either MirrorStateFault a)
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO IO (Either MirrorStateFault a)
action))

stateOutcome :: MirrorStateFault -> JobOutcome
stateOutcome :: MirrorStateFault -> JobOutcome
stateOutcome MirrorStateFault
fault = case MirrorStateFault
fault of
    StateBound LimitError
_ -> Text -> JobOutcome
DeadLettered Text
detail
    StateInvalid Text
_ -> Text -> JobOutcome
DeadLettered Text
detail
    MirrorStateFault
StateIdentityMismatch -> Text -> JobOutcome
DeadLettered Text
detail
    StateFileConflict Text
_ -> Text -> JobOutcome
Dropped Text
detail
    StateCompanionConflict Text
_ -> Text -> JobOutcome
Dropped Text
detail
    StateTransport TransportFault
_ -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
detail
    MirrorStateFault
StateRevisionMismatch -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
detail
    MirrorStateFault
StateMissingCompanion -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
detail
    MirrorStateFault
StateRetryExhausted -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
detail
  where
    detail :: Text
detail = Text
"mirror state unavailable: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorStateFault -> Text
forall b a. (Show a, IsString b) => a -> b
show MirrorStateFault
fault

requiredVersion :: MirrorJob -> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM MirrorVersion
requiredVersion :: MirrorJob
-> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM MirrorVersion
requiredVersion MirrorJob
job = ExceptT JobOutcome WorkerM MirrorVersion
-> (MirrorVersion -> ExceptT JobOutcome WorkerM MirrorVersion)
-> Maybe MirrorVersion
-> ExceptT JobOutcome WorkerM MirrorVersion
forall b a. b -> (a -> b) -> Maybe a -> b
maybe (JobOutcome -> ExceptT JobOutcome WorkerM MirrorVersion
forall a. JobOutcome -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. MonadError e m => e -> m a
throwError (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
"missing recorded mirror version")) MirrorVersion -> ExceptT JobOutcome WorkerM MirrorVersion
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Maybe MirrorVersion -> ExceptT JobOutcome WorkerM MirrorVersion)
-> (ProjectMirrorRecord -> Maybe MirrorVersion)
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM MirrorVersion
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Version -> ProjectMirrorRecord -> Maybe MirrorVersion
recordVersion (MirrorJob -> Version
jobVersion MirrorJob
job)

targetFiles :: MirrorJob -> PackageInfo -> [Artifact]
targetFiles :: MirrorJob -> PackageInfo -> [Artifact]
targetFiles MirrorJob
job PackageInfo
target = [Artifact]
-> (PackageDetails -> [Artifact])
-> Maybe PackageDetails
-> [Artifact]
forall b a. b -> (a -> b) -> Maybe a -> b
maybe [] (NonEmpty Artifact -> [Artifact]
forall a. NonEmpty a -> [a]
forall (t :: * -> *) a. Foldable t => t a -> [a]
toList (NonEmpty Artifact -> [Artifact])
-> (PackageDetails -> NonEmpty Artifact)
-> PackageDetails
-> [Artifact]
forall b c a. (b -> c) -> (a -> b) -> a -> c
. PackageDetails -> NonEmpty Artifact
pkgArtifacts) (Text -> Map Text PackageDetails -> Maybe PackageDetails
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup (Version -> Text
renderVersion (MirrorJob -> Version
jobVersion MirrorJob
job)) (PackageInfo -> Map Text PackageDetails
infoVersions PackageInfo
target))

{- An unreadable answer is not an empty store, so it reports a fault rather than let the write
declare a tag chosen without the inventory. A 404 is a store holding this package not at all. -}
probeInventory :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> WorkerM (Either JobOutcome PackageInfo)
probeInventory :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> WorkerM (Either JobOutcome PackageInfo)
probeInventory WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job = do
    probed <- IO (Either MetadataError PackageInfo)
-> WorkerM (Either MetadataError PackageInfo)
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (MirrorPublish
-> StateCharge
-> PackageName
-> IO (Either MetadataError PackageInfo)
mpProbeInventory (WorkerPolicy -> MirrorPublish
wpPublish WorkerPolicy
policy) (WorkerStateBudget -> StateCharge
wsbTargets WorkerStateBudget
charge) (MirrorJob -> PackageName
jobPackage MirrorJob
job))
    pure $ case probed of
        Right PackageInfo
inventory -> PackageInfo -> Either JobOutcome PackageInfo
forall a b. b -> Either a b
Right PackageInfo
inventory
        Left MetadataError
MetadataAbsent -> PackageInfo -> Either JobOutcome PackageInfo
forall a b. b -> Either a b
Right (PackageName
-> Map Text PackageDetails
-> Map Text Version
-> [InvalidEntry]
-> PackageInfo
PackageInfo (MirrorJob -> PackageName
jobPackage MirrorJob
job) Map Text PackageDetails
forall a. Monoid a => a
mempty Map Text Version
forall a. Monoid a => a
mempty [])
        Left (MetadataFetch FetchFault
fault) -> JobOutcome -> Either JobOutcome PackageInfo
forall a b. a -> Either a b
Left (RetryLeg -> (FetchFault -> Text) -> FetchFault -> JobOutcome
outcomeOfFetchFault RetryLeg
BeforePublish (MirrorJob -> FetchFault -> Text
probeFaultReason MirrorJob
job) FetchFault
fault)
        Left (MetadataBoundExceeded LimitError
limit) -> JobOutcome -> Either JobOutcome PackageInfo
forall a b. a -> Either a b
Left (RetryLeg -> (FetchFault -> Text) -> FetchFault -> JobOutcome
outcomeOfFetchFault RetryLeg
BeforePublish (MirrorJob -> FetchFault -> Text
probeFaultReason MirrorJob
job) (LimitError -> FetchFault
FetchBoundExceeded LimitError
limit))
        Left (MetadataAuthorisationFailure Int
status) -> JobOutcome -> Either JobOutcome PackageInfo
forall a b. a -> Either a b
Left (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish (MirrorJob -> Int -> Text
probeStatusReason MirrorJob
job Int
status))
        Left (MetadataHttpFailure Int
status Maybe RetryAfter
_) -> JobOutcome -> Either JobOutcome PackageInfo
forall a b. a -> Either a b
Left (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish (MirrorJob -> Int -> Text
probeStatusReason MirrorJob
job Int
status))
        Left MetadataError
MetadataUndecodable -> JobOutcome -> Either JobOutcome PackageInfo
forall a b. a -> Either a b
Left (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish (MirrorJob -> Text -> Text
probeParseReason MirrorJob
job Text
"invalid mirror metadata"))
        Left (MetadataNameMismatch Text
_) -> JobOutcome -> Either JobOutcome PackageInfo
forall a b. a -> Either a b
Left (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish (MirrorJob -> Text -> Text
probeParseReason MirrorJob
job Text
"mismatched package identity"))

probeFaultReason :: MirrorJob -> FetchFault -> Text
probeFaultReason :: MirrorJob -> FetchFault -> Text
probeFaultReason MirrorJob
job = \case
    FetchUrlUnformable UrlFormationError
urlErr -> Text
"unformable mirror probe URL: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> UrlFormationError -> Text
renderUrlFormationError UrlFormationError
urlErr
    FetchBoundExceeded LimitError
limitErr -> Text
"the mirror target's metadata exceeded the response bound: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> LimitError -> Text
forall b a. (Show a, IsString b) => a -> b
show LimitError
limitErr
    FetchCooldown RetryAfter
_ -> Text
"the public upstream is cooling down"
    FetchTransport TransportFault
fault -> Text
"mirror inventory probe for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" failed: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> TransportCause -> Text
forall b a. (Show a, IsString b) => a -> b
show (TransportFault -> TransportCause
tfCause TransportFault
fault)

probeStatusReason :: MirrorJob -> Int -> Text
probeStatusReason :: MirrorJob -> Int -> Text
probeStatusReason MirrorJob
job Int
code =
    Text
"the mirror target answered HTTP "
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Int -> Text
forall b a. (Show a, IsString b) => a -> b
show Int
code
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" for the inventory probe of "
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"; refusing to publish a release tag chosen without it"

probeParseReason :: MirrorJob -> Text -> Text
probeParseReason :: MirrorJob -> Text -> Text
probeParseReason MirrorJob
job Text
detail =
    Text
"could not read the mirror target's inventory for "
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" ("
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
detail
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"); refusing to publish a release tag chosen without it"

-- The complete version keeps its checked locations beside its adapter-owned upload preparation.
data Readmitted = Readmitted
    { Readmitted -> PreparedMirrorVersion
raPrepared :: PreparedMirrorVersion
    , Readmitted -> NonEmpty (ExpectedMirrorFile, RegistryUrl)
raLocations :: NonEmpty (ExpectedMirrorFile, RegistryUrl)
    , Readmitted -> Maybe Version
raUpstreamLatest :: Maybe Version
    , Readmitted -> Decision
raDecision :: Decision
    }

reevaluatePolicy :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> WorkerM (Either JobOutcome Readmitted)
reevaluatePolicy :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> WorkerM (Either JobOutcome Readmitted)
reevaluatePolicy WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job =
    IO VersionEvaluation -> WorkerM VersionEvaluation
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (WorkerPolicy
-> StateCharge -> PackageName -> Version -> IO VersionEvaluation
wpResolveVersion WorkerPolicy
policy (WorkerStateBudget -> StateCharge
wsbSource WorkerStateBudget
charge) (MirrorJob -> PackageName
jobPackage MirrorJob
job) (MirrorJob -> Version
jobVersion MirrorJob
job)) WorkerM VersionEvaluation
-> (VersionEvaluation -> WorkerM (Either JobOutcome Readmitted))
-> WorkerM (Either JobOutcome Readmitted)
forall a b. WorkerM a -> (a -> WorkerM b) -> WorkerM b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= WorkerPolicy
-> MirrorJob
-> VersionEvaluation
-> WorkerM (Either JobOutcome Readmitted)
admitEvaluation WorkerPolicy
policy MirrorJob
job

admitEvaluation :: WorkerPolicy -> MirrorJob -> VersionEvaluation -> WorkerM (Either JobOutcome Readmitted)
admitEvaluation :: WorkerPolicy
-> MirrorJob
-> VersionEvaluation
-> WorkerM (Either JobOutcome Readmitted)
admitEvaluation WorkerPolicy
policy MirrorJob
job VersionEvaluation
evaluation = case VersionEvaluation
evaluation of
    VersionCooling RetryAfter
_ -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left (Text -> JobOutcome
unresolved Text
"the public upstream is cooling down"))
    VersionEvaluation
VersionMetadataUnavailable -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left (Text -> JobOutcome
unresolved (Text
"could not re-fetch metadata to re-evaluate current policy for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job)))
    VersionEvaluation
VersionMissing -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left (Text -> JobOutcome
unresolved (Text
"the public upstream no longer offers " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job)))
    VersionPresent VersionDoc
doc Maybe Version
upstreamLatest -> case WorkerPolicy -> MirrorJob -> VersionDoc -> Maybe Filename
triggerFile WorkerPolicy
policy MirrorJob
job VersionDoc
doc of
        Maybe Filename
Nothing -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left (Text -> JobOutcome
Dropped Text
"the source inventory has no supported trigger artifact"))
        Just Filename
trigger -> do
            ctx <- IO EvalContext -> WorkerM EvalContext
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO UTCTime -> IO (Maybe DbEtag) -> IO EvalContext
mkEvalContext (WorkerPolicy -> IO UTCTime
wpNow WorkerPolicy
policy) (Maybe DbEtag -> IO (Maybe DbEtag)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure Maybe DbEtag
forall a. Maybe a
Nothing))
            (admission, decision) <- liftIO (admitArtifactWithDecision ctx (wpRules policy) (wpMinIntegrity policy) trigger (vdDetails doc))
            case outcomeOfAdmission job admission of
                Left JobOutcome
outcome -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left JobOutcome
outcome)
                Right MirrorArtifact
_ -> do
                    SimpleLogPayload -> [SkippedCheck] -> WorkerM ()
forall (m :: * -> *).
KatipContext m =>
SimpleLogPayload -> [SkippedCheck] -> m ()
logSkippedCheckEvidence
                        (Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"package" (PackageName -> Text
renderPackageName (MirrorJob -> PackageName
jobPackage MirrorJob
job)) SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"version" (Version -> Text
renderVersion (MirrorJob -> Version
jobVersion MirrorJob
job)))
                        (Decision -> [SkippedCheck]
skippedChecks Decision
decision)
                    case WorkerPolicy
-> MirrorJob
-> Filename
-> VersionDoc
-> Either JobOutcome VersionDoc
supportedVersion WorkerPolicy
policy MirrorJob
job Filename
trigger VersionDoc
doc of
                        Left JobOutcome
outcome -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left JobOutcome
outcome)
                        Right VersionDoc
supported -> case MirrorPublish
-> PackageName
-> Version
-> VersionDoc
-> Either PublishFault PreparedMirrorVersion
mpPrepareVersion (WorkerPolicy -> MirrorPublish
wpPublish WorkerPolicy
policy) (MirrorJob -> PackageName
jobPackage MirrorJob
job) (MirrorJob -> Version
jobVersion MirrorJob
job) VersionDoc
supported of
                            Left PublishFault
fault -> JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left (JobOutcome -> Either JobOutcome Readmitted)
-> WorkerM JobOutcome -> WorkerM (Either JobOutcome Readmitted)
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> MirrorJob -> Either PublishFault () -> WorkerM JobOutcome
outcomeOfPublish MirrorJob
job (PublishFault -> Either PublishFault ()
forall a b. a -> Either a b
Left PublishFault
fault)
                            Right PreparedMirrorVersion
prepared -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Either JobOutcome Readmitted
 -> WorkerM (Either JobOutcome Readmitted))
-> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a b. (a -> b) -> a -> b
$ do
                                locations <- (ExpectedMirrorFile
 -> Either JobOutcome (ExpectedMirrorFile, RegistryUrl))
-> NonEmpty ExpectedMirrorFile
-> Either JobOutcome (NonEmpty (ExpectedMirrorFile, RegistryUrl))
forall (t :: * -> *) (f :: * -> *) a b.
(Traversable t, Applicative f) =>
(a -> f b) -> t a -> f (t b)
forall (f :: * -> *) a b.
Applicative f =>
(a -> f b) -> NonEmpty a -> f (NonEmpty b)
traverse (WorkerPolicy
-> MirrorJob
-> ExpectedMirrorFile
-> Either JobOutcome (ExpectedMirrorFile, RegistryUrl)
admittedLocation WorkerPolicy
policy MirrorJob
job) (MirrorInventory -> NonEmpty ExpectedMirrorFile
inventoryFiles (PreparedMirrorVersion -> MirrorInventory
pmvInventory PreparedMirrorVersion
prepared))
                                pure (Readmitted prepared locations upstreamLatest decision)
  where
    unresolved :: Text -> JobOutcome
unresolved = Maybe Transience -> Text -> JobOutcome
retryOrDrop (VersionEvaluation -> Maybe Transience
versionTransience VersionEvaluation
evaluation)

supportedVersion :: WorkerPolicy -> MirrorJob -> Filename -> VersionDoc -> Either JobOutcome VersionDoc
supportedVersion :: WorkerPolicy
-> MirrorJob
-> Filename
-> VersionDoc
-> Either JobOutcome VersionDoc
supportedVersion WorkerPolicy
policy MirrorJob
job Filename
trigger VersionDoc
doc = do
    strong <- (VersionIntegrity -> JobOutcome)
-> Either VersionIntegrity (NonEmpty Artifact)
-> Either JobOutcome (NonEmpty Artifact)
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first (JobOutcome -> VersionIntegrity -> JobOutcome
forall a b. a -> b -> a
const (Text -> JobOutcome
Dropped Text
"no source artifact clears the integrity floor")) (MinIntegrity
-> (Artifact -> [Hash])
-> NonEmpty Artifact
-> Either VersionIntegrity (NonEmpty Artifact)
forall floor file.
IntegrityFloor floor =>
floor
-> (file -> [Hash])
-> NonEmpty file
-> Either VersionIntegrity (NonEmpty file)
partitionByFloor (WorkerPolicy -> MinIntegrity
wpMinIntegrity WorkerPolicy
policy) Artifact -> [Hash]
artHashes (PackageDetails -> NonEmpty Artifact
pkgArtifacts PackageDetails
details))
    let kept = (Artifact -> Bool) -> [Artifact] -> [Artifact]
forall a. (a -> Bool) -> [a] -> [a]
filter (Either JobOutcome RegistryUrl -> Bool
forall a b. Either a b -> Bool
isRight (Either JobOutcome RegistryUrl -> Bool)
-> (Artifact -> Either JobOutcome RegistryUrl) -> Artifact -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. WorkerPolicy -> Artifact -> Either JobOutcome RegistryUrl
sourceLocation WorkerPolicy
policy) (NonEmpty Artifact -> [Artifact]
forall a. NonEmpty a -> [a]
forall (t :: * -> *) a. Foldable t => t a -> [a]
toList NonEmpty Artifact
strong)
    survivors <- maybeToRight (locationRefusal strong) (nonEmpty kept)
    unless (any ((== unFilename trigger) . artFilename) survivors) $
        Left (Dropped ("the current location policy refuses the requested artifact of " <> renderJob job))
    pure doc{vdDetails = details{pkgArtifacts = survivors}}
  where
    details :: PackageDetails
details = VersionDoc -> PackageDetails
vdDetails VersionDoc
doc
    locationRefusal :: NonEmpty Artifact -> JobOutcome
locationRefusal (Artifact
firstArtifact :| [Artifact]
_) = JobOutcome -> Either JobOutcome RegistryUrl -> JobOutcome
forall a b. a -> Either a b -> a
fromLeft (Text -> JobOutcome
Dropped Text
"no supported artifact location") (WorkerPolicy -> Artifact -> Either JobOutcome RegistryUrl
sourceLocation WorkerPolicy
policy Artifact
firstArtifact)

sourceLocation :: WorkerPolicy -> Artifact -> Either JobOutcome RegistryUrl
sourceLocation :: WorkerPolicy -> Artifact -> Either JobOutcome RegistryUrl
sourceLocation WorkerPolicy
policy Artifact
artifact = do
    Bool -> Either JobOutcome () -> Either JobOutcome ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
unless (WorkerPolicy -> Maybe HostPort -> Bool
wpArtifactHostHonoured WorkerPolicy
policy (Text -> Maybe HostPort
hostPortAddress (Artifact -> Text
artUrl Artifact
artifact))) (Either JobOutcome () -> Either JobOutcome ())
-> Either JobOutcome () -> Either JobOutcome ()
forall a b. (a -> b) -> a -> b
$
        JobOutcome -> Either JobOutcome ()
forall a b. a -> Either a b
Left (Text -> JobOutcome
Dropped (Text
"the artifact-host policy refuses " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text -> Text
authorityLabel (Artifact -> Text
artUrl Artifact
artifact)))
    (Text -> JobOutcome)
-> Either Text RegistryUrl -> Either JobOutcome RegistryUrl
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first (JobOutcome -> Text -> JobOutcome
forall a b. a -> b -> a
const (Text -> JobOutcome
Dropped Text
"the source artifact URL violates the egress policy")) (WorkerPolicy -> Text -> Either Text RegistryUrl
wpEgressUrl WorkerPolicy
policy (Artifact -> Text
artUrl Artifact
artifact))

triggerFile :: WorkerPolicy -> MirrorJob -> VersionDoc -> Maybe Filename
triggerFile :: WorkerPolicy -> MirrorJob -> VersionDoc -> Maybe Filename
triggerFile WorkerPolicy
policy MirrorJob
job VersionDoc
doc = MirrorJob -> Maybe Filename
jobTriggerFilename MirrorJob
job Maybe Filename -> Maybe Filename -> Maybe Filename
forall a. Maybe a -> Maybe a -> Maybe a
forall (f :: * -> *) a. Alternative f => f a -> f a -> f a
<|> ((Artifact -> Bool) -> NonEmpty Artifact -> Maybe Artifact
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Maybe a
find Artifact -> Bool
supported (PackageDetails -> NonEmpty Artifact
pkgArtifacts (VersionDoc -> PackageDetails
vdDetails VersionDoc
doc)) Maybe Artifact -> (Artifact -> Maybe Filename) -> Maybe Filename
forall a b. Maybe a -> (a -> Maybe b) -> Maybe b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= Text -> Maybe Filename
mkFilename (Text -> Maybe Filename)
-> (Artifact -> Text) -> Artifact -> Maybe Filename
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Artifact -> Text
artFilename)
  where
    supported :: Artifact -> Bool
supported Artifact
artifact = MinIntegrity -> [Hash] -> VersionIntegrity
forall floor.
IntegrityFloor floor =>
floor -> [Hash] -> VersionIntegrity
classifyDigests (WorkerPolicy -> MinIntegrity
wpMinIntegrity WorkerPolicy
policy) (Artifact -> [Hash]
artHashes Artifact
artifact) VersionIntegrity -> VersionIntegrity -> Bool
forall a. Eq a => a -> a -> Bool
== VersionIntegrity
MeetsFloor Bool -> Bool -> Bool
&& Either JobOutcome RegistryUrl -> Bool
forall a b. Either a b -> Bool
isRight (WorkerPolicy -> Artifact -> Either JobOutcome RegistryUrl
sourceLocation WorkerPolicy
policy Artifact
artifact)

admittedLocation :: WorkerPolicy -> MirrorJob -> ExpectedMirrorFile -> Either JobOutcome (ExpectedMirrorFile, RegistryUrl)
admittedLocation :: WorkerPolicy
-> MirrorJob
-> ExpectedMirrorFile
-> Either JobOutcome (ExpectedMirrorFile, RegistryUrl)
admittedLocation WorkerPolicy
policy MirrorJob
job ExpectedMirrorFile
file = do
    case MinIntegrity -> [Hash] -> VersionIntegrity
forall floor.
IntegrityFloor floor =>
floor -> [Hash] -> VersionIntegrity
classifyDigests (WorkerPolicy -> MinIntegrity
wpMinIntegrity WorkerPolicy
policy) (Artifact -> [Hash]
artHashes Artifact
artifact) of
        VersionIntegrity
MeetsFloor -> () -> Either JobOutcome ()
forall a b. b -> Either a b
Right ()
        VersionIntegrity
BelowFloor -> Either JobOutcome MirrorArtifact -> Either JobOutcome ()
forall (f :: * -> *) a. Functor f => f a -> f ()
void (MirrorJob -> ArtifactAdmission -> Either JobOutcome MirrorArtifact
outcomeOfAdmission MirrorJob
job ArtifactAdmission
AdmissionBelowFloor)
        VersionIntegrity
NoIntegrity -> Either JobOutcome MirrorArtifact -> Either JobOutcome ()
forall (f :: * -> *) a. Functor f => f a -> f ()
void (MirrorJob -> ArtifactAdmission -> Either JobOutcome MirrorArtifact
outcomeOfAdmission MirrorJob
job ArtifactAdmission
AdmissionIntegrityMissing)
    location <- WorkerPolicy -> Artifact -> Either JobOutcome RegistryUrl
sourceLocation WorkerPolicy
policy Artifact
artifact
    pure (file, location)
  where
    artifact :: Artifact
artifact = ExpectedMirrorFile -> Artifact
expectedArtifact ExpectedMirrorFile
file

sourceUnavailableReason :: MirrorJob -> Text -> Text
sourceUnavailableReason :: MirrorJob -> Text -> Text
sourceUnavailableReason MirrorJob
job Text
detail =
    Text
"the source version object for "
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" was unavailable ("
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
detail
        Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"); refusing to mirror a reduced manifest"

-- | Project admission to publication or a queue outcome. 'admissionTransience' determines retry versus drop.
outcomeOfAdmission :: MirrorJob -> ArtifactAdmission -> Either JobOutcome MirrorArtifact
outcomeOfAdmission :: MirrorJob -> ArtifactAdmission -> Either JobOutcome MirrorArtifact
outcomeOfAdmission MirrorJob
job ArtifactAdmission
admission = case ArtifactAdmission
admission of
    AdmissionAdmit Filename
filename Artifact
artifact NonEmpty Hash
digests -> MirrorArtifact -> Either JobOutcome MirrorArtifact
forall a b. b -> Either a b
Right (Filename -> Artifact -> NonEmpty Hash -> MirrorArtifact
readmittedDescriptor Filename
filename Artifact
artifact NonEmpty Hash
digests)
    AdmissionDenied (Blocked Text
ruleName Maybe DbEtag
_ Reason
reason) ->
        Text -> Either JobOutcome MirrorArtifact
refused (Text
"current policy denies " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": blocked by " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
ruleName Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" (" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Reason -> Text
renderReason Reason
reason Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
")")
    AdmissionDenied Decision
_ ->
        Text -> Either JobOutcome MirrorArtifact
refused (Text
"current policy denies " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": no rule admits it")
    AdmissionUndecidable (Undecidable Transience
_ Reason
reason) ->
        Text -> Either JobOutcome MirrorArtifact
refused (Text
"current policy could not be evaluated for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Reason -> Text
renderReason Reason
reason)
    AdmissionUndecidable Decision
_ ->
        Text -> Either JobOutcome MirrorArtifact
refused (Text
"current policy could not be evaluated for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job)
    ArtifactAdmission
AdmissionFileAbsent ->
        Text -> Either JobOutcome MirrorArtifact
refused (Text
"the public upstream no longer offers the admitted artifact file of " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"; refusing to mirror a withdrawn artifact")
    ArtifactAdmission
AdmissionBelowFloor ->
        Text -> Either JobOutcome MirrorArtifact
refused (Text
"current admission policy refuses " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": its strongest integrity digest is below the configured public floor")
    ArtifactAdmission
AdmissionIntegrityMissing ->
        Text -> Either JobOutcome MirrorArtifact
refused (Text
"current admission policy refuses " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": it no longer carries any integrity digest")
  where
    refused :: Text -> Either JobOutcome MirrorArtifact
    refused :: Text -> Either JobOutcome MirrorArtifact
refused = JobOutcome -> Either JobOutcome MirrorArtifact
forall a b. a -> Either a b
Left (JobOutcome -> Either JobOutcome MirrorArtifact)
-> (Text -> JobOutcome) -> Text -> Either JobOutcome MirrorArtifact
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Maybe Transience -> Text -> JobOutcome
retryOrDrop (ArtifactAdmission -> Maybe Transience
admissionTransience ArtifactAdmission
admission)

{- The worker's one retry-versus-drop rule, over the shared transience. Only an inability the
evaluator expects to clear redelivers: the rest drop through the terminal path. -}
retryOrDrop :: Maybe Transience -> Text -> JobOutcome
retryOrDrop :: Maybe Transience -> Text -> JobOutcome
retryOrDrop Maybe Transience
transience Text
reason = case Maybe Transience
transience of
    Just (WillResolve Maybe RetryAfter
_) -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
reason
    Just Transience
WontResolve -> Text -> JobOutcome
Dropped Text
reason
    Maybe Transience
Nothing -> Text -> JobOutcome
Dropped Text
reason

{- Derive the publish descriptor from what the gate settled, so nothing the queue payload asserted
reaches the trusted-tier publish document unchecked. The size is current metadata's. -}
readmittedDescriptor :: Filename -> Artifact -> NonEmpty Hash -> MirrorArtifact
readmittedDescriptor :: Filename -> Artifact -> NonEmpty Hash -> MirrorArtifact
readmittedDescriptor Filename
filename Artifact
artifact NonEmpty Hash
digests =
    MirrorArtifact
        { maFilename :: Filename
maFilename = Filename
filename
        , maHashes :: NonEmpty Hash
maHashes = NonEmpty Hash
digests
        , maSize :: Maybe Int
maSize = Artifact -> Maybe Int
artSize Artifact
artifact
        }

-- | Classify fetch and publication faults through one retry-versus-drop table.
outcomeOfFetchFault :: RetryLeg -> (FetchFault -> Text) -> FetchFault -> JobOutcome
outcomeOfFetchFault :: RetryLeg -> (FetchFault -> Text) -> FetchFault -> JobOutcome
outcomeOfFetchFault RetryLeg
leg FetchFault -> Text
render FetchFault
fault = Text -> JobOutcome
verdict (FetchFault -> Text
render FetchFault
fault)
  where
    verdict :: Text -> JobOutcome
verdict = case FetchFault
fault of
        FetchUrlUnformable UrlFormationError
_ -> Text -> JobOutcome
Dropped
        FetchBoundExceeded LimitError
_ -> Text -> JobOutcome
DeadLettered
        FetchTransport TransportFault
_ -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
leg
        FetchCooldown RetryAfter
_ -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
leg

-- | Choose the mirror latest tag from the public tag and post-write inventory, which includes the published version.
mirrorLatest :: Maybe Version -> [Version] -> Version -> Version
mirrorLatest :: Maybe Version -> [Version] -> Version -> Version
mirrorLatest Maybe Version
upstreamLatest [Version]
inventory Version
published =
    Version -> Maybe Version -> Version
forall a. a -> Maybe a -> a
fromMaybe Version
published (Maybe Version -> [Version] -> Maybe Version
selectLatest Maybe Version
upstreamLatest (Version
published Version -> [Version] -> [Version]
forall a. a -> [a] -> [a]
: [Version]
inventory))

-- Keep fixed inputs lazy so inspecting the context does not force the plan or target inventory.
data CopyPass = CopyPass
    { CopyPass -> WorkerStateBudget
cpCharge :: ~WorkerStateBudget
    , CopyPass -> WorkerPolicy
cpPolicy :: ~WorkerPolicy
    , CopyPass -> MirrorJob
cpJob :: ~MirrorJob
    , CopyPass -> PublishPlan
cpPlan :: ~PublishPlan
    , CopyPass -> [Artifact]
cpTargetFiles :: ~[Artifact]
    , CopyPass -> Readmitted
cpLatest :: ~Readmitted
    }

-- Each fragment requires current admission before the pass traverses all its admitted locations.
copyRecorded :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> PackageInfo -> Readmitted -> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
copyRecorded :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> PackageInfo
-> Readmitted
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
copyRecorded WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job PackageInfo
target Readmitted
latest ProjectMirrorRecord
record = do
    version <- MirrorJob
-> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM MirrorVersion
requiredVersion MirrorJob
job ProjectMirrorRecord
record
    foldM (copyFragment context) record (toList (versionFragments version))
  where
    context :: CopyPass
context =
        CopyPass
            { cpCharge :: WorkerStateBudget
cpCharge = WorkerStateBudget
charge
            , cpPolicy :: WorkerPolicy
cpPolicy = WorkerPolicy
policy
            , cpJob :: MirrorJob
cpJob = MirrorJob
job
            , cpPlan :: PublishPlan
cpPlan = Version -> PublishPlan
PublishPlan (Maybe Version -> [Version] -> Version -> Version
mirrorLatest (Readmitted -> Maybe Version
raUpstreamLatest Readmitted
latest) ((PackageDetails -> Version) -> [PackageDetails] -> [Version]
forall a b. (a -> b) -> [a] -> [b]
map PackageDetails -> Version
pkgVersion (Map Text PackageDetails -> [PackageDetails]
forall k a. Map k a -> [a]
Map.elems (PackageInfo -> Map Text PackageDetails
infoVersions PackageInfo
target))) (MirrorJob -> Version
jobVersion MirrorJob
job))
            , cpTargetFiles :: [Artifact]
cpTargetFiles = MirrorJob -> PackageInfo -> [Artifact]
targetFiles MirrorJob
job PackageInfo
target
            , cpLatest :: Readmitted
cpLatest = Readmitted
latest
            }

copyFragment :: CopyPass -> ProjectMirrorRecord -> StoredMirrorFragment -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
copyFragment :: CopyPass
-> ProjectMirrorRecord
-> StoredMirrorFragment
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
copyFragment CopyPass
context ProjectMirrorRecord
current StoredMirrorFragment
fragment = do
    version <- MirrorJob
-> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM MirrorVersion
requiredVersion MirrorJob
job ProjectMirrorRecord
current
    let files = (Artifact -> Bool) -> [Artifact] -> [Artifact]
forall a. (a -> Bool) -> [a] -> [a]
filter ([Artifact] -> MirrorVersion -> Artifact -> Bool
needsCopy (CopyPass -> [Artifact]
cpTargetFiles CopyPass
context) MirrorVersion
version) (NonEmpty Artifact -> [Artifact]
forall a. NonEmpty a -> [a]
forall (t :: * -> *) a. Foldable t => t a -> [a]
toList (PackageDetails -> NonEmpty Artifact
pkgArtifacts (StoredMirrorFragment -> PackageDetails
fragmentDetails StoredMirrorFragment
fragment)))
    if null files
        then pure current
        else do
            admitted <- ExceptT (admitEvaluation (cpPolicy context) job{jobTriggerFilename = Nothing} (VersionPresent (fragmentDocument fragment) (raUpstreamLatest (cpLatest context))))
            foldM (copyFile context admitted) current (toList (raLocations admitted))
  where
    job :: MirrorJob
job = CopyPass -> MirrorJob
cpJob CopyPass
context

needsCopy :: [Artifact] -> MirrorVersion -> Artifact -> Bool
needsCopy :: [Artifact] -> MirrorVersion -> Artifact -> Bool
needsCopy [Artifact]
currentFiles MirrorVersion
version Artifact
artifact =
    let name :: Text
name = Artifact -> Text
artFilename Artifact
artifact
        matches :: [Artifact]
matches = (Artifact -> Bool) -> [Artifact] -> [Artifact]
forall a. (a -> Bool) -> [a] -> [a]
filter ((Text -> Text -> Bool
forall a. Eq a => a -> a -> Bool
== Text
name) (Text -> Bool) -> (Artifact -> Text) -> Artifact -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Artifact -> Text
artFilename) [Artifact]
currentFiles
        verified :: Maybe Hash
verified = Text -> Map Text Hash -> Maybe Hash
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Text
name (MirrorVersion -> Map Text Hash
versionVerifiedIdentities MirrorVersion
version)
     in Text -> Map Text CompanionState -> Maybe CompanionState
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Text
name (MirrorVersion -> Map Text CompanionState
versionCompanions MirrorVersion
version) Maybe CompanionState -> Maybe CompanionState -> Bool
forall a. Eq a => a -> a -> Bool
== CompanionState -> Maybe CompanionState
forall a. a -> Maybe a
Just CompanionState
CompanionPending
            Bool -> Bool -> Bool
|| [Artifact] -> Bool
forall a. [a] -> Bool
forall (t :: * -> *) a. Foldable t => t a -> Bool
null [Artifact]
matches
            Bool -> Bool -> Bool
|| Bool
-> (ExpectedMirrorFile -> Bool) -> Maybe ExpectedMirrorFile -> Bool
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Bool
True (\ExpectedMirrorFile
file -> Bool -> Bool
not ((Artifact -> Bool) -> [Artifact] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
all (ExpectedMirrorFile -> Maybe Hash -> Artifact -> Bool
expectedTargetMatchesWith ExpectedMirrorFile
file Maybe Hash
verified) [Artifact]
matches)) (Text -> Map Text ExpectedMirrorFile -> Maybe ExpectedMirrorFile
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Text
name (MirrorVersion -> Map Text ExpectedMirrorFile
versionFiles MirrorVersion
version))

copyFile :: CopyPass -> Readmitted -> ProjectMirrorRecord -> (ExpectedMirrorFile, RegistryUrl) -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
copyFile :: CopyPass
-> Readmitted
-> ProjectMirrorRecord
-> (ExpectedMirrorFile, RegistryUrl)
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
copyFile CopyPass
context Readmitted
admitted ProjectMirrorRecord
current (ExpectedMirrorFile
file, RegistryUrl
location) = do
    version <- MirrorJob
-> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM MirrorVersion
requiredVersion MirrorJob
job ProjectMirrorRecord
current
    let name = Artifact -> Text
artFilename (ExpectedMirrorFile -> Artifact
expectedArtifact ExpectedMirrorFile
file)
        matches = (Artifact -> Bool) -> [Artifact] -> [Artifact]
forall a. (a -> Bool) -> [a] -> [a]
filter ((Text -> Text -> Bool
forall a. Eq a => a -> a -> Bool
== Text
name) (Text -> Bool) -> (Artifact -> Text) -> Artifact -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Artifact -> Text
artFilename) (CopyPass -> [Artifact]
cpTargetFiles CopyPass
context)
        verified = Text -> Map Text Hash -> Maybe Hash
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Text
name (MirrorVersion -> Map Text Hash
versionVerifiedIdentities MirrorVersion
version)
        locations = RegistryUrl
location RegistryUrl -> [RegistryUrl] -> NonEmpty RegistryUrl
forall a. a -> [a] -> NonEmpty a
:| [RegistryUrl
candidate | (ExpectedMirrorFile
incoming, RegistryUrl
candidate) <- NonEmpty (ExpectedMirrorFile, RegistryUrl)
-> [(ExpectedMirrorFile, RegistryUrl)]
forall a. NonEmpty a -> [a]
forall (t :: * -> *) a. Foldable t => t a -> [a]
toList (Readmitted -> NonEmpty (ExpectedMirrorFile, RegistryUrl)
raLocations (CopyPass -> Readmitted
cpLatest CopyPass
context)), ExpectedMirrorFile -> ExpectedMirrorFile -> Bool
expectedIdentityMatches ExpectedMirrorFile
file ExpectedMirrorFile
incoming, RegistryUrl
candidate RegistryUrl -> RegistryUrl -> Bool
forall a. Eq a => a -> a -> Bool
/= RegistryUrl
location]
    when (any (targetIdentityConflict file verified) matches) $
        throwError (Dropped ("mirror target identity conflicts: " <> name))
    let targetProven = (Artifact -> Bool) -> [Artifact] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
all (ExpectedMirrorFile -> Maybe Hash -> Artifact -> Bool
expectedTargetMatchesWith ExpectedMirrorFile
file Maybe Hash
verified) [Artifact]
matches
        needsNativeMetadata = Maybe MirrorCompanion -> Bool
forall a. Maybe a -> Bool
isJust (MirrorPublish -> Maybe MirrorCompanion
mpMirrorCompanion (WorkerPolicy -> MirrorPublish
wpPublish WorkerPolicy
policy)) Bool -> Bool -> Bool
&& Text -> Map Text CompanionState -> Maybe CompanionState
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Text
name (MirrorVersion -> Map Text CompanionState
versionCompanions MirrorVersion
version) Maybe CompanionState -> Maybe CompanionState -> Bool
forall a. Eq a => a -> a -> Bool
== CompanionState -> Maybe CompanionState
forall a. a -> Maybe a
Just CompanionState
CompanionPending
    uploaded <- case matches of
        [] -> WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> PublishPlan
-> Readmitted
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
mirrorArtifact WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job (CopyPass -> PublishPlan
cpPlan CopyPass
context) Readmitted
admitted ExpectedMirrorFile
file NonEmpty RegistryUrl
locations ProjectMirrorRecord
current
        [Artifact]
_ | Bool
targetProven Bool -> Bool -> Bool
&& Bool -> Bool
not Bool
needsNativeMetadata -> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ProjectMirrorRecord
current
        [Artifact]
_ -> CopyPass
-> Readmitted
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> [Artifact]
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
verifyExistingTarget CopyPass
context Readmitted
admitted ExpectedMirrorFile
file NonEmpty RegistryUrl
locations [Artifact]
matches ProjectMirrorRecord
current
    retained <- retainPending charge policy job (pmvInventory (raPrepared admitted)) file locations uploaded
    lift recordWorkerProgress
    pure retained
  where
    charge :: WorkerStateBudget
charge = CopyPass -> WorkerStateBudget
cpCharge CopyPass
context
    policy :: WorkerPolicy
policy = CopyPass -> WorkerPolicy
cpPolicy CopyPass
context
    job :: MirrorJob
job = CopyPass -> MirrorJob
cpJob CopyPass
context

verifyExistingTarget :: CopyPass -> Readmitted -> ExpectedMirrorFile -> NonEmpty RegistryUrl -> [Artifact] -> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
verifyExistingTarget :: CopyPass
-> Readmitted
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> [Artifact]
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
verifyExistingTarget CopyPass
context Readmitted
admitted ExpectedMirrorFile
file NonEmpty RegistryUrl
locations [Artifact]
matches ProjectMirrorRecord
current = do
    (bytes, retained) <- WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> ProjectMirrorRecord
-> ExceptT
     JobOutcome WorkerM (VerifiedArtifactBytes, ProjectMirrorRecord)
fetchVerified (CopyPass -> WorkerStateBudget
cpCharge CopyPass
context) (CopyPass -> WorkerPolicy
cpPolicy CopyPass
context) (CopyPass -> MirrorJob
cpJob CopyPass
context) ExpectedMirrorFile
file NonEmpty RegistryUrl
locations ProjectMirrorRecord
current
    for_ (mpMirrorCompanion (wpPublish (cpPolicy context))) $ \MirrorCompanion
_ ->
        ExceptT JobOutcome WorkerM PreparedMirrorUpload
-> ExceptT JobOutcome WorkerM ()
forall (f :: * -> *) a. Functor f => f a -> f ()
void (MirrorJob
-> WorkerM (Either PublishFault PreparedMirrorUpload)
-> ExceptT JobOutcome WorkerM PreparedMirrorUpload
forall a.
MirrorJob
-> WorkerM (Either PublishFault a) -> ExceptT JobOutcome WorkerM a
publishAction (CopyPass -> MirrorJob
cpJob CopyPass
context) (IO (Either PublishFault PreparedMirrorUpload)
-> WorkerM (Either PublishFault PreparedMirrorUpload)
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (PreparedMirrorVersion
-> StateCharge
-> ExpectedMirrorFile
-> VerifiedArtifactBytes
-> IO (Either PublishFault PreparedMirrorUpload)
pmvPrepareUpload (Readmitted -> PreparedMirrorVersion
raPrepared Readmitted
admitted) (WorkerStateBudget -> StateCharge
wsbArchive (CopyPass -> WorkerStateBudget
cpCharge CopyPass
context)) ExpectedMirrorFile
file VerifiedArtifactBytes
bytes)))
    let proof = Hash -> Maybe Hash
forall a. a -> Maybe a
Just (VerifiedArtifactBytes -> Hash
verifiedTargetHash VerifiedArtifactBytes
bytes)
    when (any (targetIdentityConflict file proof) matches) $
        throwError (Dropped ("mirror target identity conflicts: " <> name))
    unless (all (expectedTargetMatchesWith file proof) matches) $
        throwError (Retried BeforePublish ("mirror file lacks compatible current identity: " <> name))
    pure retained
  where
    name :: Text
name = Artifact -> Text
artFilename (ExpectedMirrorFile -> Artifact
expectedArtifact ExpectedMirrorFile
file)

retainPending :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> MirrorInventory -> ExpectedMirrorFile -> NonEmpty RegistryUrl -> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
retainPending :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> MirrorInventory
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
retainPending WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job MirrorInventory
inventory ExpectedMirrorFile
file NonEmpty RegistryUrl
locations ProjectMirrorRecord
record = do
    version <- MirrorJob
-> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM MirrorVersion
requiredVersion MirrorJob
job ProjectMirrorRecord
record
    case Map.lookup (artFilename (expectedArtifact file)) (versionCompanions version) of
        Just CompanionState
CompanionPending -> case WorkerPolicy -> Maybe WorkerMirrorState
wpMirrorState WorkerPolicy
policy of
            Maybe WorkerMirrorState
Nothing -> JobOutcome -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
forall a. JobOutcome -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. MonadError e m => e -> m a
throwError (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
"declared companion requires durable mirror state")
            Just WorkerMirrorState
binding -> do
                let native :: Maybe MirrorCompanion
native = MirrorPublish -> Maybe MirrorCompanion
mpMirrorCompanion (WorkerPolicy -> MirrorPublish
wpPublish WorkerPolicy
policy)
                candidates <- NonEmpty RegistryUrl
-> (RegistryUrl -> ExceptT JobOutcome WorkerM RegistryUrl)
-> ExceptT JobOutcome WorkerM (NonEmpty RegistryUrl)
forall (t :: * -> *) (f :: * -> *) a b.
(Traversable t, Applicative f) =>
t a -> (a -> f b) -> f (t b)
for NonEmpty RegistryUrl
locations ((RegistryUrl -> ExceptT JobOutcome WorkerM RegistryUrl)
 -> ExceptT JobOutcome WorkerM (NonEmpty RegistryUrl))
-> (RegistryUrl -> ExceptT JobOutcome WorkerM RegistryUrl)
-> ExceptT JobOutcome WorkerM (NonEmpty RegistryUrl)
forall a b. (a -> b) -> a -> b
$ \RegistryUrl
location -> do
                    url <- case Maybe MirrorCompanion
native of
                        Maybe MirrorCompanion
Nothing -> Text -> ExceptT JobOutcome WorkerM Text
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> Text
sidecarUrl (RegistryUrl -> Text
registryUrlText RegistryUrl
location))
                        Just MirrorCompanion
companion -> MirrorJob
-> WorkerM (Either PublishFault Text)
-> ExceptT JobOutcome WorkerM Text
forall a.
MirrorJob
-> WorkerM (Either PublishFault a) -> ExceptT JobOutcome WorkerM a
publishAction MirrorJob
job (Either PublishFault Text -> WorkerM (Either PublishFault Text)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (MirrorCompanion
-> ExpectedMirrorFile -> Text -> Either PublishFault Text
mcLocation MirrorCompanion
companion ExpectedMirrorFile
file (RegistryUrl -> Text
registryUrlText RegistryUrl
location)))
                    either throwError pure (sourceLocation policy (expectedArtifact file){artUrl = url})
                let limits = (WorkerMirrorState -> Limits
wmsLimits WorkerMirrorState
binding){maxMirrorArtifactBytes = maxMetadataBytes (wmsLimits binding)}
                bytes <- fetchLocations (wsbCompanions charge) limits policy candidates
                deadlines <- asks (ecDeadlines . wrClient)
                checked <- case native of
                    Maybe MirrorCompanion
Nothing -> ByteString -> ExceptT JobOutcome WorkerM ByteString
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ByteString
bytes
                    Just MirrorCompanion
companion -> MirrorJob
-> WorkerM (Either PublishFault ByteString)
-> ExceptT JobOutcome WorkerM ByteString
forall a.
MirrorJob
-> WorkerM (Either PublishFault a) -> ExceptT JobOutcome WorkerM a
publishAction MirrorJob
job (IO (Either PublishFault ByteString)
-> WorkerM (Either PublishFault ByteString)
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (MirrorCompanion
-> Deadlines
-> Limits
-> StateCharge
-> MirrorInventory
-> ExpectedMirrorFile
-> ByteString
-> IO (Either PublishFault ByteString)
mcValidate MirrorCompanion
companion Deadlines
deadlines Limits
limits (WorkerStateBudget -> StateCharge
wsbCompanions WorkerStateBudget
charge) MirrorInventory
inventory ExpectedMirrorFile
file ByteString
bytes))
                verified <- either (throwError . stateOutcome) pure (verifySourceCompanionBytes (wmsLimits binding) (msfCompanions (wpMirrorFormat policy)) file checked)
                stateAction (retainCompanion charge binding (jobVersion job) verified record)
        Maybe CompanionState
_ -> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ProjectMirrorRecord
record

mirrorArtifact :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> PublishPlan -> Readmitted -> ExpectedMirrorFile -> NonEmpty RegistryUrl -> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
mirrorArtifact :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> PublishPlan
-> Readmitted
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
mirrorArtifact WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job PublishPlan
plan Readmitted
admitted ExpectedMirrorFile
file NonEmpty RegistryUrl
locations ProjectMirrorRecord
record = do
    (verified, retained) <- WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> ProjectMirrorRecord
-> ExceptT
     JobOutcome WorkerM (VerifiedArtifactBytes, ProjectMirrorRecord)
fetchVerified WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job ExpectedMirrorFile
file NonEmpty RegistryUrl
locations ProjectMirrorRecord
record
    ready <- publishAction job (liftIO (pmvPrepareUpload (raPrepared admitted) (wsbArchive charge) file verified))
    outcome <- lift (katipAddContext (mirrorAdmissionPayload job (raDecision admitted) file (verifiedTargetHash verified)) (publishVerified job plan ready))
    case outcome of
        JobOutcome
Succeeded -> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ProjectMirrorRecord
retained
        JobOutcome
failure -> JobOutcome -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
forall a. JobOutcome -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. MonadError e m => e -> m a
throwError JobOutcome
failure

fetchVerified :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> ExpectedMirrorFile -> NonEmpty RegistryUrl -> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM (VerifiedArtifactBytes, ProjectMirrorRecord)
fetchVerified :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> ProjectMirrorRecord
-> ExceptT
     JobOutcome WorkerM (VerifiedArtifactBytes, ProjectMirrorRecord)
fetchVerified WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job ExpectedMirrorFile
file NonEmpty RegistryUrl
locations ProjectMirrorRecord
record = do
    bytes <- StateCharge
-> Limits
-> WorkerPolicy
-> NonEmpty RegistryUrl
-> ExceptT JobOutcome WorkerM ByteString
fetchLocations (WorkerStateBudget -> StateCharge
wsbArtifacts WorkerStateBudget
charge) (WorkerPolicy -> Limits
wpArtifactLimits WorkerPolicy
policy) WorkerPolicy
policy NonEmpty RegistryUrl
locations
    verified <- publishAction job (pure (verifyMirrorBytes file bytes))
    retained <- stateAction (persistVerified charge (wpMirrorState policy) (jobVersion job) verified record)
    pure (verified, retained)

fetchLocations :: StateCharge -> Limits -> WorkerPolicy -> NonEmpty RegistryUrl -> ExceptT JobOutcome WorkerM ByteString
fetchLocations :: StateCharge
-> Limits
-> WorkerPolicy
-> NonEmpty RegistryUrl
-> ExceptT JobOutcome WorkerM ByteString
fetchLocations StateCharge
charge Limits
limits WorkerPolicy
policy (RegistryUrl
location :| [RegistryUrl]
alternatives) = do
    result <- WorkerM (Either FetchFault ByteString)
-> ExceptT JobOutcome WorkerM (Either FetchFault ByteString)
forall (m :: * -> *) a. Monad m => m a -> ExceptT JobOutcome m a
forall (t :: (* -> *) -> * -> *) (m :: * -> *) a.
(MonadTrans t, Monad m) =>
m a -> t m a
lift (StateCharge
-> Limits
-> (Maybe ClientCredential
    -> Text -> Either UrlFormationError Request)
-> RegistryUrl
-> WorkerM (Either FetchFault ByteString)
fetchArtifactBytesCharged StateCharge
charge Limits
limits (AdapterArtifact
-> Maybe ClientCredential
-> Text
-> Either UrlFormationError Request
artifactByUrl (WorkerPolicy -> AdapterArtifact
wpArtifact WorkerPolicy
policy)) RegistryUrl
location)
    case (result, nonEmpty alternatives) of
        (Right ByteString
bytes, Maybe (NonEmpty RegistryUrl)
_) -> ByteString -> ExceptT JobOutcome WorkerM ByteString
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ByteString
bytes
        (Left (FetchTransport TransportFault
_), Just NonEmpty RegistryUrl
remaining) -> StateCharge
-> Limits
-> WorkerPolicy
-> NonEmpty RegistryUrl
-> ExceptT JobOutcome WorkerM ByteString
fetchLocations StateCharge
charge Limits
limits WorkerPolicy
policy NonEmpty RegistryUrl
remaining
        (Left FetchFault
fault, Maybe (NonEmpty RegistryUrl)
_) -> JobOutcome -> ExceptT JobOutcome WorkerM ByteString
forall a. JobOutcome -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. MonadError e m => e -> m a
throwError (RetryLeg -> (FetchFault -> Text) -> FetchFault -> JobOutcome
outcomeOfFetchFault RetryLeg
BeforePublish (RegistryUrl -> FetchFault -> Text
artifactFetchReason RegistryUrl
location) FetchFault
fault)

publishAction :: MirrorJob -> WorkerM (Either PublishFault a) -> ExceptT JobOutcome WorkerM a
publishAction :: forall a.
MirrorJob
-> WorkerM (Either PublishFault a) -> ExceptT JobOutcome WorkerM a
publishAction MirrorJob
job WorkerM (Either PublishFault a)
action =
    WorkerM (Either JobOutcome a) -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. m (Either e a) -> ExceptT e m a
ExceptT (WorkerM (Either JobOutcome a) -> ExceptT JobOutcome WorkerM a)
-> WorkerM (Either JobOutcome a) -> ExceptT JobOutcome WorkerM a
forall a b. (a -> b) -> a -> b
$
        WorkerM (Either PublishFault a)
action WorkerM (Either PublishFault a)
-> (Either PublishFault a -> WorkerM (Either JobOutcome a))
-> WorkerM (Either JobOutcome a)
forall a b. WorkerM a -> (a -> WorkerM b) -> WorkerM b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= \case
            Right a
value -> Either JobOutcome a -> WorkerM (Either JobOutcome a)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (a -> Either JobOutcome a
forall a b. b -> Either a b
Right a
value)
            Left PublishFault
fault -> JobOutcome -> Either JobOutcome a
forall a b. a -> Either a b
Left (JobOutcome -> Either JobOutcome a)
-> WorkerM JobOutcome -> WorkerM (Either JobOutcome a)
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> MirrorJob -> Either PublishFault () -> WorkerM JobOutcome
outcomeOfPublish MirrorJob
job (PublishFault -> Either PublishFault ()
forall a b. a -> Either a b
Left PublishFault
fault)

artifactFetchReason :: RegistryUrl -> FetchFault -> Text
artifactFetchReason :: RegistryUrl -> FetchFault -> Text
artifactFetchReason RegistryUrl
location = \case
    FetchUrlUnformable UrlFormationError
urlErr -> Text
"unformable artifact URL: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> UrlFormationError -> Text
renderUrlFormationError UrlFormationError
urlErr
    FetchBoundExceeded LimitError
limitErr -> Text
"artifact exceeded the response bound: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> LimitError -> Text
forall b a. (Show a, IsString b) => a -> b
show LimitError
limitErr
    FetchCooldown RetryAfter
_ -> Text
"the public upstream is cooling down"
    FetchTransport TransportFault
fault -> Text
"artifact fetch from " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text -> Text
authorityLabel (RegistryUrl -> Text
registryUrlText RegistryUrl
location) Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" failed: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> TransportCause -> Text
forall b a. (Show a, IsString b) => a -> b
show (TransportFault -> TransportCause
tfCause TransportFault
fault)

mirrorAdmissionPayload :: MirrorJob -> Decision -> ExpectedMirrorFile -> Hash -> SimpleLogPayload
mirrorAdmissionPayload :: MirrorJob
-> Decision -> ExpectedMirrorFile -> Hash -> SimpleLogPayload
mirrorAdmissionPayload MirrorJob
job Decision
decision ExpectedMirrorFile
file Hash
digest =
    Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"event" (Text
"mirror_publication" :: Text)
        SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"package" (PackageName -> Text
renderPackageName (MirrorJob -> PackageName
jobPackage MirrorJob
job))
        SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"version" (Version -> Text
renderVersion (MirrorJob -> Version
jobVersion MirrorJob
job))
        SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"ecosystem" (Ecosystem -> Text
ecosystemName (PackageName -> Ecosystem
pkgEcosystem (MirrorJob -> PackageName
jobPackage MirrorJob
job)))
        SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"target" (Text
"mirrorTarget" :: Text)
        SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"artifact" (Artifact -> Text
artFilename Artifact
artifact)
        SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Value -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"digest" ([Pair] -> Value
object [Key
"algorithm" Key -> Text -> Pair
forall v. ToJSON v => Key -> v -> Pair
forall e kv v. (KeyValue e kv, ToJSON v) => Key -> v -> kv
.= HashAlg -> Text
renderHashAlg (Hash -> HashAlg
hashAlg Hash
digest), Key
"value" Key -> Text -> Pair
forall v. ToJSON v => Key -> v -> Pair
forall e kv v. (KeyValue e kv, ToJSON v) => Key -> v -> kv
.= Hash -> Text
hashValue Hash
digest])
        SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"admission" (RuleEvidence -> Decision -> Text
renderDecision (PackageName -> Version -> RuleEvidence
identityEvidence (MirrorJob -> PackageName
jobPackage MirrorJob
job) (MirrorJob -> Version
jobVersion MirrorJob
job)) Decision
decision)
        SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> SimpleLogPayload
ruleField
  where
    artifact :: Artifact
artifact = ExpectedMirrorFile -> Artifact
expectedArtifact ExpectedMirrorFile
file
    ruleField :: SimpleLogPayload
ruleField = case Decision
decision of
        Admitted Text
name Reason
_ [SkippedCheck]
_ -> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"rule" Text
name
        Blocked Text
name Maybe DbEtag
_ Reason
_ -> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"rule" Text
name
        BlockedByDefault [Reason]
_ -> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"rule" (Text
"BlockedByDefault" :: Text)
        Undecidable Transience
_ Reason
_ -> SimpleLogPayload
forall a. Monoid a => a
mempty

publishVerified :: MirrorJob -> PublishPlan -> PreparedMirrorUpload -> WorkerM JobOutcome
publishVerified :: MirrorJob
-> PublishPlan -> PreparedMirrorUpload -> WorkerM JobOutcome
publishVerified MirrorJob
job PublishPlan
plan PreparedMirrorUpload
prepared = do
    metrics <- (WorkerRuntime -> WorkerMetricsPort) -> WorkerM WorkerMetricsPort
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> WorkerMetricsPort
wrMetrics
    (result, seconds) <- timedSeconds (liftIO (publishPrepared prepared plan))
    liftIO (wmpMirrorPublishDuration metrics seconds)
    outcomeOfPublish job result

outcomeOfPublish :: MirrorJob -> Either PublishFault () -> WorkerM JobOutcome
outcomeOfPublish :: MirrorJob -> Either PublishFault () -> WorkerM JobOutcome
outcomeOfPublish MirrorJob
job = \case
    Right () -> JobOutcome
Succeeded JobOutcome -> WorkerM () -> WorkerM JobOutcome
forall a b. a -> WorkerM b -> WorkerM a
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
InfoS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"mirrored artifact published: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job))
    Left (PublishIntegrityMismatch Text
detail) -> do
        Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"artifact integrity mismatch, refusing to publish: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
detail))
        JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
Dropped (Text
"integrity mismatch: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
detail))
    Left (PublishRejected PublishError
err) -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
AfterPublish (Text
"registry rejected publish: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> PublishError -> Text
forall b a. (Show a, IsString b) => a -> b
show PublishError
err))
    Left (PublishSourceUnavailable Text
detail) -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
SourceUnavailable (MirrorJob -> Text -> Text
sourceUnavailableReason MirrorJob
job Text
detail))
    Left (PublishFetch FetchFault
fault) -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (RetryLeg -> (FetchFault -> Text) -> FetchFault -> JobOutcome
outcomeOfFetchFault RetryLeg
AfterPublish FetchFault -> Text
publishFaultReason FetchFault
fault)

-- The mirror target is operator-configured, so its rendered transport detail is diagnosable
-- rather than attacker-supplied.
publishFaultReason :: FetchFault -> Text
publishFaultReason :: FetchFault -> Text
publishFaultReason = \case
    FetchUrlUnformable UrlFormationError
urlErr -> Text
"unformable publish URL: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> UrlFormationError -> Text
renderUrlFormationError UrlFormationError
urlErr
    FetchBoundExceeded LimitError
limitErr -> Text
"the publication target's response exceeded the response bound: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> LimitError -> Text
forall b a. (Show a, IsString b) => a -> b
show LimitError
limitErr
    FetchCooldown RetryAfter
_ -> Text
"the public upstream is cooling down"
    FetchTransport TransportFault
fault -> Text
"publish transport failure: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> TransportFault -> Text
forall b a. (Show a, IsString b) => a -> b
show TransportFault
fault