module Ecluse.Core.Worker.Job (
JobOutcome (..),
RetryLeg (..),
mirrorLatest,
outcomeOfAdmission,
outcomeOfFetchFault,
processJob,
) where
import Control.Monad (foldM)
import Control.Monad.Except (throwError)
import Data.Aeson (object, (.=))
import Data.Map.Strict qualified as Map
import Data.Traversable (for)
import Katip (Severity (DebugS, ErrorS, InfoS), SimpleLogPayload, katipAddContext, katipAddNamespace, logFM, ls, sl)
import UnliftIO (withRunInIO)
import Ecluse.Core.Ecosystem (ecosystemName)
import Ecluse.Core.Fault (tfCause)
import Ecluse.Core.Mirror.State
import Ecluse.Core.Mirror.State.Format (MirrorStateFormat (msfCompanions))
import Ecluse.Core.Package (Artifact (artFilename, artHashes, artSize, artUrl), Hash, PackageDetails (pkgArtifacts, pkgVersion), PackageInfo (..), hashAlg, hashValue, pkgEcosystem, renderHashAlg, renderPackageName)
import Ecluse.Core.Package.Admission (
ArtifactAdmission (
AdmissionAdmit,
AdmissionBelowFloor,
AdmissionDenied,
AdmissionFileAbsent,
AdmissionIntegrityMissing,
AdmissionUndecidable
),
admissionTransience,
admitArtifactWithDecision,
)
import Ecluse.Core.Package.Integrity (VersionIntegrity (..), classifyDigests, partitionByFloor)
import Ecluse.Core.Queue (MirrorJob (jobPackage, jobTraceContext, jobTriggerFilename, jobVersion))
import Ecluse.Core.Registry (
FetchFault (FetchBoundExceeded, FetchCooldown, FetchTransport, FetchUrlUnformable),
MirrorArtifact (MirrorArtifact, maFilename, maHashes, maSize),
PublishFault (PublishFetch, PublishIntegrityMismatch, PublishRejected, PublishSourceUnavailable),
renderUrlFormationError,
)
import Ecluse.Core.Registry.Adapter.Capability (AdapterArtifact (artifactByUrl))
import Ecluse.Core.Registry.Exchange.Types (ExchangeClient (ecDeadlines))
import Ecluse.Core.Registry.Metadata (
MetadataError (..),
VersionDoc (vdDetails),
VersionEvaluation (VersionCooling, VersionMetadataUnavailable, VersionMissing, VersionPresent),
versionTransience,
)
import Ecluse.Core.Registry.MirrorUpload
import Ecluse.Core.Registry.Publish (
MirrorCompanion (..),
MirrorPublish (mpMirrorCompanion, mpNativeComplete, mpPrepareVersion, mpProbeInventory),
PreparedMirrorUpload,
PreparedMirrorVersion,
PublishPlan (PublishPlan),
pmvInventory,
pmvPrepareUpload,
publishPrepared,
)
import Ecluse.Core.Registry.Request (sidecarUrl)
import Ecluse.Core.Rules.Outage (logSkippedCheckEvidence)
import Ecluse.Core.Rules.Render (renderDecision, renderReason)
import Ecluse.Core.Rules.Types (Decision (Admitted, Blocked, BlockedByDefault, Undecidable), Transience (WillResolve, WontResolve), identityEvidence, mkEvalContext, skippedChecks)
import Ecluse.Core.Security (Limits (maxMetadataBytes, maxMirrorArtifactBytes), authorityLabel, hostPortAddress)
import Ecluse.Core.Security.Egress (RegistryUrl, registryUrlText)
import Ecluse.Core.Server.Path (Filename, mkFilename, unFilename)
import Ecluse.Core.Telemetry.Record (WorkerMetricsPort (..), timedSeconds)
import Ecluse.Core.Telemetry.Span (JobSpanOutcome (JobSpanOutcome), WorkerTracingPort (..))
import Ecluse.Core.Version (Version, renderVersion, selectLatest)
import Ecluse.Core.Worker.Fetch (fetchArtifactBytesCharged)
import Ecluse.Core.Worker.Managed
import Ecluse.Core.Worker.Types
data JobOutcome
=
Succeeded
|
Dropped Text
|
SourceUnavailable Text
|
DeadLettered Text
|
Retried RetryLeg Text
deriving stock (JobOutcome -> JobOutcome -> Bool
(JobOutcome -> JobOutcome -> Bool)
-> (JobOutcome -> JobOutcome -> Bool) -> Eq JobOutcome
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: JobOutcome -> JobOutcome -> Bool
== :: JobOutcome -> JobOutcome -> Bool
$c/= :: JobOutcome -> JobOutcome -> Bool
/= :: JobOutcome -> JobOutcome -> Bool
Eq, Int -> JobOutcome -> ShowS
[JobOutcome] -> ShowS
JobOutcome -> String
(Int -> JobOutcome -> ShowS)
-> (JobOutcome -> String)
-> ([JobOutcome] -> ShowS)
-> Show JobOutcome
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> JobOutcome -> ShowS
showsPrec :: Int -> JobOutcome -> ShowS
$cshow :: JobOutcome -> String
show :: JobOutcome -> String
$cshowList :: [JobOutcome] -> ShowS
showList :: [JobOutcome] -> ShowS
Show)
data RetryLeg
=
BeforePublish
|
AfterPublish
deriving stock (RetryLeg -> RetryLeg -> Bool
(RetryLeg -> RetryLeg -> Bool)
-> (RetryLeg -> RetryLeg -> Bool) -> Eq RetryLeg
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: RetryLeg -> RetryLeg -> Bool
== :: RetryLeg -> RetryLeg -> Bool
$c/= :: RetryLeg -> RetryLeg -> Bool
/= :: RetryLeg -> RetryLeg -> Bool
Eq, Int -> RetryLeg -> ShowS
[RetryLeg] -> ShowS
RetryLeg -> String
(Int -> RetryLeg -> ShowS)
-> (RetryLeg -> String) -> ([RetryLeg] -> ShowS) -> Show RetryLeg
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> RetryLeg -> ShowS
showsPrec :: Int -> RetryLeg -> ShowS
$cshow :: RetryLeg -> String
show :: RetryLeg -> String
$cshowList :: [RetryLeg] -> ShowS
showList :: [RetryLeg] -> ShowS
Show)
jobSpanOutcome :: JobOutcome -> JobSpanOutcome
jobSpanOutcome :: JobOutcome -> JobSpanOutcome
jobSpanOutcome = \case
JobOutcome
Succeeded -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"succeeded" Maybe Text
forall a. Maybe a
Nothing
Dropped Text
reason -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"dropped" (Text -> Maybe Text
forall a. a -> Maybe a
Just Text
reason)
SourceUnavailable Text
reason -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"source-unavailable" (Text -> Maybe Text
forall a. a -> Maybe a
Just Text
reason)
DeadLettered Text
reason -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"dead-lettered" (Text -> Maybe Text
forall a. a -> Maybe a
Just Text
reason)
Retried RetryLeg
_ Text
reason -> Text -> Maybe Text -> JobSpanOutcome
JobSpanOutcome Text
"retried" (Text -> Maybe Text
forall a. a -> Maybe a
Just Text
reason)
processJob :: MirrorJob -> WorkerM JobOutcome
processJob :: MirrorJob -> WorkerM JobOutcome
processJob MirrorJob
job = Namespace -> WorkerM JobOutcome -> WorkerM JobOutcome
forall (m :: * -> *) a. KatipContext m => Namespace -> m a -> m a
katipAddNamespace Namespace
"job" (WorkerM JobOutcome -> WorkerM JobOutcome)
-> WorkerM JobOutcome -> WorkerM JobOutcome
forall a b. (a -> b) -> a -> b
$ do
Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
DebugS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"starting mirror job for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job))
tracing <- (WorkerRuntime -> WorkerTracingPort) -> WorkerM WorkerTracingPort
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> WorkerTracingPort
wrTracing
runtime <- ask
withRunInIO $ \forall a. WorkerM a -> IO a
runInIO ->
WorkerTracingPort
-> forall a.
PackageName
-> Version
-> Maybe RemoteSpanContext
-> (a -> JobSpanOutcome)
-> IO a
-> IO a
wtpMirrorJobSpan WorkerTracingPort
tracing (MirrorJob -> PackageName
jobPackage MirrorJob
job) (MirrorJob -> Version
jobVersion MirrorJob
job) (MirrorJob -> Maybe RemoteSpanContext
jobTraceContext MirrorJob
job) JobOutcome -> JobSpanOutcome
jobSpanOutcome (IO JobOutcome -> IO JobOutcome) -> IO JobOutcome -> IO JobOutcome
forall a b. (a -> b) -> a -> b
$
WorkerM JobOutcome -> IO JobOutcome
forall a. WorkerM a -> IO a
runInIO (WorkerM JobOutcome -> IO JobOutcome)
-> WorkerM JobOutcome -> IO JobOutcome
forall a b. (a -> b) -> a -> b
$
WorkerRuntime
-> forall (m :: * -> *) a.
(KatipContext m, MonadIO m) =>
m a -> m a
wrInjectTraceContext WorkerRuntime
runtime (MirrorJob -> WorkerM JobOutcome
reevaluateThenMirror MirrorJob
job)
reevaluateThenMirror :: MirrorJob -> WorkerM JobOutcome
reevaluateThenMirror :: MirrorJob -> WorkerM JobOutcome
reevaluateThenMirror MirrorJob
job = do
policies <- (WorkerRuntime -> WorkerPolicies) -> WorkerM WorkerPolicies
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> WorkerPolicies
wrPolicies
case Map.lookup (pkgEcosystem (jobPackage job)) policies of
Maybe WorkerPolicy
Nothing -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
Dropped (MirrorJob -> Text
noPolicyReason MirrorJob
job))
Just WorkerPolicy
policy
| WorkerPolicy -> PackageName -> Bool
wpFirstParty WorkerPolicy
policy (MirrorJob -> PackageName
jobPackage MirrorJob
job) -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
Dropped (MirrorJob -> Text
firstPartyReason MirrorJob
job))
| Bool
otherwise -> WorkerPolicy
-> (WorkerStateBudget -> WorkerM JobOutcome) -> WorkerM JobOutcome
withMirrorBudget WorkerPolicy
policy (\WorkerStateBudget
charge -> WorkerStateBudget
-> WorkerPolicy -> MirrorJob -> WorkerM JobOutcome
mirrorUnlessPresent WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job)
withMirrorBudget :: WorkerPolicy -> (WorkerStateBudget -> WorkerM JobOutcome) -> WorkerM JobOutcome
withMirrorBudget :: WorkerPolicy
-> (WorkerStateBudget -> WorkerM JobOutcome) -> WorkerM JobOutcome
withMirrorBudget WorkerPolicy
policy WorkerStateBudget -> WorkerM JobOutcome
action = case WorkerPolicy -> Maybe WorkerMirrorState
wpMirrorState WorkerPolicy
policy of
Maybe WorkerMirrorState
Nothing -> WorkerStateBudget -> WorkerM JobOutcome
action (StateCharge
-> StateCharge
-> StateCharge
-> StateCharge
-> StateCharge
-> StateCharge
-> StateCharge
-> WorkerStateBudget
WorkerStateBudget StateCharge
forall {b}. b -> IO ()
ignore StateCharge
forall {b}. b -> IO ()
ignore StateCharge
forall {b}. b -> IO ()
ignore StateCharge
forall {b}. b -> IO ()
ignore StateCharge
forall {b}. b -> IO ()
ignore StateCharge
forall {b}. b -> IO ()
ignore StateCharge
forall {b}. b -> IO ()
ignore)
where
ignore :: b -> IO ()
ignore = IO () -> b -> IO ()
forall a b. a -> b -> a
const (() -> IO ()
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ())
Just WorkerMirrorState
binding -> ((forall a. WorkerM a -> IO a) -> IO JobOutcome)
-> WorkerM JobOutcome
forall b. ((forall a. WorkerM a -> IO a) -> IO b) -> WorkerM b
forall (m :: * -> *) b.
MonadUnliftIO m =>
((forall a. m a -> IO a) -> IO b) -> m b
withRunInIO (((forall a. WorkerM a -> IO a) -> IO JobOutcome)
-> WorkerM JobOutcome)
-> ((forall a. WorkerM a -> IO a) -> IO JobOutcome)
-> WorkerM JobOutcome
forall a b. (a -> b) -> a -> b
$ \forall a. WorkerM a -> IO a
run ->
JobOutcome -> Maybe JobOutcome -> JobOutcome
forall a. a -> Maybe a -> a
fromMaybe (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
"worker memory admission refused managed mirror job") (Maybe JobOutcome -> JobOutcome)
-> IO (Maybe JobOutcome) -> IO JobOutcome
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> WorkerMirrorState
-> forall a. (WorkerStateBudget -> IO a) -> IO (Maybe a)
wmsWithinBudget WorkerMirrorState
binding (WorkerM JobOutcome -> IO JobOutcome
forall a. WorkerM a -> IO a
run (WorkerM JobOutcome -> IO JobOutcome)
-> (WorkerStateBudget -> WorkerM JobOutcome)
-> WorkerStateBudget
-> IO JobOutcome
forall b c a. (b -> c) -> (a -> b) -> a -> c
. WorkerStateBudget -> WorkerM JobOutcome
action)
noPolicyReason :: MirrorJob -> Text
noPolicyReason :: MirrorJob -> Text
noPolicyReason MirrorJob
job =
Text
"no rule policy is configured for the "
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Ecosystem -> Text
ecosystemName (PackageName -> Ecosystem
pkgEcosystem (MirrorJob -> PackageName
jobPackage MirrorJob
job))
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" ecosystem; refusing to mirror "
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job
firstPartyReason :: MirrorJob -> Text
firstPartyReason :: MirrorJob -> Text
firstPartyReason MirrorJob
job =
Text
"this deployment owns the namespace of "
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"; refusing to mirror public content under a first-party name"
mirrorUnlessPresent :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> WorkerM JobOutcome
mirrorUnlessPresent :: WorkerStateBudget
-> WorkerPolicy -> MirrorJob -> WorkerM JobOutcome
mirrorUnlessPresent WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job = do
result <- ExceptT JobOutcome WorkerM Severity
-> WorkerM (Either JobOutcome Severity)
forall e (m :: * -> *) a. ExceptT e m a -> m (Either e a)
runExceptT (ExceptT JobOutcome WorkerM Severity
-> WorkerM (Either JobOutcome Severity))
-> ExceptT JobOutcome WorkerM Severity
-> WorkerM (Either JobOutcome Severity)
forall a b. (a -> b) -> a -> b
$ do
target <- WorkerM (Either JobOutcome PackageInfo)
-> ExceptT JobOutcome WorkerM PackageInfo
forall e (m :: * -> *) a. m (Either e a) -> ExceptT e m a
ExceptT (WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> WorkerM (Either JobOutcome PackageInfo)
probeInventory WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job)
if isNothing (wpMirrorState policy) && triggerPresent (\Text
name -> (Artifact -> Bool) -> [Artifact] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
any ((Text -> Text -> Bool
forall a. Eq a => a -> a -> Bool
== Text
name) (Text -> Bool) -> (Artifact -> Text) -> Artifact -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Artifact -> Text
artFilename) (MirrorJob -> PackageInfo -> [Artifact]
targetFiles MirrorJob
job PackageInfo
target)) && mpNativeComplete (wpPublish policy) (jobPackage job) (jobVersion job) target
then pure DebugS
else do
previous <- readRecord charge policy job Nothing
case previous >>= recordVersion (jobVersion job) of
Just MirrorVersion
version
| (Text -> Bool) -> Bool
triggerPresent (Text -> Map Text ExpectedMirrorFile -> Bool
forall k a. Ord k => k -> Map k a -> Bool
`Map.member` MirrorVersion -> Map Text ExpectedMirrorFile
versionFiles MirrorVersion
version)
, MirrorVersion -> [Artifact] -> MirrorCompletion
checkMirrorCompletion MirrorVersion
version (MirrorJob -> PackageInfo -> [Artifact]
targetFiles MirrorJob
job PackageInfo
target) MirrorCompletion -> MirrorCompletion -> Bool
forall a. Eq a => a -> a -> Bool
== MirrorCompletion
MirrorComplete ->
Severity
DebugS Severity
-> ExceptT JobOutcome WorkerM ()
-> ExceptT JobOutcome WorkerM Severity
forall a b.
a -> ExceptT JobOutcome WorkerM b -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ WorkerStateBudget
-> WorkerPolicy -> MirrorVersion -> ExceptT JobOutcome WorkerM ()
verifyRecordedCompanions WorkerStateBudget
charge WorkerPolicy
policy MirrorVersion
version
Maybe MirrorVersion
_ -> Severity
InfoS Severity
-> ExceptT JobOutcome WorkerM ()
-> ExceptT JobOutcome WorkerM Severity
forall a b.
a -> ExceptT JobOutcome WorkerM b -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> Int
-> PackageInfo
-> Maybe ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ()
completePass WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job Int
3 PackageInfo
target Maybe ProjectMirrorRecord
previous
case result of
Left JobOutcome
failure -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure JobOutcome
failure
Right Severity
severity -> JobOutcome
Succeeded JobOutcome -> WorkerM () -> WorkerM JobOutcome
forall a b. a -> WorkerM b -> WorkerM a
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
severity (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"mirror version complete: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job))
where
triggerPresent :: (Text -> Bool) -> Bool
triggerPresent Text -> Bool
contains = Bool -> (Filename -> Bool) -> Maybe Filename -> Bool
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Bool
True (Text -> Bool
contains (Text -> Bool) -> (Filename -> Text) -> Filename -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Filename -> Text
unFilename) (MirrorJob -> Maybe Filename
jobTriggerFilename MirrorJob
job)
completePass :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> Int -> PackageInfo -> Maybe ProjectMirrorRecord -> ExceptT JobOutcome WorkerM ()
completePass :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> Int
-> PackageInfo
-> Maybe ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ()
completePass WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job Int
attempts PackageInfo
target Maybe ProjectMirrorRecord
previous = do
admitted <- WorkerM (Either JobOutcome Readmitted)
-> ExceptT JobOutcome WorkerM Readmitted
forall e (m :: * -> *) a. m (Either e a) -> ExceptT e m a
ExceptT (WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> WorkerM (Either JobOutcome Readmitted)
reevaluatePolicy WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job)
record <- stateAction (persistInventory charge (wpMirrorFormat policy) (wpArtifactLimits policy) (wpMirrorState policy) (pmvInventory (raPrepared admitted)) previous)
copied <- copyRecorded charge policy job target admitted record
finalTarget <- ExceptT (probeInventory charge policy job)
latest <- readRecord charge policy job (Just copied)
current <- maybe (throwError (Retried AfterPublish "mirror record disappeared before completion")) pure latest
version <- requiredVersion job current
let copiedNames = Map Text ExpectedMirrorFile
-> (MirrorVersion -> Map Text ExpectedMirrorFile)
-> Maybe MirrorVersion
-> Map Text ExpectedMirrorFile
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Map Text ExpectedMirrorFile
forall a. Monoid a => a
mempty MirrorVersion -> Map Text ExpectedMirrorFile
versionFiles (Version -> ProjectMirrorRecord -> Maybe MirrorVersion
recordVersion (MirrorJob -> Version
jobVersion MirrorJob
job) ProjectMirrorRecord
record)
case checkMirrorCompletion version (targetFiles job finalTarget) of
MirrorCompletion
MirrorComplete -> WorkerStateBudget
-> WorkerPolicy -> MirrorVersion -> ExceptT JobOutcome WorkerM ()
verifyRecordedCompanions WorkerStateBudget
charge WorkerPolicy
policy MirrorVersion
version
MirrorIncomplete NonEmpty MirrorCompletionFault
faults
| (MirrorCompletionFault -> Bool)
-> NonEmpty MirrorCompletionFault -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
any MirrorCompletionFault -> Bool
conflicting NonEmpty MirrorCompletionFault
faults -> JobOutcome -> ExceptT JobOutcome WorkerM ()
forall a. JobOutcome -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. MonadError e m => e -> m a
throwError (Text -> JobOutcome
Dropped (Text
"mirror target identity conflicts: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> NonEmpty MirrorCompletionFault -> Text
forall b a. (Show a, IsString b) => a -> b
show NonEmpty MirrorCompletionFault
faults))
| Int
attempts Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Int
1 Bool -> Bool -> Bool
&& (MirrorCompletionFault -> Bool)
-> NonEmpty MirrorCompletionFault -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
any (Map Text ExpectedMirrorFile -> MirrorCompletionFault -> Bool
forall {a}. Map Text a -> MirrorCompletionFault -> Bool
newlyRequired Map Text ExpectedMirrorFile
copiedNames) NonEmpty MirrorCompletionFault
faults -> WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> Int
-> PackageInfo
-> Maybe ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ()
completePass WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job (Int
attempts Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1) PackageInfo
finalTarget (ProjectMirrorRecord -> Maybe ProjectMirrorRecord
forall a. a -> Maybe a
Just ProjectMirrorRecord
current)
| Bool
otherwise -> JobOutcome -> ExceptT JobOutcome WorkerM ()
forall a. JobOutcome -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. MonadError e m => e -> m a
throwError (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
AfterPublish (Text
"mirror remains incomplete: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> NonEmpty MirrorCompletionFault -> Text
forall b a. (Show a, IsString b) => a -> b
show NonEmpty MirrorCompletionFault
faults))
where
conflicting :: MirrorCompletionFault -> Bool
conflicting (ConflictingMirrorFile Text
_) = Bool
True
conflicting MirrorCompletionFault
_ = Bool
False
newlyRequired :: Map Text a -> MirrorCompletionFault -> Bool
newlyRequired Map Text a
known (MissingMirrorFile Text
name) = Text -> Map Text a -> Bool
forall k a. Ord k => k -> Map k a -> Bool
Map.notMember Text
name Map Text a
known
newlyRequired Map Text a
known (PendingMirrorCompanion Text
name) = Text -> Map Text a -> Bool
forall k a. Ord k => k -> Map k a -> Bool
Map.notMember Text
name Map Text a
known
newlyRequired Map Text a
_ MirrorCompletionFault
_ = Bool
False
readRecord :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> Maybe ProjectMirrorRecord -> ExceptT JobOutcome WorkerM (Maybe ProjectMirrorRecord)
readRecord :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> Maybe ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM (Maybe ProjectMirrorRecord)
readRecord WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job Maybe ProjectMirrorRecord
fallback = case WorkerPolicy -> Maybe WorkerMirrorState
wpMirrorState WorkerPolicy
policy of
Maybe WorkerMirrorState
Nothing -> Maybe ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM (Maybe ProjectMirrorRecord)
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure Maybe ProjectMirrorRecord
fallback
Just WorkerMirrorState
binding -> IO (Either MirrorStateFault (Maybe ProjectMirrorRecord))
-> ExceptT JobOutcome WorkerM (Maybe ProjectMirrorRecord)
forall a.
IO (Either MirrorStateFault a) -> ExceptT JobOutcome WorkerM a
stateAction (WorkerStateBudget
-> WorkerMirrorState
-> PackageName
-> IO (Either MirrorStateFault (Maybe ProjectMirrorRecord))
readMirrorRecord WorkerStateBudget
charge WorkerMirrorState
binding (MirrorJob -> PackageName
jobPackage MirrorJob
job))
verifyRecordedCompanions :: WorkerStateBudget -> WorkerPolicy -> MirrorVersion -> ExceptT JobOutcome WorkerM ()
verifyRecordedCompanions :: WorkerStateBudget
-> WorkerPolicy -> MirrorVersion -> ExceptT JobOutcome WorkerM ()
verifyRecordedCompanions WorkerStateBudget
charge WorkerPolicy
policy MirrorVersion
version = WorkerM (Either JobOutcome ()) -> ExceptT JobOutcome WorkerM ()
forall e (m :: * -> *) a. m (Either e a) -> ExceptT e m a
ExceptT (WorkerM (Either JobOutcome ()) -> ExceptT JobOutcome WorkerM ())
-> WorkerM (Either JobOutcome ()) -> ExceptT JobOutcome WorkerM ()
forall a b. (a -> b) -> a -> b
$ ((forall a. WorkerM a -> IO a) -> IO (Either JobOutcome ()))
-> WorkerM (Either JobOutcome ())
forall b. ((forall a. WorkerM a -> IO a) -> IO b) -> WorkerM b
forall (m :: * -> *) b.
MonadUnliftIO m =>
((forall a. m a -> IO a) -> IO b) -> m b
withRunInIO (((forall a. WorkerM a -> IO a) -> IO (Either JobOutcome ()))
-> WorkerM (Either JobOutcome ()))
-> ((forall a. WorkerM a -> IO a) -> IO (Either JobOutcome ()))
-> WorkerM (Either JobOutcome ())
forall a b. (a -> b) -> a -> b
$ \forall a. WorkerM a -> IO a
run ->
(MirrorStateFault -> JobOutcome)
-> Either MirrorStateFault () -> Either JobOutcome ()
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first MirrorStateFault -> JobOutcome
stateOutcome (Either MirrorStateFault () -> Either JobOutcome ())
-> IO (Either MirrorStateFault ()) -> IO (Either JobOutcome ())
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> WorkerStateBudget
-> IO ()
-> Maybe WorkerMirrorState
-> MirrorVersion
-> IO (Either MirrorStateFault ())
verifyCompanions WorkerStateBudget
charge (WorkerM () -> IO ()
forall a. WorkerM a -> IO a
run WorkerM ()
recordWorkerProgress) (WorkerPolicy -> Maybe WorkerMirrorState
wpMirrorState WorkerPolicy
policy) MirrorVersion
version
stateAction :: IO (Either MirrorStateFault a) -> ExceptT JobOutcome WorkerM a
stateAction :: forall a.
IO (Either MirrorStateFault a) -> ExceptT JobOutcome WorkerM a
stateAction IO (Either MirrorStateFault a)
action = WorkerM (Either JobOutcome a) -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. m (Either e a) -> ExceptT e m a
ExceptT ((Either MirrorStateFault a -> Either JobOutcome a)
-> WorkerM (Either MirrorStateFault a)
-> WorkerM (Either JobOutcome a)
forall a b. (a -> b) -> WorkerM a -> WorkerM b
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
fmap ((MirrorStateFault -> JobOutcome)
-> Either MirrorStateFault a -> Either JobOutcome a
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first MirrorStateFault -> JobOutcome
stateOutcome) (IO (Either MirrorStateFault a)
-> WorkerM (Either MirrorStateFault a)
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO IO (Either MirrorStateFault a)
action))
stateOutcome :: MirrorStateFault -> JobOutcome
stateOutcome :: MirrorStateFault -> JobOutcome
stateOutcome MirrorStateFault
fault = case MirrorStateFault
fault of
StateBound LimitError
_ -> Text -> JobOutcome
DeadLettered Text
detail
StateInvalid Text
_ -> Text -> JobOutcome
DeadLettered Text
detail
MirrorStateFault
StateIdentityMismatch -> Text -> JobOutcome
DeadLettered Text
detail
StateFileConflict Text
_ -> Text -> JobOutcome
Dropped Text
detail
StateCompanionConflict Text
_ -> Text -> JobOutcome
Dropped Text
detail
StateTransport TransportFault
_ -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
detail
MirrorStateFault
StateRevisionMismatch -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
detail
MirrorStateFault
StateMissingCompanion -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
detail
MirrorStateFault
StateRetryExhausted -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
detail
where
detail :: Text
detail = Text
"mirror state unavailable: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorStateFault -> Text
forall b a. (Show a, IsString b) => a -> b
show MirrorStateFault
fault
requiredVersion :: MirrorJob -> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM MirrorVersion
requiredVersion :: MirrorJob
-> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM MirrorVersion
requiredVersion MirrorJob
job = ExceptT JobOutcome WorkerM MirrorVersion
-> (MirrorVersion -> ExceptT JobOutcome WorkerM MirrorVersion)
-> Maybe MirrorVersion
-> ExceptT JobOutcome WorkerM MirrorVersion
forall b a. b -> (a -> b) -> Maybe a -> b
maybe (JobOutcome -> ExceptT JobOutcome WorkerM MirrorVersion
forall a. JobOutcome -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. MonadError e m => e -> m a
throwError (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
"missing recorded mirror version")) MirrorVersion -> ExceptT JobOutcome WorkerM MirrorVersion
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Maybe MirrorVersion -> ExceptT JobOutcome WorkerM MirrorVersion)
-> (ProjectMirrorRecord -> Maybe MirrorVersion)
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM MirrorVersion
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Version -> ProjectMirrorRecord -> Maybe MirrorVersion
recordVersion (MirrorJob -> Version
jobVersion MirrorJob
job)
targetFiles :: MirrorJob -> PackageInfo -> [Artifact]
targetFiles :: MirrorJob -> PackageInfo -> [Artifact]
targetFiles MirrorJob
job PackageInfo
target = [Artifact]
-> (PackageDetails -> [Artifact])
-> Maybe PackageDetails
-> [Artifact]
forall b a. b -> (a -> b) -> Maybe a -> b
maybe [] (NonEmpty Artifact -> [Artifact]
forall a. NonEmpty a -> [a]
forall (t :: * -> *) a. Foldable t => t a -> [a]
toList (NonEmpty Artifact -> [Artifact])
-> (PackageDetails -> NonEmpty Artifact)
-> PackageDetails
-> [Artifact]
forall b c a. (b -> c) -> (a -> b) -> a -> c
. PackageDetails -> NonEmpty Artifact
pkgArtifacts) (Text -> Map Text PackageDetails -> Maybe PackageDetails
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup (Version -> Text
renderVersion (MirrorJob -> Version
jobVersion MirrorJob
job)) (PackageInfo -> Map Text PackageDetails
infoVersions PackageInfo
target))
probeInventory :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> WorkerM (Either JobOutcome PackageInfo)
probeInventory :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> WorkerM (Either JobOutcome PackageInfo)
probeInventory WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job = do
probed <- IO (Either MetadataError PackageInfo)
-> WorkerM (Either MetadataError PackageInfo)
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (MirrorPublish
-> StateCharge
-> PackageName
-> IO (Either MetadataError PackageInfo)
mpProbeInventory (WorkerPolicy -> MirrorPublish
wpPublish WorkerPolicy
policy) (WorkerStateBudget -> StateCharge
wsbTargets WorkerStateBudget
charge) (MirrorJob -> PackageName
jobPackage MirrorJob
job))
pure $ case probed of
Right PackageInfo
inventory -> PackageInfo -> Either JobOutcome PackageInfo
forall a b. b -> Either a b
Right PackageInfo
inventory
Left MetadataError
MetadataAbsent -> PackageInfo -> Either JobOutcome PackageInfo
forall a b. b -> Either a b
Right (PackageName
-> Map Text PackageDetails
-> Map Text Version
-> [InvalidEntry]
-> PackageInfo
PackageInfo (MirrorJob -> PackageName
jobPackage MirrorJob
job) Map Text PackageDetails
forall a. Monoid a => a
mempty Map Text Version
forall a. Monoid a => a
mempty [])
Left (MetadataFetch FetchFault
fault) -> JobOutcome -> Either JobOutcome PackageInfo
forall a b. a -> Either a b
Left (RetryLeg -> (FetchFault -> Text) -> FetchFault -> JobOutcome
outcomeOfFetchFault RetryLeg
BeforePublish (MirrorJob -> FetchFault -> Text
probeFaultReason MirrorJob
job) FetchFault
fault)
Left (MetadataBoundExceeded LimitError
limit) -> JobOutcome -> Either JobOutcome PackageInfo
forall a b. a -> Either a b
Left (RetryLeg -> (FetchFault -> Text) -> FetchFault -> JobOutcome
outcomeOfFetchFault RetryLeg
BeforePublish (MirrorJob -> FetchFault -> Text
probeFaultReason MirrorJob
job) (LimitError -> FetchFault
FetchBoundExceeded LimitError
limit))
Left (MetadataAuthorisationFailure Int
status) -> JobOutcome -> Either JobOutcome PackageInfo
forall a b. a -> Either a b
Left (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish (MirrorJob -> Int -> Text
probeStatusReason MirrorJob
job Int
status))
Left (MetadataHttpFailure Int
status Maybe RetryAfter
_) -> JobOutcome -> Either JobOutcome PackageInfo
forall a b. a -> Either a b
Left (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish (MirrorJob -> Int -> Text
probeStatusReason MirrorJob
job Int
status))
Left MetadataError
MetadataUndecodable -> JobOutcome -> Either JobOutcome PackageInfo
forall a b. a -> Either a b
Left (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish (MirrorJob -> Text -> Text
probeParseReason MirrorJob
job Text
"invalid mirror metadata"))
Left (MetadataNameMismatch Text
_) -> JobOutcome -> Either JobOutcome PackageInfo
forall a b. a -> Either a b
Left (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish (MirrorJob -> Text -> Text
probeParseReason MirrorJob
job Text
"mismatched package identity"))
probeFaultReason :: MirrorJob -> FetchFault -> Text
probeFaultReason :: MirrorJob -> FetchFault -> Text
probeFaultReason MirrorJob
job = \case
FetchUrlUnformable UrlFormationError
urlErr -> Text
"unformable mirror probe URL: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> UrlFormationError -> Text
renderUrlFormationError UrlFormationError
urlErr
FetchBoundExceeded LimitError
limitErr -> Text
"the mirror target's metadata exceeded the response bound: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> LimitError -> Text
forall b a. (Show a, IsString b) => a -> b
show LimitError
limitErr
FetchCooldown RetryAfter
_ -> Text
"the public upstream is cooling down"
FetchTransport TransportFault
fault -> Text
"mirror inventory probe for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" failed: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> TransportCause -> Text
forall b a. (Show a, IsString b) => a -> b
show (TransportFault -> TransportCause
tfCause TransportFault
fault)
probeStatusReason :: MirrorJob -> Int -> Text
probeStatusReason :: MirrorJob -> Int -> Text
probeStatusReason MirrorJob
job Int
code =
Text
"the mirror target answered HTTP "
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Int -> Text
forall b a. (Show a, IsString b) => a -> b
show Int
code
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" for the inventory probe of "
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"; refusing to publish a release tag chosen without it"
probeParseReason :: MirrorJob -> Text -> Text
probeParseReason :: MirrorJob -> Text -> Text
probeParseReason MirrorJob
job Text
detail =
Text
"could not read the mirror target's inventory for "
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" ("
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
detail
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"); refusing to publish a release tag chosen without it"
data Readmitted = Readmitted
{ Readmitted -> PreparedMirrorVersion
raPrepared :: PreparedMirrorVersion
, Readmitted -> NonEmpty (ExpectedMirrorFile, RegistryUrl)
raLocations :: NonEmpty (ExpectedMirrorFile, RegistryUrl)
, Readmitted -> Maybe Version
raUpstreamLatest :: Maybe Version
, Readmitted -> Decision
raDecision :: Decision
}
reevaluatePolicy :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> WorkerM (Either JobOutcome Readmitted)
reevaluatePolicy :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> WorkerM (Either JobOutcome Readmitted)
reevaluatePolicy WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job =
IO VersionEvaluation -> WorkerM VersionEvaluation
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (WorkerPolicy
-> StateCharge -> PackageName -> Version -> IO VersionEvaluation
wpResolveVersion WorkerPolicy
policy (WorkerStateBudget -> StateCharge
wsbSource WorkerStateBudget
charge) (MirrorJob -> PackageName
jobPackage MirrorJob
job) (MirrorJob -> Version
jobVersion MirrorJob
job)) WorkerM VersionEvaluation
-> (VersionEvaluation -> WorkerM (Either JobOutcome Readmitted))
-> WorkerM (Either JobOutcome Readmitted)
forall a b. WorkerM a -> (a -> WorkerM b) -> WorkerM b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= WorkerPolicy
-> MirrorJob
-> VersionEvaluation
-> WorkerM (Either JobOutcome Readmitted)
admitEvaluation WorkerPolicy
policy MirrorJob
job
admitEvaluation :: WorkerPolicy -> MirrorJob -> VersionEvaluation -> WorkerM (Either JobOutcome Readmitted)
admitEvaluation :: WorkerPolicy
-> MirrorJob
-> VersionEvaluation
-> WorkerM (Either JobOutcome Readmitted)
admitEvaluation WorkerPolicy
policy MirrorJob
job VersionEvaluation
evaluation = case VersionEvaluation
evaluation of
VersionCooling RetryAfter
_ -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left (Text -> JobOutcome
unresolved Text
"the public upstream is cooling down"))
VersionEvaluation
VersionMetadataUnavailable -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left (Text -> JobOutcome
unresolved (Text
"could not re-fetch metadata to re-evaluate current policy for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job)))
VersionEvaluation
VersionMissing -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left (Text -> JobOutcome
unresolved (Text
"the public upstream no longer offers " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job)))
VersionPresent VersionDoc
doc Maybe Version
upstreamLatest -> case WorkerPolicy -> MirrorJob -> VersionDoc -> Maybe Filename
triggerFile WorkerPolicy
policy MirrorJob
job VersionDoc
doc of
Maybe Filename
Nothing -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left (Text -> JobOutcome
Dropped Text
"the source inventory has no supported trigger artifact"))
Just Filename
trigger -> do
ctx <- IO EvalContext -> WorkerM EvalContext
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO UTCTime -> IO (Maybe DbEtag) -> IO EvalContext
mkEvalContext (WorkerPolicy -> IO UTCTime
wpNow WorkerPolicy
policy) (Maybe DbEtag -> IO (Maybe DbEtag)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure Maybe DbEtag
forall a. Maybe a
Nothing))
(admission, decision) <- liftIO (admitArtifactWithDecision ctx (wpRules policy) (wpMinIntegrity policy) trigger (vdDetails doc))
case outcomeOfAdmission job admission of
Left JobOutcome
outcome -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left JobOutcome
outcome)
Right MirrorArtifact
_ -> do
SimpleLogPayload -> [SkippedCheck] -> WorkerM ()
forall (m :: * -> *).
KatipContext m =>
SimpleLogPayload -> [SkippedCheck] -> m ()
logSkippedCheckEvidence
(Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"package" (PackageName -> Text
renderPackageName (MirrorJob -> PackageName
jobPackage MirrorJob
job)) SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"version" (Version -> Text
renderVersion (MirrorJob -> Version
jobVersion MirrorJob
job)))
(Decision -> [SkippedCheck]
skippedChecks Decision
decision)
case WorkerPolicy
-> MirrorJob
-> Filename
-> VersionDoc
-> Either JobOutcome VersionDoc
supportedVersion WorkerPolicy
policy MirrorJob
job Filename
trigger VersionDoc
doc of
Left JobOutcome
outcome -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left JobOutcome
outcome)
Right VersionDoc
supported -> case MirrorPublish
-> PackageName
-> Version
-> VersionDoc
-> Either PublishFault PreparedMirrorVersion
mpPrepareVersion (WorkerPolicy -> MirrorPublish
wpPublish WorkerPolicy
policy) (MirrorJob -> PackageName
jobPackage MirrorJob
job) (MirrorJob -> Version
jobVersion MirrorJob
job) VersionDoc
supported of
Left PublishFault
fault -> JobOutcome -> Either JobOutcome Readmitted
forall a b. a -> Either a b
Left (JobOutcome -> Either JobOutcome Readmitted)
-> WorkerM JobOutcome -> WorkerM (Either JobOutcome Readmitted)
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> MirrorJob -> Either PublishFault () -> WorkerM JobOutcome
outcomeOfPublish MirrorJob
job (PublishFault -> Either PublishFault ()
forall a b. a -> Either a b
Left PublishFault
fault)
Right PreparedMirrorVersion
prepared -> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted))
-> Either JobOutcome Readmitted
-> WorkerM (Either JobOutcome Readmitted)
forall a b. (a -> b) -> a -> b
$ do
locations <- (ExpectedMirrorFile
-> Either JobOutcome (ExpectedMirrorFile, RegistryUrl))
-> NonEmpty ExpectedMirrorFile
-> Either JobOutcome (NonEmpty (ExpectedMirrorFile, RegistryUrl))
forall (t :: * -> *) (f :: * -> *) a b.
(Traversable t, Applicative f) =>
(a -> f b) -> t a -> f (t b)
forall (f :: * -> *) a b.
Applicative f =>
(a -> f b) -> NonEmpty a -> f (NonEmpty b)
traverse (WorkerPolicy
-> MirrorJob
-> ExpectedMirrorFile
-> Either JobOutcome (ExpectedMirrorFile, RegistryUrl)
admittedLocation WorkerPolicy
policy MirrorJob
job) (MirrorInventory -> NonEmpty ExpectedMirrorFile
inventoryFiles (PreparedMirrorVersion -> MirrorInventory
pmvInventory PreparedMirrorVersion
prepared))
pure (Readmitted prepared locations upstreamLatest decision)
where
unresolved :: Text -> JobOutcome
unresolved = Maybe Transience -> Text -> JobOutcome
retryOrDrop (VersionEvaluation -> Maybe Transience
versionTransience VersionEvaluation
evaluation)
supportedVersion :: WorkerPolicy -> MirrorJob -> Filename -> VersionDoc -> Either JobOutcome VersionDoc
supportedVersion :: WorkerPolicy
-> MirrorJob
-> Filename
-> VersionDoc
-> Either JobOutcome VersionDoc
supportedVersion WorkerPolicy
policy MirrorJob
job Filename
trigger VersionDoc
doc = do
strong <- (VersionIntegrity -> JobOutcome)
-> Either VersionIntegrity (NonEmpty Artifact)
-> Either JobOutcome (NonEmpty Artifact)
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first (JobOutcome -> VersionIntegrity -> JobOutcome
forall a b. a -> b -> a
const (Text -> JobOutcome
Dropped Text
"no source artifact clears the integrity floor")) (MinIntegrity
-> (Artifact -> [Hash])
-> NonEmpty Artifact
-> Either VersionIntegrity (NonEmpty Artifact)
forall floor file.
IntegrityFloor floor =>
floor
-> (file -> [Hash])
-> NonEmpty file
-> Either VersionIntegrity (NonEmpty file)
partitionByFloor (WorkerPolicy -> MinIntegrity
wpMinIntegrity WorkerPolicy
policy) Artifact -> [Hash]
artHashes (PackageDetails -> NonEmpty Artifact
pkgArtifacts PackageDetails
details))
let kept = (Artifact -> Bool) -> [Artifact] -> [Artifact]
forall a. (a -> Bool) -> [a] -> [a]
filter (Either JobOutcome RegistryUrl -> Bool
forall a b. Either a b -> Bool
isRight (Either JobOutcome RegistryUrl -> Bool)
-> (Artifact -> Either JobOutcome RegistryUrl) -> Artifact -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. WorkerPolicy -> Artifact -> Either JobOutcome RegistryUrl
sourceLocation WorkerPolicy
policy) (NonEmpty Artifact -> [Artifact]
forall a. NonEmpty a -> [a]
forall (t :: * -> *) a. Foldable t => t a -> [a]
toList NonEmpty Artifact
strong)
survivors <- maybeToRight (locationRefusal strong) (nonEmpty kept)
unless (any ((== unFilename trigger) . artFilename) survivors) $
Left (Dropped ("the current location policy refuses the requested artifact of " <> renderJob job))
pure doc{vdDetails = details{pkgArtifacts = survivors}}
where
details :: PackageDetails
details = VersionDoc -> PackageDetails
vdDetails VersionDoc
doc
locationRefusal :: NonEmpty Artifact -> JobOutcome
locationRefusal (Artifact
firstArtifact :| [Artifact]
_) = JobOutcome -> Either JobOutcome RegistryUrl -> JobOutcome
forall a b. a -> Either a b -> a
fromLeft (Text -> JobOutcome
Dropped Text
"no supported artifact location") (WorkerPolicy -> Artifact -> Either JobOutcome RegistryUrl
sourceLocation WorkerPolicy
policy Artifact
firstArtifact)
sourceLocation :: WorkerPolicy -> Artifact -> Either JobOutcome RegistryUrl
sourceLocation :: WorkerPolicy -> Artifact -> Either JobOutcome RegistryUrl
sourceLocation WorkerPolicy
policy Artifact
artifact = do
Bool -> Either JobOutcome () -> Either JobOutcome ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
unless (WorkerPolicy -> Maybe HostPort -> Bool
wpArtifactHostHonoured WorkerPolicy
policy (Text -> Maybe HostPort
hostPortAddress (Artifact -> Text
artUrl Artifact
artifact))) (Either JobOutcome () -> Either JobOutcome ())
-> Either JobOutcome () -> Either JobOutcome ()
forall a b. (a -> b) -> a -> b
$
JobOutcome -> Either JobOutcome ()
forall a b. a -> Either a b
Left (Text -> JobOutcome
Dropped (Text
"the artifact-host policy refuses " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text -> Text
authorityLabel (Artifact -> Text
artUrl Artifact
artifact)))
(Text -> JobOutcome)
-> Either Text RegistryUrl -> Either JobOutcome RegistryUrl
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first (JobOutcome -> Text -> JobOutcome
forall a b. a -> b -> a
const (Text -> JobOutcome
Dropped Text
"the source artifact URL violates the egress policy")) (WorkerPolicy -> Text -> Either Text RegistryUrl
wpEgressUrl WorkerPolicy
policy (Artifact -> Text
artUrl Artifact
artifact))
triggerFile :: WorkerPolicy -> MirrorJob -> VersionDoc -> Maybe Filename
triggerFile :: WorkerPolicy -> MirrorJob -> VersionDoc -> Maybe Filename
triggerFile WorkerPolicy
policy MirrorJob
job VersionDoc
doc = MirrorJob -> Maybe Filename
jobTriggerFilename MirrorJob
job Maybe Filename -> Maybe Filename -> Maybe Filename
forall a. Maybe a -> Maybe a -> Maybe a
forall (f :: * -> *) a. Alternative f => f a -> f a -> f a
<|> ((Artifact -> Bool) -> NonEmpty Artifact -> Maybe Artifact
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Maybe a
find Artifact -> Bool
supported (PackageDetails -> NonEmpty Artifact
pkgArtifacts (VersionDoc -> PackageDetails
vdDetails VersionDoc
doc)) Maybe Artifact -> (Artifact -> Maybe Filename) -> Maybe Filename
forall a b. Maybe a -> (a -> Maybe b) -> Maybe b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= Text -> Maybe Filename
mkFilename (Text -> Maybe Filename)
-> (Artifact -> Text) -> Artifact -> Maybe Filename
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Artifact -> Text
artFilename)
where
supported :: Artifact -> Bool
supported Artifact
artifact = MinIntegrity -> [Hash] -> VersionIntegrity
forall floor.
IntegrityFloor floor =>
floor -> [Hash] -> VersionIntegrity
classifyDigests (WorkerPolicy -> MinIntegrity
wpMinIntegrity WorkerPolicy
policy) (Artifact -> [Hash]
artHashes Artifact
artifact) VersionIntegrity -> VersionIntegrity -> Bool
forall a. Eq a => a -> a -> Bool
== VersionIntegrity
MeetsFloor Bool -> Bool -> Bool
&& Either JobOutcome RegistryUrl -> Bool
forall a b. Either a b -> Bool
isRight (WorkerPolicy -> Artifact -> Either JobOutcome RegistryUrl
sourceLocation WorkerPolicy
policy Artifact
artifact)
admittedLocation :: WorkerPolicy -> MirrorJob -> ExpectedMirrorFile -> Either JobOutcome (ExpectedMirrorFile, RegistryUrl)
admittedLocation :: WorkerPolicy
-> MirrorJob
-> ExpectedMirrorFile
-> Either JobOutcome (ExpectedMirrorFile, RegistryUrl)
admittedLocation WorkerPolicy
policy MirrorJob
job ExpectedMirrorFile
file = do
case MinIntegrity -> [Hash] -> VersionIntegrity
forall floor.
IntegrityFloor floor =>
floor -> [Hash] -> VersionIntegrity
classifyDigests (WorkerPolicy -> MinIntegrity
wpMinIntegrity WorkerPolicy
policy) (Artifact -> [Hash]
artHashes Artifact
artifact) of
VersionIntegrity
MeetsFloor -> () -> Either JobOutcome ()
forall a b. b -> Either a b
Right ()
VersionIntegrity
BelowFloor -> Either JobOutcome MirrorArtifact -> Either JobOutcome ()
forall (f :: * -> *) a. Functor f => f a -> f ()
void (MirrorJob -> ArtifactAdmission -> Either JobOutcome MirrorArtifact
outcomeOfAdmission MirrorJob
job ArtifactAdmission
AdmissionBelowFloor)
VersionIntegrity
NoIntegrity -> Either JobOutcome MirrorArtifact -> Either JobOutcome ()
forall (f :: * -> *) a. Functor f => f a -> f ()
void (MirrorJob -> ArtifactAdmission -> Either JobOutcome MirrorArtifact
outcomeOfAdmission MirrorJob
job ArtifactAdmission
AdmissionIntegrityMissing)
location <- WorkerPolicy -> Artifact -> Either JobOutcome RegistryUrl
sourceLocation WorkerPolicy
policy Artifact
artifact
pure (file, location)
where
artifact :: Artifact
artifact = ExpectedMirrorFile -> Artifact
expectedArtifact ExpectedMirrorFile
file
sourceUnavailableReason :: MirrorJob -> Text -> Text
sourceUnavailableReason :: MirrorJob -> Text -> Text
sourceUnavailableReason MirrorJob
job Text
detail =
Text
"the source version object for "
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" was unavailable ("
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
detail
Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"); refusing to mirror a reduced manifest"
outcomeOfAdmission :: MirrorJob -> ArtifactAdmission -> Either JobOutcome MirrorArtifact
outcomeOfAdmission :: MirrorJob -> ArtifactAdmission -> Either JobOutcome MirrorArtifact
outcomeOfAdmission MirrorJob
job ArtifactAdmission
admission = case ArtifactAdmission
admission of
AdmissionAdmit Filename
filename Artifact
artifact NonEmpty Hash
digests -> MirrorArtifact -> Either JobOutcome MirrorArtifact
forall a b. b -> Either a b
Right (Filename -> Artifact -> NonEmpty Hash -> MirrorArtifact
readmittedDescriptor Filename
filename Artifact
artifact NonEmpty Hash
digests)
AdmissionDenied (Blocked Text
ruleName Maybe DbEtag
_ Reason
reason) ->
Text -> Either JobOutcome MirrorArtifact
refused (Text
"current policy denies " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": blocked by " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
ruleName Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" (" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Reason -> Text
renderReason Reason
reason Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
")")
AdmissionDenied Decision
_ ->
Text -> Either JobOutcome MirrorArtifact
refused (Text
"current policy denies " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": no rule admits it")
AdmissionUndecidable (Undecidable Transience
_ Reason
reason) ->
Text -> Either JobOutcome MirrorArtifact
refused (Text
"current policy could not be evaluated for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Reason -> Text
renderReason Reason
reason)
AdmissionUndecidable Decision
_ ->
Text -> Either JobOutcome MirrorArtifact
refused (Text
"current policy could not be evaluated for " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job)
ArtifactAdmission
AdmissionFileAbsent ->
Text -> Either JobOutcome MirrorArtifact
refused (Text
"the public upstream no longer offers the admitted artifact file of " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"; refusing to mirror a withdrawn artifact")
ArtifactAdmission
AdmissionBelowFloor ->
Text -> Either JobOutcome MirrorArtifact
refused (Text
"current admission policy refuses " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": its strongest integrity digest is below the configured public floor")
ArtifactAdmission
AdmissionIntegrityMissing ->
Text -> Either JobOutcome MirrorArtifact
refused (Text
"current admission policy refuses " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
": it no longer carries any integrity digest")
where
refused :: Text -> Either JobOutcome MirrorArtifact
refused :: Text -> Either JobOutcome MirrorArtifact
refused = JobOutcome -> Either JobOutcome MirrorArtifact
forall a b. a -> Either a b
Left (JobOutcome -> Either JobOutcome MirrorArtifact)
-> (Text -> JobOutcome) -> Text -> Either JobOutcome MirrorArtifact
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Maybe Transience -> Text -> JobOutcome
retryOrDrop (ArtifactAdmission -> Maybe Transience
admissionTransience ArtifactAdmission
admission)
retryOrDrop :: Maybe Transience -> Text -> JobOutcome
retryOrDrop :: Maybe Transience -> Text -> JobOutcome
retryOrDrop Maybe Transience
transience Text
reason = case Maybe Transience
transience of
Just (WillResolve Maybe RetryAfter
_) -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
reason
Just Transience
WontResolve -> Text -> JobOutcome
Dropped Text
reason
Maybe Transience
Nothing -> Text -> JobOutcome
Dropped Text
reason
readmittedDescriptor :: Filename -> Artifact -> NonEmpty Hash -> MirrorArtifact
readmittedDescriptor :: Filename -> Artifact -> NonEmpty Hash -> MirrorArtifact
readmittedDescriptor Filename
filename Artifact
artifact NonEmpty Hash
digests =
MirrorArtifact
{ maFilename :: Filename
maFilename = Filename
filename
, maHashes :: NonEmpty Hash
maHashes = NonEmpty Hash
digests
, maSize :: Maybe Int
maSize = Artifact -> Maybe Int
artSize Artifact
artifact
}
outcomeOfFetchFault :: RetryLeg -> (FetchFault -> Text) -> FetchFault -> JobOutcome
outcomeOfFetchFault :: RetryLeg -> (FetchFault -> Text) -> FetchFault -> JobOutcome
outcomeOfFetchFault RetryLeg
leg FetchFault -> Text
render FetchFault
fault = Text -> JobOutcome
verdict (FetchFault -> Text
render FetchFault
fault)
where
verdict :: Text -> JobOutcome
verdict = case FetchFault
fault of
FetchUrlUnformable UrlFormationError
_ -> Text -> JobOutcome
Dropped
FetchBoundExceeded LimitError
_ -> Text -> JobOutcome
DeadLettered
FetchTransport TransportFault
_ -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
leg
FetchCooldown RetryAfter
_ -> RetryLeg -> Text -> JobOutcome
Retried RetryLeg
leg
mirrorLatest :: Maybe Version -> [Version] -> Version -> Version
mirrorLatest :: Maybe Version -> [Version] -> Version -> Version
mirrorLatest Maybe Version
upstreamLatest [Version]
inventory Version
published =
Version -> Maybe Version -> Version
forall a. a -> Maybe a -> a
fromMaybe Version
published (Maybe Version -> [Version] -> Maybe Version
selectLatest Maybe Version
upstreamLatest (Version
published Version -> [Version] -> [Version]
forall a. a -> [a] -> [a]
: [Version]
inventory))
data CopyPass = CopyPass
{ CopyPass -> WorkerStateBudget
cpCharge :: ~WorkerStateBudget
, CopyPass -> WorkerPolicy
cpPolicy :: ~WorkerPolicy
, CopyPass -> MirrorJob
cpJob :: ~MirrorJob
, CopyPass -> PublishPlan
cpPlan :: ~PublishPlan
, CopyPass -> [Artifact]
cpTargetFiles :: ~[Artifact]
, CopyPass -> Readmitted
cpLatest :: ~Readmitted
}
copyRecorded :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> PackageInfo -> Readmitted -> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
copyRecorded :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> PackageInfo
-> Readmitted
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
copyRecorded WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job PackageInfo
target Readmitted
latest ProjectMirrorRecord
record = do
version <- MirrorJob
-> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM MirrorVersion
requiredVersion MirrorJob
job ProjectMirrorRecord
record
foldM (copyFragment context) record (toList (versionFragments version))
where
context :: CopyPass
context =
CopyPass
{ cpCharge :: WorkerStateBudget
cpCharge = WorkerStateBudget
charge
, cpPolicy :: WorkerPolicy
cpPolicy = WorkerPolicy
policy
, cpJob :: MirrorJob
cpJob = MirrorJob
job
, cpPlan :: PublishPlan
cpPlan = Version -> PublishPlan
PublishPlan (Maybe Version -> [Version] -> Version -> Version
mirrorLatest (Readmitted -> Maybe Version
raUpstreamLatest Readmitted
latest) ((PackageDetails -> Version) -> [PackageDetails] -> [Version]
forall a b. (a -> b) -> [a] -> [b]
map PackageDetails -> Version
pkgVersion (Map Text PackageDetails -> [PackageDetails]
forall k a. Map k a -> [a]
Map.elems (PackageInfo -> Map Text PackageDetails
infoVersions PackageInfo
target))) (MirrorJob -> Version
jobVersion MirrorJob
job))
, cpTargetFiles :: [Artifact]
cpTargetFiles = MirrorJob -> PackageInfo -> [Artifact]
targetFiles MirrorJob
job PackageInfo
target
, cpLatest :: Readmitted
cpLatest = Readmitted
latest
}
copyFragment :: CopyPass -> ProjectMirrorRecord -> StoredMirrorFragment -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
copyFragment :: CopyPass
-> ProjectMirrorRecord
-> StoredMirrorFragment
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
copyFragment CopyPass
context ProjectMirrorRecord
current StoredMirrorFragment
fragment = do
version <- MirrorJob
-> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM MirrorVersion
requiredVersion MirrorJob
job ProjectMirrorRecord
current
let files = (Artifact -> Bool) -> [Artifact] -> [Artifact]
forall a. (a -> Bool) -> [a] -> [a]
filter ([Artifact] -> MirrorVersion -> Artifact -> Bool
needsCopy (CopyPass -> [Artifact]
cpTargetFiles CopyPass
context) MirrorVersion
version) (NonEmpty Artifact -> [Artifact]
forall a. NonEmpty a -> [a]
forall (t :: * -> *) a. Foldable t => t a -> [a]
toList (PackageDetails -> NonEmpty Artifact
pkgArtifacts (StoredMirrorFragment -> PackageDetails
fragmentDetails StoredMirrorFragment
fragment)))
if null files
then pure current
else do
admitted <- ExceptT (admitEvaluation (cpPolicy context) job{jobTriggerFilename = Nothing} (VersionPresent (fragmentDocument fragment) (raUpstreamLatest (cpLatest context))))
foldM (copyFile context admitted) current (toList (raLocations admitted))
where
job :: MirrorJob
job = CopyPass -> MirrorJob
cpJob CopyPass
context
needsCopy :: [Artifact] -> MirrorVersion -> Artifact -> Bool
needsCopy :: [Artifact] -> MirrorVersion -> Artifact -> Bool
needsCopy [Artifact]
currentFiles MirrorVersion
version Artifact
artifact =
let name :: Text
name = Artifact -> Text
artFilename Artifact
artifact
matches :: [Artifact]
matches = (Artifact -> Bool) -> [Artifact] -> [Artifact]
forall a. (a -> Bool) -> [a] -> [a]
filter ((Text -> Text -> Bool
forall a. Eq a => a -> a -> Bool
== Text
name) (Text -> Bool) -> (Artifact -> Text) -> Artifact -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Artifact -> Text
artFilename) [Artifact]
currentFiles
verified :: Maybe Hash
verified = Text -> Map Text Hash -> Maybe Hash
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Text
name (MirrorVersion -> Map Text Hash
versionVerifiedIdentities MirrorVersion
version)
in Text -> Map Text CompanionState -> Maybe CompanionState
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Text
name (MirrorVersion -> Map Text CompanionState
versionCompanions MirrorVersion
version) Maybe CompanionState -> Maybe CompanionState -> Bool
forall a. Eq a => a -> a -> Bool
== CompanionState -> Maybe CompanionState
forall a. a -> Maybe a
Just CompanionState
CompanionPending
Bool -> Bool -> Bool
|| [Artifact] -> Bool
forall a. [a] -> Bool
forall (t :: * -> *) a. Foldable t => t a -> Bool
null [Artifact]
matches
Bool -> Bool -> Bool
|| Bool
-> (ExpectedMirrorFile -> Bool) -> Maybe ExpectedMirrorFile -> Bool
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Bool
True (\ExpectedMirrorFile
file -> Bool -> Bool
not ((Artifact -> Bool) -> [Artifact] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
all (ExpectedMirrorFile -> Maybe Hash -> Artifact -> Bool
expectedTargetMatchesWith ExpectedMirrorFile
file Maybe Hash
verified) [Artifact]
matches)) (Text -> Map Text ExpectedMirrorFile -> Maybe ExpectedMirrorFile
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Text
name (MirrorVersion -> Map Text ExpectedMirrorFile
versionFiles MirrorVersion
version))
copyFile :: CopyPass -> Readmitted -> ProjectMirrorRecord -> (ExpectedMirrorFile, RegistryUrl) -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
copyFile :: CopyPass
-> Readmitted
-> ProjectMirrorRecord
-> (ExpectedMirrorFile, RegistryUrl)
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
copyFile CopyPass
context Readmitted
admitted ProjectMirrorRecord
current (ExpectedMirrorFile
file, RegistryUrl
location) = do
version <- MirrorJob
-> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM MirrorVersion
requiredVersion MirrorJob
job ProjectMirrorRecord
current
let name = Artifact -> Text
artFilename (ExpectedMirrorFile -> Artifact
expectedArtifact ExpectedMirrorFile
file)
matches = (Artifact -> Bool) -> [Artifact] -> [Artifact]
forall a. (a -> Bool) -> [a] -> [a]
filter ((Text -> Text -> Bool
forall a. Eq a => a -> a -> Bool
== Text
name) (Text -> Bool) -> (Artifact -> Text) -> Artifact -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Artifact -> Text
artFilename) (CopyPass -> [Artifact]
cpTargetFiles CopyPass
context)
verified = Text -> Map Text Hash -> Maybe Hash
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Text
name (MirrorVersion -> Map Text Hash
versionVerifiedIdentities MirrorVersion
version)
locations = RegistryUrl
location RegistryUrl -> [RegistryUrl] -> NonEmpty RegistryUrl
forall a. a -> [a] -> NonEmpty a
:| [RegistryUrl
candidate | (ExpectedMirrorFile
incoming, RegistryUrl
candidate) <- NonEmpty (ExpectedMirrorFile, RegistryUrl)
-> [(ExpectedMirrorFile, RegistryUrl)]
forall a. NonEmpty a -> [a]
forall (t :: * -> *) a. Foldable t => t a -> [a]
toList (Readmitted -> NonEmpty (ExpectedMirrorFile, RegistryUrl)
raLocations (CopyPass -> Readmitted
cpLatest CopyPass
context)), ExpectedMirrorFile -> ExpectedMirrorFile -> Bool
expectedIdentityMatches ExpectedMirrorFile
file ExpectedMirrorFile
incoming, RegistryUrl
candidate RegistryUrl -> RegistryUrl -> Bool
forall a. Eq a => a -> a -> Bool
/= RegistryUrl
location]
when (any (targetIdentityConflict file verified) matches) $
throwError (Dropped ("mirror target identity conflicts: " <> name))
let targetProven = (Artifact -> Bool) -> [Artifact] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
all (ExpectedMirrorFile -> Maybe Hash -> Artifact -> Bool
expectedTargetMatchesWith ExpectedMirrorFile
file Maybe Hash
verified) [Artifact]
matches
needsNativeMetadata = Maybe MirrorCompanion -> Bool
forall a. Maybe a -> Bool
isJust (MirrorPublish -> Maybe MirrorCompanion
mpMirrorCompanion (WorkerPolicy -> MirrorPublish
wpPublish WorkerPolicy
policy)) Bool -> Bool -> Bool
&& Text -> Map Text CompanionState -> Maybe CompanionState
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Text
name (MirrorVersion -> Map Text CompanionState
versionCompanions MirrorVersion
version) Maybe CompanionState -> Maybe CompanionState -> Bool
forall a. Eq a => a -> a -> Bool
== CompanionState -> Maybe CompanionState
forall a. a -> Maybe a
Just CompanionState
CompanionPending
uploaded <- case matches of
[] -> WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> PublishPlan
-> Readmitted
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
mirrorArtifact WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job (CopyPass -> PublishPlan
cpPlan CopyPass
context) Readmitted
admitted ExpectedMirrorFile
file NonEmpty RegistryUrl
locations ProjectMirrorRecord
current
[Artifact]
_ | Bool
targetProven Bool -> Bool -> Bool
&& Bool -> Bool
not Bool
needsNativeMetadata -> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ProjectMirrorRecord
current
[Artifact]
_ -> CopyPass
-> Readmitted
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> [Artifact]
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
verifyExistingTarget CopyPass
context Readmitted
admitted ExpectedMirrorFile
file NonEmpty RegistryUrl
locations [Artifact]
matches ProjectMirrorRecord
current
retained <- retainPending charge policy job (pmvInventory (raPrepared admitted)) file locations uploaded
lift recordWorkerProgress
pure retained
where
charge :: WorkerStateBudget
charge = CopyPass -> WorkerStateBudget
cpCharge CopyPass
context
policy :: WorkerPolicy
policy = CopyPass -> WorkerPolicy
cpPolicy CopyPass
context
job :: MirrorJob
job = CopyPass -> MirrorJob
cpJob CopyPass
context
verifyExistingTarget :: CopyPass -> Readmitted -> ExpectedMirrorFile -> NonEmpty RegistryUrl -> [Artifact] -> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
verifyExistingTarget :: CopyPass
-> Readmitted
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> [Artifact]
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
verifyExistingTarget CopyPass
context Readmitted
admitted ExpectedMirrorFile
file NonEmpty RegistryUrl
locations [Artifact]
matches ProjectMirrorRecord
current = do
(bytes, retained) <- WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> ProjectMirrorRecord
-> ExceptT
JobOutcome WorkerM (VerifiedArtifactBytes, ProjectMirrorRecord)
fetchVerified (CopyPass -> WorkerStateBudget
cpCharge CopyPass
context) (CopyPass -> WorkerPolicy
cpPolicy CopyPass
context) (CopyPass -> MirrorJob
cpJob CopyPass
context) ExpectedMirrorFile
file NonEmpty RegistryUrl
locations ProjectMirrorRecord
current
for_ (mpMirrorCompanion (wpPublish (cpPolicy context))) $ \MirrorCompanion
_ ->
ExceptT JobOutcome WorkerM PreparedMirrorUpload
-> ExceptT JobOutcome WorkerM ()
forall (f :: * -> *) a. Functor f => f a -> f ()
void (MirrorJob
-> WorkerM (Either PublishFault PreparedMirrorUpload)
-> ExceptT JobOutcome WorkerM PreparedMirrorUpload
forall a.
MirrorJob
-> WorkerM (Either PublishFault a) -> ExceptT JobOutcome WorkerM a
publishAction (CopyPass -> MirrorJob
cpJob CopyPass
context) (IO (Either PublishFault PreparedMirrorUpload)
-> WorkerM (Either PublishFault PreparedMirrorUpload)
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (PreparedMirrorVersion
-> StateCharge
-> ExpectedMirrorFile
-> VerifiedArtifactBytes
-> IO (Either PublishFault PreparedMirrorUpload)
pmvPrepareUpload (Readmitted -> PreparedMirrorVersion
raPrepared Readmitted
admitted) (WorkerStateBudget -> StateCharge
wsbArchive (CopyPass -> WorkerStateBudget
cpCharge CopyPass
context)) ExpectedMirrorFile
file VerifiedArtifactBytes
bytes)))
let proof = Hash -> Maybe Hash
forall a. a -> Maybe a
Just (VerifiedArtifactBytes -> Hash
verifiedTargetHash VerifiedArtifactBytes
bytes)
when (any (targetIdentityConflict file proof) matches) $
throwError (Dropped ("mirror target identity conflicts: " <> name))
unless (all (expectedTargetMatchesWith file proof) matches) $
throwError (Retried BeforePublish ("mirror file lacks compatible current identity: " <> name))
pure retained
where
name :: Text
name = Artifact -> Text
artFilename (ExpectedMirrorFile -> Artifact
expectedArtifact ExpectedMirrorFile
file)
retainPending :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> MirrorInventory -> ExpectedMirrorFile -> NonEmpty RegistryUrl -> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
retainPending :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> MirrorInventory
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
retainPending WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job MirrorInventory
inventory ExpectedMirrorFile
file NonEmpty RegistryUrl
locations ProjectMirrorRecord
record = do
version <- MirrorJob
-> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM MirrorVersion
requiredVersion MirrorJob
job ProjectMirrorRecord
record
case Map.lookup (artFilename (expectedArtifact file)) (versionCompanions version) of
Just CompanionState
CompanionPending -> case WorkerPolicy -> Maybe WorkerMirrorState
wpMirrorState WorkerPolicy
policy of
Maybe WorkerMirrorState
Nothing -> JobOutcome -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
forall a. JobOutcome -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. MonadError e m => e -> m a
throwError (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
BeforePublish Text
"declared companion requires durable mirror state")
Just WorkerMirrorState
binding -> do
let native :: Maybe MirrorCompanion
native = MirrorPublish -> Maybe MirrorCompanion
mpMirrorCompanion (WorkerPolicy -> MirrorPublish
wpPublish WorkerPolicy
policy)
candidates <- NonEmpty RegistryUrl
-> (RegistryUrl -> ExceptT JobOutcome WorkerM RegistryUrl)
-> ExceptT JobOutcome WorkerM (NonEmpty RegistryUrl)
forall (t :: * -> *) (f :: * -> *) a b.
(Traversable t, Applicative f) =>
t a -> (a -> f b) -> f (t b)
for NonEmpty RegistryUrl
locations ((RegistryUrl -> ExceptT JobOutcome WorkerM RegistryUrl)
-> ExceptT JobOutcome WorkerM (NonEmpty RegistryUrl))
-> (RegistryUrl -> ExceptT JobOutcome WorkerM RegistryUrl)
-> ExceptT JobOutcome WorkerM (NonEmpty RegistryUrl)
forall a b. (a -> b) -> a -> b
$ \RegistryUrl
location -> do
url <- case Maybe MirrorCompanion
native of
Maybe MirrorCompanion
Nothing -> Text -> ExceptT JobOutcome WorkerM Text
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> Text
sidecarUrl (RegistryUrl -> Text
registryUrlText RegistryUrl
location))
Just MirrorCompanion
companion -> MirrorJob
-> WorkerM (Either PublishFault Text)
-> ExceptT JobOutcome WorkerM Text
forall a.
MirrorJob
-> WorkerM (Either PublishFault a) -> ExceptT JobOutcome WorkerM a
publishAction MirrorJob
job (Either PublishFault Text -> WorkerM (Either PublishFault Text)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (MirrorCompanion
-> ExpectedMirrorFile -> Text -> Either PublishFault Text
mcLocation MirrorCompanion
companion ExpectedMirrorFile
file (RegistryUrl -> Text
registryUrlText RegistryUrl
location)))
either throwError pure (sourceLocation policy (expectedArtifact file){artUrl = url})
let limits = (WorkerMirrorState -> Limits
wmsLimits WorkerMirrorState
binding){maxMirrorArtifactBytes = maxMetadataBytes (wmsLimits binding)}
bytes <- fetchLocations (wsbCompanions charge) limits policy candidates
deadlines <- asks (ecDeadlines . wrClient)
checked <- case native of
Maybe MirrorCompanion
Nothing -> ByteString -> ExceptT JobOutcome WorkerM ByteString
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ByteString
bytes
Just MirrorCompanion
companion -> MirrorJob
-> WorkerM (Either PublishFault ByteString)
-> ExceptT JobOutcome WorkerM ByteString
forall a.
MirrorJob
-> WorkerM (Either PublishFault a) -> ExceptT JobOutcome WorkerM a
publishAction MirrorJob
job (IO (Either PublishFault ByteString)
-> WorkerM (Either PublishFault ByteString)
forall a. IO a -> WorkerM a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (MirrorCompanion
-> Deadlines
-> Limits
-> StateCharge
-> MirrorInventory
-> ExpectedMirrorFile
-> ByteString
-> IO (Either PublishFault ByteString)
mcValidate MirrorCompanion
companion Deadlines
deadlines Limits
limits (WorkerStateBudget -> StateCharge
wsbCompanions WorkerStateBudget
charge) MirrorInventory
inventory ExpectedMirrorFile
file ByteString
bytes))
verified <- either (throwError . stateOutcome) pure (verifySourceCompanionBytes (wmsLimits binding) (msfCompanions (wpMirrorFormat policy)) file checked)
stateAction (retainCompanion charge binding (jobVersion job) verified record)
Maybe CompanionState
_ -> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ProjectMirrorRecord
record
mirrorArtifact :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> PublishPlan -> Readmitted -> ExpectedMirrorFile -> NonEmpty RegistryUrl -> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
mirrorArtifact :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> PublishPlan
-> Readmitted
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
mirrorArtifact WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job PublishPlan
plan Readmitted
admitted ExpectedMirrorFile
file NonEmpty RegistryUrl
locations ProjectMirrorRecord
record = do
(verified, retained) <- WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> ProjectMirrorRecord
-> ExceptT
JobOutcome WorkerM (VerifiedArtifactBytes, ProjectMirrorRecord)
fetchVerified WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job ExpectedMirrorFile
file NonEmpty RegistryUrl
locations ProjectMirrorRecord
record
ready <- publishAction job (liftIO (pmvPrepareUpload (raPrepared admitted) (wsbArchive charge) file verified))
outcome <- lift (katipAddContext (mirrorAdmissionPayload job (raDecision admitted) file (verifiedTargetHash verified)) (publishVerified job plan ready))
case outcome of
JobOutcome
Succeeded -> ProjectMirrorRecord
-> ExceptT JobOutcome WorkerM ProjectMirrorRecord
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ProjectMirrorRecord
retained
JobOutcome
failure -> JobOutcome -> ExceptT JobOutcome WorkerM ProjectMirrorRecord
forall a. JobOutcome -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. MonadError e m => e -> m a
throwError JobOutcome
failure
fetchVerified :: WorkerStateBudget -> WorkerPolicy -> MirrorJob -> ExpectedMirrorFile -> NonEmpty RegistryUrl -> ProjectMirrorRecord -> ExceptT JobOutcome WorkerM (VerifiedArtifactBytes, ProjectMirrorRecord)
fetchVerified :: WorkerStateBudget
-> WorkerPolicy
-> MirrorJob
-> ExpectedMirrorFile
-> NonEmpty RegistryUrl
-> ProjectMirrorRecord
-> ExceptT
JobOutcome WorkerM (VerifiedArtifactBytes, ProjectMirrorRecord)
fetchVerified WorkerStateBudget
charge WorkerPolicy
policy MirrorJob
job ExpectedMirrorFile
file NonEmpty RegistryUrl
locations ProjectMirrorRecord
record = do
bytes <- StateCharge
-> Limits
-> WorkerPolicy
-> NonEmpty RegistryUrl
-> ExceptT JobOutcome WorkerM ByteString
fetchLocations (WorkerStateBudget -> StateCharge
wsbArtifacts WorkerStateBudget
charge) (WorkerPolicy -> Limits
wpArtifactLimits WorkerPolicy
policy) WorkerPolicy
policy NonEmpty RegistryUrl
locations
verified <- publishAction job (pure (verifyMirrorBytes file bytes))
retained <- stateAction (persistVerified charge (wpMirrorState policy) (jobVersion job) verified record)
pure (verified, retained)
fetchLocations :: StateCharge -> Limits -> WorkerPolicy -> NonEmpty RegistryUrl -> ExceptT JobOutcome WorkerM ByteString
fetchLocations :: StateCharge
-> Limits
-> WorkerPolicy
-> NonEmpty RegistryUrl
-> ExceptT JobOutcome WorkerM ByteString
fetchLocations StateCharge
charge Limits
limits WorkerPolicy
policy (RegistryUrl
location :| [RegistryUrl]
alternatives) = do
result <- WorkerM (Either FetchFault ByteString)
-> ExceptT JobOutcome WorkerM (Either FetchFault ByteString)
forall (m :: * -> *) a. Monad m => m a -> ExceptT JobOutcome m a
forall (t :: (* -> *) -> * -> *) (m :: * -> *) a.
(MonadTrans t, Monad m) =>
m a -> t m a
lift (StateCharge
-> Limits
-> (Maybe ClientCredential
-> Text -> Either UrlFormationError Request)
-> RegistryUrl
-> WorkerM (Either FetchFault ByteString)
fetchArtifactBytesCharged StateCharge
charge Limits
limits (AdapterArtifact
-> Maybe ClientCredential
-> Text
-> Either UrlFormationError Request
artifactByUrl (WorkerPolicy -> AdapterArtifact
wpArtifact WorkerPolicy
policy)) RegistryUrl
location)
case (result, nonEmpty alternatives) of
(Right ByteString
bytes, Maybe (NonEmpty RegistryUrl)
_) -> ByteString -> ExceptT JobOutcome WorkerM ByteString
forall a. a -> ExceptT JobOutcome WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ByteString
bytes
(Left (FetchTransport TransportFault
_), Just NonEmpty RegistryUrl
remaining) -> StateCharge
-> Limits
-> WorkerPolicy
-> NonEmpty RegistryUrl
-> ExceptT JobOutcome WorkerM ByteString
fetchLocations StateCharge
charge Limits
limits WorkerPolicy
policy NonEmpty RegistryUrl
remaining
(Left FetchFault
fault, Maybe (NonEmpty RegistryUrl)
_) -> JobOutcome -> ExceptT JobOutcome WorkerM ByteString
forall a. JobOutcome -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. MonadError e m => e -> m a
throwError (RetryLeg -> (FetchFault -> Text) -> FetchFault -> JobOutcome
outcomeOfFetchFault RetryLeg
BeforePublish (RegistryUrl -> FetchFault -> Text
artifactFetchReason RegistryUrl
location) FetchFault
fault)
publishAction :: MirrorJob -> WorkerM (Either PublishFault a) -> ExceptT JobOutcome WorkerM a
publishAction :: forall a.
MirrorJob
-> WorkerM (Either PublishFault a) -> ExceptT JobOutcome WorkerM a
publishAction MirrorJob
job WorkerM (Either PublishFault a)
action =
WorkerM (Either JobOutcome a) -> ExceptT JobOutcome WorkerM a
forall e (m :: * -> *) a. m (Either e a) -> ExceptT e m a
ExceptT (WorkerM (Either JobOutcome a) -> ExceptT JobOutcome WorkerM a)
-> WorkerM (Either JobOutcome a) -> ExceptT JobOutcome WorkerM a
forall a b. (a -> b) -> a -> b
$
WorkerM (Either PublishFault a)
action WorkerM (Either PublishFault a)
-> (Either PublishFault a -> WorkerM (Either JobOutcome a))
-> WorkerM (Either JobOutcome a)
forall a b. WorkerM a -> (a -> WorkerM b) -> WorkerM b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= \case
Right a
value -> Either JobOutcome a -> WorkerM (Either JobOutcome a)
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (a -> Either JobOutcome a
forall a b. b -> Either a b
Right a
value)
Left PublishFault
fault -> JobOutcome -> Either JobOutcome a
forall a b. a -> Either a b
Left (JobOutcome -> Either JobOutcome a)
-> WorkerM JobOutcome -> WorkerM (Either JobOutcome a)
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> MirrorJob -> Either PublishFault () -> WorkerM JobOutcome
outcomeOfPublish MirrorJob
job (PublishFault -> Either PublishFault ()
forall a b. a -> Either a b
Left PublishFault
fault)
artifactFetchReason :: RegistryUrl -> FetchFault -> Text
artifactFetchReason :: RegistryUrl -> FetchFault -> Text
artifactFetchReason RegistryUrl
location = \case
FetchUrlUnformable UrlFormationError
urlErr -> Text
"unformable artifact URL: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> UrlFormationError -> Text
renderUrlFormationError UrlFormationError
urlErr
FetchBoundExceeded LimitError
limitErr -> Text
"artifact exceeded the response bound: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> LimitError -> Text
forall b a. (Show a, IsString b) => a -> b
show LimitError
limitErr
FetchCooldown RetryAfter
_ -> Text
"the public upstream is cooling down"
FetchTransport TransportFault
fault -> Text
"artifact fetch from " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text -> Text
authorityLabel (RegistryUrl -> Text
registryUrlText RegistryUrl
location) Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" failed: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> TransportCause -> Text
forall b a. (Show a, IsString b) => a -> b
show (TransportFault -> TransportCause
tfCause TransportFault
fault)
mirrorAdmissionPayload :: MirrorJob -> Decision -> ExpectedMirrorFile -> Hash -> SimpleLogPayload
mirrorAdmissionPayload :: MirrorJob
-> Decision -> ExpectedMirrorFile -> Hash -> SimpleLogPayload
mirrorAdmissionPayload MirrorJob
job Decision
decision ExpectedMirrorFile
file Hash
digest =
Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"event" (Text
"mirror_publication" :: Text)
SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"package" (PackageName -> Text
renderPackageName (MirrorJob -> PackageName
jobPackage MirrorJob
job))
SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"version" (Version -> Text
renderVersion (MirrorJob -> Version
jobVersion MirrorJob
job))
SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"ecosystem" (Ecosystem -> Text
ecosystemName (PackageName -> Ecosystem
pkgEcosystem (MirrorJob -> PackageName
jobPackage MirrorJob
job)))
SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"target" (Text
"mirrorTarget" :: Text)
SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"artifact" (Artifact -> Text
artFilename Artifact
artifact)
SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Value -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"digest" ([Pair] -> Value
object [Key
"algorithm" Key -> Text -> Pair
forall v. ToJSON v => Key -> v -> Pair
forall e kv v. (KeyValue e kv, ToJSON v) => Key -> v -> kv
.= HashAlg -> Text
renderHashAlg (Hash -> HashAlg
hashAlg Hash
digest), Key
"value" Key -> Text -> Pair
forall v. ToJSON v => Key -> v -> Pair
forall e kv v. (KeyValue e kv, ToJSON v) => Key -> v -> kv
.= Hash -> Text
hashValue Hash
digest])
SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"admission" (RuleEvidence -> Decision -> Text
renderDecision (PackageName -> Version -> RuleEvidence
identityEvidence (MirrorJob -> PackageName
jobPackage MirrorJob
job) (MirrorJob -> Version
jobVersion MirrorJob
job)) Decision
decision)
SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> SimpleLogPayload
ruleField
where
artifact :: Artifact
artifact = ExpectedMirrorFile -> Artifact
expectedArtifact ExpectedMirrorFile
file
ruleField :: SimpleLogPayload
ruleField = case Decision
decision of
Admitted Text
name Reason
_ [SkippedCheck]
_ -> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"rule" Text
name
Blocked Text
name Maybe DbEtag
_ Reason
_ -> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"rule" Text
name
BlockedByDefault [Reason]
_ -> Text -> Text -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"rule" (Text
"BlockedByDefault" :: Text)
Undecidable Transience
_ Reason
_ -> SimpleLogPayload
forall a. Monoid a => a
mempty
publishVerified :: MirrorJob -> PublishPlan -> PreparedMirrorUpload -> WorkerM JobOutcome
publishVerified :: MirrorJob
-> PublishPlan -> PreparedMirrorUpload -> WorkerM JobOutcome
publishVerified MirrorJob
job PublishPlan
plan PreparedMirrorUpload
prepared = do
metrics <- (WorkerRuntime -> WorkerMetricsPort) -> WorkerM WorkerMetricsPort
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks WorkerRuntime -> WorkerMetricsPort
wrMetrics
(result, seconds) <- timedSeconds (liftIO (publishPrepared prepared plan))
liftIO (wmpMirrorPublishDuration metrics seconds)
outcomeOfPublish job result
outcomeOfPublish :: MirrorJob -> Either PublishFault () -> WorkerM JobOutcome
outcomeOfPublish :: MirrorJob -> Either PublishFault () -> WorkerM JobOutcome
outcomeOfPublish MirrorJob
job = \case
Right () -> JobOutcome
Succeeded JobOutcome -> WorkerM () -> WorkerM JobOutcome
forall a b. a -> WorkerM b -> WorkerM a
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
InfoS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"mirrored artifact published: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> MirrorJob -> Text
renderJob MirrorJob
job))
Left (PublishIntegrityMismatch Text
detail) -> do
Severity -> LogStr -> WorkerM ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"artifact integrity mismatch, refusing to publish: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
detail))
JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
Dropped (Text
"integrity mismatch: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
detail))
Left (PublishRejected PublishError
err) -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (RetryLeg -> Text -> JobOutcome
Retried RetryLeg
AfterPublish (Text
"registry rejected publish: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> PublishError -> Text
forall b a. (Show a, IsString b) => a -> b
show PublishError
err))
Left (PublishSourceUnavailable Text
detail) -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> JobOutcome
SourceUnavailable (MirrorJob -> Text -> Text
sourceUnavailableReason MirrorJob
job Text
detail))
Left (PublishFetch FetchFault
fault) -> JobOutcome -> WorkerM JobOutcome
forall a. a -> WorkerM a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (RetryLeg -> (FetchFault -> Text) -> FetchFault -> JobOutcome
outcomeOfFetchFault RetryLeg
AfterPublish FetchFault -> Text
publishFaultReason FetchFault
fault)
publishFaultReason :: FetchFault -> Text
publishFaultReason :: FetchFault -> Text
publishFaultReason = \case
FetchUrlUnformable UrlFormationError
urlErr -> Text
"unformable publish URL: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> UrlFormationError -> Text
renderUrlFormationError UrlFormationError
urlErr
FetchBoundExceeded LimitError
limitErr -> Text
"the publication target's response exceeded the response bound: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> LimitError -> Text
forall b a. (Show a, IsString b) => a -> b
show LimitError
limitErr
FetchCooldown RetryAfter
_ -> Text
"the public upstream is cooling down"
FetchTransport TransportFault
fault -> Text
"publish transport failure: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> TransportFault -> Text
forall b a. (Show a, IsString b) => a -> b
show TransportFault
fault