module Ecluse.Composition.Worker (
workerPoliciesWithState,
mirrorTransportFor,
) where
import Data.Map.Strict qualified as Map
import Ecluse.Composition (PublishTarget (ptCredentials, ptEcosystem, ptMirrorUrl, ptPresentation, ptUploadUrl, ptUsername))
import Ecluse.Composition.MirrorState (MirrorStateBinding (msbStore, msbTarget))
import Ecluse.Core.Credential (ClientCredential (ClientCredential), mintSecret)
import Ecluse.Core.Ecosystem (Ecosystem, parseEcosystem)
import Ecluse.Core.Registry.Adapter (adapterFor, adapterPublish)
import Ecluse.Core.Registry.Adapter.Capability (AdapterMetadata (metadataChargeFactors, metadataRead), AdapterPublish (publishCodec, publishMirrorRequirements), MirrorRequirements (mirrorStateFormat, mirrorWorkerRead), WorkerMetadataRead (..))
import Ecluse.Core.Registry.Metadata (fetchVersionDetails)
import Ecluse.Core.Registry.Metadata.Fetch (chargingSelectedReads)
import Ecluse.Core.Registry.Metadata.Fetch.Preparation (PrepareRead (..), prepareRead)
import Ecluse.Core.Registry.Origin (anonymousOrigin, chargingNativeReads, controlPublicOrigin)
import Ecluse.Core.Registry.Publish (
MirrorPublish,
MirrorTransport (MirrorTransport, ptClient, ptLimits, ptMintCredential, ptTracing),
newMirrorPublish,
)
import Ecluse.Core.Security (Limits (maxMirrorArtifactBytes), Origin (UntrustedOrigin), thgPublicHostPort)
import Ecluse.Core.Security.Egress (registryUrlText)
import Ecluse.Core.Server.Admission.Meter (charge, newChargeRegion, withMemoryEntry)
import Ecluse.Core.Server.Cache (Source (Source))
import Ecluse.Core.Server.Context (
PackumentDeps,
pdArtifact,
pdEgressUrl,
pdFirstParty,
pdLimits,
pdMetadata,
pdMinIntegrity,
pdNow,
pdPublicBase,
pdPublicBaseUrl,
pdRules,
pdTarballHostGate,
tarballHostHonoured,
)
import Ecluse.Core.Server.MemoryModel (decodedDocumentCharge)
import Ecluse.Core.Server.Metadata (ecosystemMetadataReads, publicMetadataClient, uncachedPublicMetadataClient)
import Ecluse.Core.Telemetry.Upstream (PublicReadKind (WorkerMetadata))
import Ecluse.Core.Worker (WorkerPolicies, WorkerPolicy (..))
import Ecluse.Core.Worker.Types (WorkerMirrorState (..), WorkerStateBudget (..))
import Ecluse.Runtime.Env (Env, envMemoryMeter, envMetadataCache, envMetrics, envPrivateClient, envPublicClient, envPublicControl, envTelemetry)
import Ecluse.Runtime.Server (MountBinding (bindingPackumentDeps, bindingPrefix))
import Ecluse.Runtime.Telemetry.Instruments (metricsPortOf)
import Ecluse.Runtime.Telemetry.Tracing (tracingPortOf)
workerPoliciesWithState :: Env -> Map Ecosystem MirrorStateBinding -> [MountBinding] -> [PublishTarget] -> Int -> WorkerPolicies
workerPoliciesWithState :: Env
-> Map Ecosystem MirrorStateBinding
-> [MountBinding]
-> [PublishTarget]
-> Int
-> WorkerPolicies
workerPoliciesWithState Env
env Map Ecosystem MirrorStateBinding
states [MountBinding]
bindings [PublishTarget]
targets Int
artifactMaxBytes =
[(Ecosystem, WorkerPolicy)] -> WorkerPolicies
forall k a. Ord k => [(k, a)] -> Map k a
Map.fromList
[ (Ecosystem
eco, (Env
-> MirrorRequirements
-> PackumentDeps
-> MirrorPublish
-> Int
-> WorkerPolicy
workerPolicyFor Env
env MirrorRequirements
requirements PackumentDeps
deps MirrorPublish
publish Int
artifactMaxBytes){wpMirrorState = managedState eco deps})
| MountBinding
binding <- [MountBinding]
bindings
, let Text
prefixHead :| [Text]
_ = MountBinding -> NonEmpty Text
bindingPrefix MountBinding
binding
, let deps :: PackumentDeps
deps = MountBinding -> PackumentDeps
bindingPackumentDeps MountBinding
binding
, Just Ecosystem
eco <- [Text -> Maybe Ecosystem
parseEcosystem Text
prefixHead]
, Just (MirrorPublish
publish, MirrorRequirements
requirements) <- [Env
-> PackumentDeps
-> Map Ecosystem PublishTarget
-> Int
-> Ecosystem
-> Maybe (MirrorPublish, MirrorRequirements)
mirrorPublishFor Env
env PackumentDeps
deps Map Ecosystem PublishTarget
targetsByEcosystem Int
artifactMaxBytes Ecosystem
eco]
]
where
managedState :: Ecosystem -> PackumentDeps -> Maybe WorkerMirrorState
managedState Ecosystem
eco PackumentDeps
deps = do
binding <- Ecosystem
-> Map Ecosystem MirrorStateBinding -> Maybe MirrorStateBinding
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Ecosystem
eco Map Ecosystem MirrorStateBinding
states
let limits = (PackumentDeps -> Limits
pdLimits PackumentDeps
deps){maxMirrorArtifactBytes = artifactMaxBytes}
pure
WorkerMirrorState
{ wmsStore = msbStore binding
, wmsTarget = msbTarget binding
, wmsLimits = limits
, wmsWithinBudget = \WorkerStateBudget -> IO a
action ->
(Either Shed a -> Maybe a) -> IO (Either Shed a) -> IO (Maybe a)
forall a b. (a -> b) -> IO a -> IO b
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
fmap Either Shed a -> Maybe a
forall l r. Either l r -> Maybe r
rightToMaybe (IO (Either Shed a) -> IO (Maybe a))
-> ((MemoryTicket -> IO a) -> IO (Either Shed a))
-> (MemoryTicket -> IO a)
-> IO (Maybe a)
forall b c a. (b -> c) -> (a -> b) -> a -> c
. MetricsPort
-> MemoryMeter -> (MemoryTicket -> IO a) -> IO (Either Shed a)
forall (m :: * -> *) a.
MonadUnliftIO m =>
MetricsPort
-> MemoryMeter -> (MemoryTicket -> m a) -> m (Either Shed a)
withMemoryEntry (Metrics -> MetricsPort
metricsPortOf (Env -> Metrics
envMetrics Env
env)) (Env -> MemoryMeter
envMemoryMeter Env
env) ((MemoryTicket -> IO a) -> IO (Maybe a))
-> (MemoryTicket -> IO a) -> IO (Maybe a)
forall a b. (a -> b) -> a -> b
$ \MemoryTicket
ticket -> do
stateCharge <- (Int -> IO ()) -> IO (Int -> IO ())
newChargeRegion (MemoryTicket -> Int -> IO ()
charge MemoryTicket
ticket)
recordCharge <- newChargeRegion (charge ticket)
companionCharge <- newChargeRegion (charge ticket)
artifactCharge <- newChargeRegion (charge ticket)
archiveCharge <- newChargeRegion (charge ticket)
sourceCharge <- newChargeRegion (charge ticket)
targetCharge <- newChargeRegion (charge ticket)
action WorkerStateBudget{wsbState = stateCharge, wsbRecords = recordCharge, wsbCompanions = companionCharge, wsbArtifacts = artifactCharge, wsbArchive = archiveCharge, wsbSource = sourceCharge, wsbTargets = targetCharge}
}
targetsByEcosystem :: Map Ecosystem PublishTarget
targetsByEcosystem = [(Ecosystem, PublishTarget)] -> Map Ecosystem PublishTarget
forall k a. Ord k => [(k, a)] -> Map k a
Map.fromList [(PublishTarget -> Ecosystem
ptEcosystem PublishTarget
target, PublishTarget
target) | PublishTarget
target <- [PublishTarget]
targets]
mirrorPublishFor :: Env -> PackumentDeps -> Map.Map Ecosystem PublishTarget -> Int -> Ecosystem -> Maybe (MirrorPublish, MirrorRequirements)
mirrorPublishFor :: Env
-> PackumentDeps
-> Map Ecosystem PublishTarget
-> Int
-> Ecosystem
-> Maybe (MirrorPublish, MirrorRequirements)
mirrorPublishFor Env
env PackumentDeps
deps Map Ecosystem PublishTarget
targets Int
artifactMaxBytes Ecosystem
eco = do
target <- Ecosystem -> Map Ecosystem PublishTarget -> Maybe PublishTarget
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Ecosystem
eco Map Ecosystem PublishTarget
targets
adapter <- adapterFor eco
publish <- adapterPublish adapter
let transport = Env -> PackumentDeps -> PublishTarget -> MirrorTransport
mirrorTransportFor Env
env PackumentDeps
deps PublishTarget
target
bounded = MirrorTransport
transport{ptLimits = (ptLimits transport){maxMirrorArtifactBytes = artifactMaxBytes}}
pure (newMirrorPublish bounded (ptMirrorUrl target) (ptUploadUrl target) (publishCodec publish (ptPresentation target)), publishMirrorRequirements publish)
mirrorTransportFor :: Env -> PackumentDeps -> PublishTarget -> MirrorTransport
mirrorTransportFor :: Env -> PackumentDeps -> PublishTarget -> MirrorTransport
mirrorTransportFor Env
env PackumentDeps
deps PublishTarget
target =
MirrorTransport
{ ptClient :: ExchangeClient
ptClient = Env -> ExchangeClient
envPrivateClient Env
env
, ptMintCredential :: IO (Maybe ClientCredential)
ptMintCredential = ClientCredential -> Maybe ClientCredential
forall a. a -> Maybe a
Just (ClientCredential -> Maybe ClientCredential)
-> (Secret -> ClientCredential) -> Secret -> Maybe ClientCredential
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Maybe Text -> Secret -> ClientCredential
ClientCredential (PublishTarget -> Maybe Text
ptUsername PublishTarget
target) (Secret -> Maybe ClientCredential)
-> IO Secret -> IO (Maybe ClientCredential)
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> CredentialProvider -> IO Secret
mintSecret (PublishTarget -> CredentialProvider
ptCredentials PublishTarget
target)
, ptLimits :: Limits
ptLimits = PackumentDeps -> Limits
pdLimits PackumentDeps
deps
, ptTracing :: TracingPort
ptTracing = Telemetry -> TracingPort
tracingPortOf (Env -> Telemetry
envTelemetry Env
env)
}
workerPolicyFor :: Env -> MirrorRequirements -> PackumentDeps -> MirrorPublish -> Int -> WorkerPolicy
workerPolicyFor :: Env
-> MirrorRequirements
-> PackumentDeps
-> MirrorPublish
-> Int
-> WorkerPolicy
workerPolicyFor Env
env MirrorRequirements
requirements PackumentDeps
deps MirrorPublish
publish Int
artifactMaxBytes =
WorkerPolicy
{ wpFirstParty :: PackageName -> Bool
wpFirstParty = PackumentDeps -> PackageName -> Bool
pdFirstParty PackumentDeps
deps
, wpResolveVersion :: (Int -> IO ()) -> PackageName -> Version -> IO VersionEvaluation
wpResolveVersion = \Int -> IO ()
pay PackageName
name Version
version -> (Int -> IO ()) -> IO MetadataClient
client Int -> IO ()
pay IO MetadataClient
-> (MetadataClient -> IO VersionEvaluation) -> IO VersionEvaluation
forall a b. IO a -> (a -> IO b) -> IO b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= \MetadataClient
metadata -> MetadataClient -> PackageName -> Version -> IO VersionEvaluation
fetchVersionDetails MetadataClient
metadata PackageName
name Version
version
, wpRules :: [PreparedRule]
wpRules = PackumentDeps -> [PreparedRule]
pdRules PackumentDeps
deps
, wpMinIntegrity :: MinIntegrity
wpMinIntegrity = PackumentDeps -> MinIntegrity
pdMinIntegrity PackumentDeps
deps
, wpArtifactHostHonoured :: Maybe HostPort -> Bool
wpArtifactHostHonoured =
Origin -> PackumentDeps -> Maybe HostPort -> Maybe HostPort -> Bool
tarballHostHonoured Origin
UntrustedOrigin PackumentDeps
deps (TarballHostGate -> Maybe HostPort
thgPublicHostPort (PackumentDeps -> TarballHostGate
pdTarballHostGate PackumentDeps
deps))
, wpArtifact :: AdapterArtifact
wpArtifact = PackumentDeps -> AdapterArtifact
pdArtifact PackumentDeps
deps
, wpEgressUrl :: Text -> Either Text RegistryUrl
wpEgressUrl = PackumentDeps -> Text -> Either Text RegistryUrl
pdEgressUrl PackumentDeps
deps
, wpPublish :: MirrorPublish
wpPublish = MirrorPublish
publish
, wpMirrorFormat :: MirrorStateFormat
wpMirrorFormat = MirrorRequirements -> MirrorStateFormat
mirrorStateFormat MirrorRequirements
requirements
, wpArtifactLimits :: Limits
wpArtifactLimits = (PackumentDeps -> Limits
pdLimits PackumentDeps
deps){maxMirrorArtifactBytes = artifactMaxBytes}
, wpMirrorState :: Maybe WorkerMirrorState
wpMirrorState = Maybe WorkerMirrorState
forall a. Maybe a
Nothing
, wpNow :: IO UTCTime
wpNow = PackumentDeps -> IO UTCTime
pdNow PackumentDeps
deps
}
where
client :: (Int -> IO ()) -> IO MetadataClient
client Int -> IO ()
pay = case MirrorRequirements -> WorkerMetadataRead
mirrorWorkerRead MirrorRequirements
requirements of
ChargedWorkerRead PrepareRead
nativeRead ->
MetadataClient -> IO MetadataClient
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (MetadataClient -> IO MetadataClient)
-> (PrepareRead -> MetadataClient)
-> PrepareRead
-> IO MetadataClient
forall b c a. (b -> c) -> (a -> b) -> a -> c
. MetadataReads Public -> MetadataClient
uncachedPublicMetadataClient (MetadataReads Public -> MetadataClient)
-> (PrepareRead -> MetadataReads Public)
-> PrepareRead
-> MetadataClient
forall b c a. (b -> c) -> (a -> b) -> a -> c
. OriginFor Public -> PrepareRead -> MetadataReads Public
forall {posture}.
OriginFor posture -> PrepareRead -> MetadataReads posture
readCapability OriginFor Public
publicOrigin (PrepareRead -> IO MetadataClient)
-> PrepareRead -> IO MetadataClient
forall a b. (a -> b) -> a -> b
$
(OriginClient -> IO (Either MetadataError EcosystemRead))
-> PrepareRead
PrepareRead ((Either MetadataError EcosystemRead
-> Either MetadataError EcosystemRead)
-> IO (Either MetadataError EcosystemRead)
-> IO (Either MetadataError EcosystemRead)
forall a b. (a -> b) -> IO a -> IO b
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
fmap ((EcosystemRead -> EcosystemRead)
-> Either MetadataError EcosystemRead
-> Either MetadataError EcosystemRead
forall a b.
(a -> b) -> Either MetadataError a -> Either MetadataError b
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
fmap ((ByteString -> Int)
-> (Int -> IO ()) -> EcosystemRead -> EcosystemRead
chargingSelectedReads ByteString -> Int
decodedDocumentCharge Int -> IO ()
pay)) (IO (Either MetadataError EcosystemRead)
-> IO (Either MetadataError EcosystemRead))
-> (OriginClient -> IO (Either MetadataError EcosystemRead))
-> OriginClient
-> IO (Either MetadataError EcosystemRead)
forall b c a. (b -> c) -> (a -> b) -> a -> c
. PrepareRead
-> OriginClient -> IO (Either MetadataError EcosystemRead)
prepareRead PrepareRead
nativeRead)
NativeChargedWorkerRead PrepareRead
nativeRead ->
(\OriginFor Public
origin -> MetadataReads Public -> MetadataClient
uncachedPublicMetadataClient (OriginFor Public -> PrepareRead -> MetadataReads Public
forall {posture}.
OriginFor posture -> PrepareRead -> MetadataReads posture
readCapability OriginFor Public
origin PrepareRead
nativeRead))
(OriginFor Public -> MetadataClient)
-> IO (OriginFor Public) -> IO MetadataClient
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> ChargeFactors
-> (Int -> IO ()) -> OriginFor Public -> IO (OriginFor Public)
forall posture.
ChargeFactors
-> (Int -> IO ()) -> OriginFor posture -> IO (OriginFor posture)
chargingNativeReads (AdapterMetadata -> ChargeFactors
metadataChargeFactors (PackumentDeps -> AdapterMetadata
pdMetadata PackumentDeps
deps)) Int -> IO ()
pay OriginFor Public
publicOrigin
WorkerMetadataRead
CachedWorkerRead -> MetadataClient -> IO MetadataClient
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (MetadataCache -> Source -> MetadataReads Public -> MetadataClient
publicMetadataClient (Env -> MetadataCache
envMetadataCache Env
env) (Text -> Source
Source (RegistryUrl -> Text
registryUrlText RegistryUrl
publicBaseUrl)) (OriginFor Public -> PrepareRead -> MetadataReads Public
forall {posture}.
OriginFor posture -> PrepareRead -> MetadataReads posture
readCapability OriginFor Public
publicOrigin (AdapterMetadata -> PrepareRead
metadataRead (PackumentDeps -> AdapterMetadata
pdMetadata PackumentDeps
deps))))
readCapability :: OriginFor posture -> PrepareRead -> MetadataReads posture
readCapability OriginFor posture
origin PrepareRead
ecosystemRead =
PrepareRead
-> TracingPort
-> MetricsPort
-> (PackageName -> MetadataError -> IO ())
-> (PackageName -> PackageInfo -> IO ())
-> (PackageName -> IO ())
-> OriginFor posture
-> MetadataReads posture
forall posture.
PrepareRead
-> TracingPort
-> MetricsPort
-> (PackageName -> MetadataError -> IO ())
-> (PackageName -> PackageInfo -> IO ())
-> (PackageName -> IO ())
-> OriginFor posture
-> MetadataReads posture
ecosystemMetadataReads
PrepareRead
ecosystemRead
(Telemetry -> TracingPort
tracingPortOf (Env -> Telemetry
envTelemetry Env
env))
(Metrics -> MetricsPort
metricsPortOf (Env -> Metrics
envMetrics Env
env))
(\PackageName
_ MetadataError
_ -> () -> IO ()
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ())
(\PackageName
_ PackageInfo
_ -> () -> IO ()
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ())
(\PackageName
_ -> () -> IO ()
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ())
OriginFor posture
origin
publicBaseUrl :: RegistryUrl
publicBaseUrl = PackumentDeps -> RegistryUrl
pdPublicBaseUrl PackumentDeps
deps
publicOrigin :: OriginFor Public
publicOrigin = PublicControl
-> Maybe PublicReadKind -> OriginFor Public -> OriginFor Public
controlPublicOrigin (Env -> PublicControl
envPublicControl Env
env) (PublicReadKind -> Maybe PublicReadKind
forall a. a -> Maybe a
Just PublicReadKind
WorkerMetadata) (Limits -> ExchangeClient -> BaseRequest -> OriginFor Public
anonymousOrigin (PackumentDeps -> Limits
pdLimits PackumentDeps
deps) (Env -> ExchangeClient
envPublicClient Env
env) (PackumentDeps -> BaseRequest
pdPublicBase PackumentDeps
deps))