module Ecluse.Core.Registry.RubyGems.Handlers.CompactIndex (serveDirectory, serveInfo) where
import Crypto.Hash (Digest, SHA256, hash)
import Data.ByteArray.Encoding (Base (Base64), convertToBase)
import Data.ByteString qualified as BS
import Data.ByteString.Lazy qualified as LBS
import Network.HTTP.Client (brRead, responseBody, responseHeaders, responseStatus)
import Network.HTTP.Types (statusCode)
import Network.Wai (Request, ResponseReceived)
import UnliftIO (evaluate, withRunInIO)
import Ecluse.Core.Credential (ClientCredential)
import Ecluse.Core.Fault.RetryAfter (responseRetryAfter)
import Ecluse.Core.Package (PackageName)
import Ecluse.Core.Registry (BodyOutcome (SuccessBody, UnreadStatus), FetchFault (FetchBoundExceeded, FetchUrlUnformable))
import Ecluse.Core.Registry.CachedDocument (rubyGemsCached)
import Ecluse.Core.Registry.Exchange (formThen, runExchangeUsing)
import Ecluse.Core.Registry.Exchange.Types (ExchangeClient)
import Ecluse.Core.Registry.JsonStream (StreamResult (streamValue), readStepsWithEnd)
import Ecluse.Core.Registry.Metadata (MetadataError (..), metadataResponse)
import Ecluse.Core.Registry.Metadata.RecordStream (finishRecordSteps)
import Ecluse.Core.Registry.Public (ReadControl (..))
import Ecluse.Core.Registry.Request (BaseRequest, baseRequestUrl, sealRequest)
import Ecluse.Core.Registry.RubyGems.Directory
import Ecluse.Core.Registry.RubyGems.Document (GemDocument (..), GemRow (..))
import Ecluse.Core.Registry.RubyGems.Handlers.Budget (newNativeDocumentBudget, newNativeReadBudget, withNativeRequest)
import Ecluse.Core.Registry.RubyGems.Request (gemDirectoryRequestUnder)
import Ecluse.Core.Registry.RubyGems.Route.Response (RubyResponse, rubyReplies)
import Ecluse.Core.Registry.RubyGems.Serving (serialiseRubyGemsDocument)
import Ecluse.Core.Security (BodyLimit (MetadataBodyLimit), LimitError (BodyTooLarge), Limits (..))
import Ecluse.Core.Security.Egress (registryUrlText)
import Ecluse.Core.Server.Admission.Meter (ChargePart, MemoryTicket, awaitingFlight, charge, chargePart, newChargePart, newChargeRegion, servingPart, stepDown)
import Ecluse.Core.Server.Cache (resolveDirectory)
import Ecluse.Core.Server.Cache.Types (directoryKey)
import Ecluse.Core.Server.Context
import Ecluse.Core.Server.Metadata (recordedFetch)
import Ecluse.Core.Server.Pipeline.Diagnostics (logDirectoryFailure, logDirectoryInvalidEntries)
import Ecluse.Core.Server.Pipeline.Origin (ReadFailure (ReadInvalid, ReadUnavailable), metadataReadFailure, publicSource)
import Ecluse.Core.Server.Pipeline.Packument (PackumentReplies (..), answerDigestedMetadataBytes, mapPackumentReplies, withObservedAdmittedDocument)
import Ecluse.Core.Server.Pipeline.Shared (flightOf, forwardedCredential, retryAfterHeaders, withListingAdmission)
import Ecluse.Core.Server.Response (Refusal, mkRefusal)
import Ecluse.Core.Telemetry.Metrics qualified as Metric
import Ecluse.Core.Telemetry.Record (MetricsPort (mpServeDecision))
import Ecluse.Core.Telemetry.Span (TracingPort (spanDirectoryDecode, spanDirectoryFetch))
import Ecluse.Core.Telemetry.Upstream (PublicReadKind (Listing))
serveDirectory :: Request -> (RubyResponse -> IO ResponseReceived) -> Handler ResponseReceived
serveDirectory :: Request
-> (RubyResponse -> IO ResponseReceived)
-> Handler ResponseReceived
serveDirectory Request
request RubyResponse -> IO ResponseReceived
respond =
(ServeRuntime
-> (ResponseHeaders -> Handler ResponseReceived)
-> (MemoryTicket -> Handler (Decision, RubyResponse))
-> ((Decision, RubyResponse) -> Handler ResponseReceived)
-> Handler ResponseReceived)
-> Request
-> (RubyResponse -> Handler ResponseReceived)
-> (PackumentDeps -> (Decision, RubyResponse))
-> (MemoryTicket
-> PackumentDeps -> Handler (Decision, RubyResponse))
-> ((Decision, RubyResponse) -> Handler ResponseReceived)
-> Handler ResponseReceived
forall a.
(ServeRuntime
-> (ResponseHeaders -> Handler ResponseReceived)
-> (MemoryTicket -> Handler a)
-> (a -> Handler ResponseReceived)
-> Handler ResponseReceived)
-> Request
-> (RubyResponse -> Handler ResponseReceived)
-> (PackumentDeps -> a)
-> (MemoryTicket -> PackumentDeps -> Handler a)
-> (a -> Handler ResponseReceived)
-> Handler ResponseReceived
withNativeRequest
ServeRuntime
-> (ResponseHeaders -> Handler ResponseReceived)
-> (MemoryTicket -> Handler (Decision, RubyResponse))
-> ((Decision, RubyResponse) -> Handler ResponseReceived)
-> Handler ResponseReceived
forall (m :: * -> *) received response.
MonadUnliftIO m =>
ServeRuntime
-> (ResponseHeaders -> m received)
-> (MemoryTicket -> m response)
-> (response -> m received)
-> m received
withListingAdmission
Request
request
(IO ResponseReceived -> Handler ResponseReceived
forall a. IO a -> Handler a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO ResponseReceived -> Handler ResponseReceived)
-> (RubyResponse -> IO ResponseReceived)
-> RubyResponse
-> Handler ResponseReceived
forall b c a. (b -> c) -> (a -> b) -> a -> c
. RubyResponse -> IO ResponseReceived
respond)
((Decision, RubyResponse)
-> PackumentDeps -> (Decision, RubyResponse)
forall a b. a -> b -> a
const (Decision
Metric.Unavailable, MetadataError -> RubyResponse
directoryFailure (Int -> Maybe RetryAfter -> MetadataError
MetadataHttpFailure Int
503 Maybe RetryAfter
forall a. Maybe a
Nothing)))
(Request
-> MemoryTicket
-> PackumentDeps
-> Handler (Decision, RubyResponse)
answerDirectory Request
request)
( \(Decision
decision, RubyResponse
response) -> do
metrics <- (RequestCtx -> MetricsPort) -> Handler MetricsPort
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks (ServeRuntime -> MetricsPort
srMetrics (ServeRuntime -> MetricsPort)
-> (RequestCtx -> ServeRuntime) -> RequestCtx -> MetricsPort
forall b c a. (b -> c) -> (a -> b) -> a -> c
. RequestCtx -> ServeRuntime
ctxRuntime)
liftIO (mpServeDecision metrics decision *> respond response)
)
answerDirectory :: Request -> MemoryTicket -> PackumentDeps -> Handler (Metric.Decision, RubyResponse)
answerDirectory :: Request
-> MemoryTicket
-> PackumentDeps
-> Handler (Decision, RubyResponse)
answerDirectory Request
request MemoryTicket
ticket PackumentDeps
deps = do
privateRead <- IO ChargePart -> Handler ChargePart
forall a. IO a -> Handler a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (MemoryTicket -> IO ChargePart
newChargePart MemoryTicket
ticket)
publicRead <- liftIO (newChargePart ticket)
private <- readPrivateDirectory request deps privateRead
case private of
Left MetadataError
fault -> (Decision, RubyResponse) -> Handler (Decision, RubyResponse)
forall a. a -> Handler a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (MetadataError -> Decision
privateDecision MetadataError
fault, MetadataError -> RubyResponse
privateDirectoryFailure MetadataError
fault)
Right Maybe GemDirectory
privateView -> do
public <- MemoryTicket
-> PackumentDeps
-> ChargePart
-> Handler (Either MetadataError GemDirectory)
readPublicDirectory MemoryTicket
ticket PackumentDeps
deps ChargePart
publicRead
case (public, privateView) of
(Left MetadataError
fault, Just GemDirectory
own) | Bool -> Bool
not (MetadataError -> Bool
boundBreach MetadataError
fault) Bool -> Bool -> Bool
&& Bool -> Bool
not (ByteString -> Bool
BS.null (GemDirectory -> ByteString
directoryBytes GemDirectory
own)) -> [ChargePart] -> ByteString -> Handler (Decision, RubyResponse)
forall {t :: * -> *}.
Foldable t =>
t ChargePart -> ByteString -> Handler (Decision, RubyResponse)
answerBody [ChargePart
publicRead] (ByteString
"---\n" ByteString -> ByteString -> ByteString
forall a. Semigroup a => a -> a -> a
<> GemDirectory -> ByteString
directoryBytes GemDirectory
own)
(Left MetadataError
fault, Maybe GemDirectory
_) -> (Decision, RubyResponse) -> Handler (Decision, RubyResponse)
forall a. a -> Handler a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Decision
Metric.Unavailable, MetadataError -> RubyResponse
directoryFailure MetadataError
fault)
(Right GemDirectory
shared, Maybe GemDirectory
_) -> do
let privateBytes :: Int
privateBytes = Int -> (GemDirectory -> Int) -> Maybe GemDirectory -> Int
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Int
0 (ByteString -> Int
BS.length (ByteString -> Int)
-> (GemDirectory -> ByteString) -> GemDirectory -> Int
forall b c a. (b -> c) -> (a -> b) -> a -> c
. GemDirectory -> ByteString
directoryBytes) Maybe GemDirectory
privateView
if ByteString -> Int
BS.length (GemDirectory -> ByteString
directoryBytes GemDirectory
shared) Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
privateBytes Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
4 Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Limits -> Int
maxMetadataBytes Limits
limits
then (Decision, RubyResponse) -> Handler (Decision, RubyResponse)
forall a. a -> Handler a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Decision
Metric.Unavailable, MetadataError -> RubyResponse
directoryFailure (LimitError -> MetadataError
MetadataBoundExceeded (BodyLimit -> LimitError
BodyTooLarge (Int -> BodyLimit
MetadataBodyLimit (Limits -> Int
maxMetadataBytes Limits
limits)))))
else do
IO () -> Handler ()
forall a. IO a -> Handler a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (MemoryTicket -> Int -> IO ()
charge MemoryTicket
ticket (Int
2 Int -> Int -> Int
forall a. Num a => a -> a -> a
* (GemDirectory -> Int
directoryWeight GemDirectory
shared Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
privateBytes) Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int -> (GemDirectory -> Int) -> Maybe GemDirectory -> Int
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Int
0 (GemDirectory -> GemDirectory -> Int
directoryMergeWorkspace GemDirectory
shared) Maybe GemDirectory
privateView Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
4096))
let merged :: GemDirectory
merged = GemDirectory
-> (GemDirectory -> GemDirectory)
-> Maybe GemDirectory
-> GemDirectory
forall b a. b -> (a -> b) -> Maybe a -> b
maybe GemDirectory
shared (GemDirectory -> GemDirectory -> GemDirectory
mergeDirectories GemDirectory
shared) Maybe GemDirectory
privateView
[ChargePart] -> ByteString -> Handler (Decision, RubyResponse)
forall {t :: * -> *}.
Foldable t =>
t ChargePart -> ByteString -> Handler (Decision, RubyResponse)
answerBody [ChargePart
privateRead, ChargePart
publicRead] (ByteString
"---\n" ByteString -> ByteString -> ByteString
forall a. Semigroup a => a -> a -> a
<> GemDirectory -> ByteString
directoryBytes GemDirectory
merged)
where
limits :: Limits
limits = PackumentDeps -> Limits
pdLimits PackumentDeps
deps
answerBody :: t ChargePart -> ByteString -> Handler (Decision, RubyResponse)
answerBody t ChargePart
released ByteString
body = do
bytes <- ByteString -> Handler ByteString
forall (m :: * -> *) a. MonadIO m => a -> m a
evaluate ByteString
body
liftIO (traverse_ (`stepDown` 0) released)
(if overMetadataBound deps bytes then Metric.Unavailable else Metric.Admit,) <$> answerCompact deps ticket request bytes
privateDecision :: MetadataError -> Decision
privateDecision = \case
MetadataAuthorisationFailure Int
_ -> Decision
Metric.Deny
MetadataError
_ -> Decision
Metric.Unavailable
readPrivateDirectory :: Request -> PackumentDeps -> ChargePart -> Handler (Either MetadataError (Maybe GemDirectory))
readPrivateDirectory :: Request
-> PackumentDeps
-> ChargePart
-> Handler (Either MetadataError (Maybe GemDirectory))
readPrivateDirectory Request
request PackumentDeps
deps ChargePart
privateRead = case PackumentDeps -> Maybe BaseRequest
pdPrivateBase PackumentDeps
deps of
Maybe BaseRequest
Nothing -> Either MetadataError (Maybe GemDirectory)
-> Handler (Either MetadataError (Maybe GemDirectory))
forall a. a -> Handler a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Maybe GemDirectory -> Either MetadataError (Maybe GemDirectory)
forall a b. b -> Either a b
Right Maybe GemDirectory
forall a. Maybe a
Nothing)
Just BaseRequest
base -> do
runtime <- (RequestCtx -> ServeRuntime) -> Handler ServeRuntime
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks RequestCtx -> ServeRuntime
ctxRuntime
mount <- asks ctxMount
let source = MalformedRows
-> ExchangeClient
-> ReadControl
-> BaseRequest
-> Maybe ClientCredential
-> DirectorySource
DirectorySource MalformedRows
RejectMalformedRows (ServeRuntime -> ExchangeClient
srPrivateClient ServeRuntime
runtime) ReadControl
Uncontrolled BaseRequest
base (MountBinding -> Request -> Maybe ClientCredential
forwardedCredential MountBinding
mount Request
request)
withRunInIO $ \forall a. Handler a -> IO a
runInIO -> (GemDirectory -> Maybe GemDirectory)
-> Either MetadataError GemDirectory
-> Either MetadataError (Maybe GemDirectory)
forall a b.
(a -> b) -> Either MetadataError a -> Either MetadataError b
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
fmap GemDirectory -> Maybe GemDirectory
forall a. a -> Maybe a
Just (Either MetadataError GemDirectory
-> Either MetadataError (Maybe GemDirectory))
-> IO (Either MetadataError GemDirectory)
-> IO (Either MetadataError (Maybe GemDirectory))
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> (Handler () -> IO ())
-> TracingPort
-> (Int -> IO ())
-> Limits
-> DirectorySource
-> IO (Either MetadataError GemDirectory)
fetchDirectory Handler () -> IO ()
forall a. Handler a -> IO a
runInIO (ServeRuntime -> TracingPort
srTracing ServeRuntime
runtime) (ChargePart -> Int -> IO ()
chargePart ChargePart
privateRead) (PackumentDeps -> Limits
pdLimits PackumentDeps
deps) DirectorySource
source
readPublicDirectory :: MemoryTicket -> PackumentDeps -> ChargePart -> Handler (Either MetadataError GemDirectory)
readPublicDirectory :: MemoryTicket
-> PackumentDeps
-> ChargePart
-> Handler (Either MetadataError GemDirectory)
readPublicDirectory MemoryTicket
ticket PackumentDeps
deps ChargePart
publicRead = do
runtime <- (RequestCtx -> ServeRuntime) -> Handler ServeRuntime
forall r (m :: * -> *) a. MonadReader r m => (r -> a) -> m a
asks RequestCtx -> ServeRuntime
ctxRuntime
let source = MalformedRows
-> ExchangeClient
-> ReadControl
-> BaseRequest
-> Maybe ClientCredential
-> DirectorySource
DirectorySource MalformedRows
DropMalformedRows (ServeRuntime -> ExchangeClient
srPublicClient ServeRuntime
runtime) (PublicControl -> PublicReadKind -> ReadControl
Controlled (ServeRuntime -> PublicControl
srPublicControl ServeRuntime
runtime) PublicReadKind
Listing) (PackumentDeps -> BaseRequest
pdPublicBase PackumentDeps
deps) Maybe ClientCredential
forall a. Maybe a
Nothing
withRunInIO $ \forall a. Handler a -> IO a
runInIO ->
MemoryTicket
-> FlightKey
-> IO (Either MetadataError GemDirectory)
-> IO (Either MetadataError GemDirectory)
forall (m :: * -> *) a.
MonadUnliftIO m =>
MemoryTicket -> FlightKey -> m a -> m a
awaitingFlight MemoryTicket
ticket FlightKey
flight (IO (Either MetadataError GemDirectory)
-> IO (Either MetadataError GemDirectory))
-> IO (Either MetadataError GemDirectory)
-> IO (Either MetadataError GemDirectory)
forall a b. (a -> b) -> a -> b
$
MetricsPort
-> MetadataCache
-> CacheKey
-> IO (Either MetadataError GemDirectory)
-> IO (Either MetadataError GemDirectory)
resolveDirectory (ServeRuntime -> MetricsPort
srMetrics ServeRuntime
runtime) (ServeRuntime -> MetadataCache
srMetadataCache ServeRuntime
runtime) CacheKey
key (IO (Either MetadataError GemDirectory)
-> IO (Either MetadataError GemDirectory))
-> IO (Either MetadataError GemDirectory)
-> IO (Either MetadataError GemDirectory)
forall a b. (a -> b) -> a -> b
$
MetricsPort
-> Upstream
-> IO (Either MetadataError GemDirectory)
-> IO (Either MetadataError GemDirectory)
forall a.
MetricsPort
-> Upstream
-> IO (Either MetadataError a)
-> IO (Either MetadataError a)
recordedFetch (ServeRuntime -> MetricsPort
srMetrics ServeRuntime
runtime) Upstream
Metric.Public (IO (Either MetadataError GemDirectory)
-> IO (Either MetadataError GemDirectory))
-> IO (Either MetadataError GemDirectory)
-> IO (Either MetadataError GemDirectory)
forall a b. (a -> b) -> a -> b
$
(Handler () -> IO ())
-> TracingPort
-> (Int -> IO ())
-> Limits
-> DirectorySource
-> IO (Either MetadataError GemDirectory)
fetchDirectory Handler () -> IO ()
forall a. Handler a -> IO a
runInIO (ServeRuntime -> TracingPort
srTracing ServeRuntime
runtime) (ChargePart -> Int -> IO ()
chargePart (FlightKey -> ChargePart -> ChargePart
servingPart FlightKey
flight ChargePart
publicRead)) Limits
limits DirectorySource
source
where
limits :: Limits
limits = PackumentDeps -> Limits
pdLimits PackumentDeps
deps
key :: CacheKey
key = Source -> Int -> CacheKey
directoryKey (PackumentDeps -> Source
publicSource PackumentDeps
deps) (Limits -> Int
maxMetadataBytes Limits
limits)
flight :: FlightKey
flight = CacheKey -> FlightKey
flightOf CacheKey
key
boundBreach :: MetadataError -> Bool
boundBreach :: MetadataError -> Bool
boundBreach = \case
MetadataBoundExceeded LimitError
_ -> Bool
True
MetadataFetch (FetchBoundExceeded LimitError
_) -> Bool
True
MetadataError
_ -> Bool
False
serveInfo :: PackageName -> Request -> (RubyResponse -> IO ResponseReceived) -> Handler ResponseReceived
serveInfo :: PackageName
-> Request
-> (RubyResponse -> IO ResponseReceived)
-> Handler ResponseReceived
serveInfo PackageName
name Request
request RubyResponse -> IO ResponseReceived
respond =
(ServeRuntime
-> (ResponseHeaders -> Handler ResponseReceived)
-> (MemoryTicket -> Handler RubyResponse)
-> (RubyResponse -> Handler ResponseReceived)
-> Handler ResponseReceived)
-> Request
-> (RubyResponse -> Handler ResponseReceived)
-> (PackumentDeps -> RubyResponse)
-> (MemoryTicket -> PackumentDeps -> Handler RubyResponse)
-> (RubyResponse -> Handler ResponseReceived)
-> Handler ResponseReceived
forall a.
(ServeRuntime
-> (ResponseHeaders -> Handler ResponseReceived)
-> (MemoryTicket -> Handler a)
-> (a -> Handler ResponseReceived)
-> Handler ResponseReceived)
-> Request
-> (RubyResponse -> Handler ResponseReceived)
-> (PackumentDeps -> a)
-> (MemoryTicket -> PackumentDeps -> Handler a)
-> (a -> Handler ResponseReceived)
-> Handler ResponseReceived
withNativeRequest
ServeRuntime
-> (ResponseHeaders -> Handler ResponseReceived)
-> (MemoryTicket -> Handler RubyResponse)
-> (RubyResponse -> Handler ResponseReceived)
-> Handler ResponseReceived
forall (m :: * -> *) received response.
MonadUnliftIO m =>
ServeRuntime
-> (ResponseHeaders -> m received)
-> (MemoryTicket -> m response)
-> (response -> m received)
-> m received
withListingAdmission
Request
request
(IO ResponseReceived -> Handler ResponseReceived
forall a. IO a -> Handler a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO ResponseReceived -> Handler ResponseReceived)
-> (RubyResponse -> IO ResponseReceived)
-> RubyResponse
-> Handler ResponseReceived
forall b c a. (b -> c) -> (a -> b) -> a -> c
. RubyResponse -> IO ResponseReceived
respond)
(RubyResponse -> PackumentDeps -> RubyResponse
forall a b. a -> b -> a
const (MetadataError -> RubyResponse
directoryFailure (Int -> Maybe RetryAfter -> MetadataError
MetadataHttpFailure Int
503 Maybe RetryAfter
forall a. Maybe a
Nothing)))
( \MemoryTicket
ticket PackumentDeps
deps -> do
observe <- IO (Int -> IO ()) -> Handler (Int -> IO ())
forall a. IO a -> Handler a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (Int -> IO (Int -> IO ())
newNativeReadBudget (Limits -> Int
maxMetadataBytes (PackumentDeps -> Limits
pdLimits PackumentDeps
deps)))
check <- liftIO (newNativeDocumentBudget (maxMetadataBytes (pdLimits deps)) (Left (directoryFailure MetadataUndecodable)))
selected <-
withObservedAdmittedDocument
(mapPackumentReplies Left rubyReplies)
ticket
observe
check
Nothing
name
request
Nothing
( \CachedDoc
cached Int
_ Map Text Provenance
_ -> Either RubyResponse (CachedDoc, GemDocument)
-> Handler (Either RubyResponse (CachedDoc, GemDocument))
forall a. a -> Handler a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Either RubyResponse (CachedDoc, GemDocument)
-> Handler (Either RubyResponse (CachedDoc, GemDocument)))
-> Either RubyResponse (CachedDoc, GemDocument)
-> Handler (Either RubyResponse (CachedDoc, GemDocument))
forall a b. (a -> b) -> a -> b
$ case (GemDocument -> CachedDoc, CachedDoc -> Maybe GemDocument)
-> CachedDoc -> Maybe GemDocument
forall a b. (a, b) -> b
snd (GemDocument -> CachedDoc, CachedDoc -> Maybe GemDocument)
rubyGemsCached CachedDoc
cached of
Maybe GemDocument
Nothing -> RubyResponse -> Either RubyResponse (CachedDoc, GemDocument)
forall a b. a -> Either a b
Left (MetadataError -> RubyResponse
directoryFailure MetadataError
MetadataUndecodable)
Just GemDocument
document -> (CachedDoc, GemDocument)
-> Either RubyResponse (CachedDoc, GemDocument)
forall a b. b -> Either a b
Right (CachedDoc
cached, GemDocument
document)
)
case selected of
Left RubyResponse
refusal -> RubyResponse -> Handler RubyResponse
forall a. a -> Handler a
forall (f :: * -> *) a. Applicative f => a -> f a
pure RubyResponse
refusal
Right (CachedDoc
cached, GemDocument
document) -> do
let size :: Int
size = ByteString -> Int
BS.length (GemDocument -> ByteString
gemHeader GemDocument
document) Int -> Int -> Int
forall a. Num a => a -> a -> a
+ [Int] -> Int
forall a (f :: * -> *). (Foldable f, Num a) => f a -> a
sum [ByteString -> Int
BS.length (GemRow -> ByteString
grSource GemRow
row) Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1 | GemRow
row <- GemDocument -> [GemRow]
gemRows GemDocument
document]
IO () -> Handler ()
forall a. IO a -> Handler a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (MemoryTicket -> Int -> IO ()
charge MemoryTicket
ticket (Int
2 Int -> Int -> Int
forall a. Num a => a -> a -> a
* Int
size Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
4096))
case CachedDoc -> Either RenderRefused LByteString
serialiseRubyGemsDocument CachedDoc
cached of
Left RenderRefused
_ -> RubyResponse -> Handler RubyResponse
forall a. a -> Handler a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (MetadataError -> RubyResponse
directoryFailure MetadataError
MetadataUndecodable)
Right LByteString
encoded -> PackumentDeps
-> MemoryTicket -> Request -> ByteString -> Handler RubyResponse
answerCompact PackumentDeps
deps MemoryTicket
ticket Request
request (LByteString -> ByteString
LBS.toStrict LByteString
encoded)
)
(IO ResponseReceived -> Handler ResponseReceived
forall a. IO a -> Handler a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO ResponseReceived -> Handler ResponseReceived)
-> (RubyResponse -> IO ResponseReceived)
-> RubyResponse
-> Handler ResponseReceived
forall b c a. (b -> c) -> (a -> b) -> a -> c
. RubyResponse -> IO ResponseReceived
respond)
data DirectorySource = DirectorySource MalformedRows ExchangeClient ReadControl BaseRequest (Maybe ClientCredential)
data MalformedRows = DropMalformedRows | RejectMalformedRows
fetchDirectory :: (Handler () -> IO ()) -> TracingPort -> (Int -> IO ()) -> Limits -> DirectorySource -> IO (Either MetadataError GemDirectory)
fetchDirectory :: (Handler () -> IO ())
-> TracingPort
-> (Int -> IO ())
-> Limits
-> DirectorySource
-> IO (Either MetadataError GemDirectory)
fetchDirectory Handler () -> IO ()
runInIO TracingPort
tracing Int -> IO ()
pay Limits
limits (DirectorySource MalformedRows
policy ExchangeClient
client ReadControl
control BaseRequest
base Maybe ClientCredential
credential) = do
workspace <- (Int -> IO ()) -> IO (Int -> IO ())
newChargeRegion Int -> IO ()
pay
result <-
metadataResponse
<$> spanDirectoryFetch
tracing
( formThen
FetchUrlUnformable
( \Request
upstream -> ReadControl
-> ExchangeClient
-> ProgressFloor
-> Request
-> (Response BodyReader
-> IO
(Either LimitError (BodyOutcome (StreamResult DirectoryRead))))
-> IO
(Either FetchFault (BodyOutcome (StreamResult DirectoryRead)))
forall a.
ReadControl
-> ExchangeClient
-> ProgressFloor
-> Request
-> (Response BodyReader -> IO (Either LimitError a))
-> IO (Either FetchFault a)
runExchangeUsing ReadControl
control ExchangeClient
client (Limits -> ProgressFloor
progressFloor Limits
limits) Request
upstream ((Response BodyReader
-> IO
(Either LimitError (BodyOutcome (StreamResult DirectoryRead))))
-> IO
(Either FetchFault (BodyOutcome (StreamResult DirectoryRead))))
-> (Response BodyReader
-> IO
(Either LimitError (BodyOutcome (StreamResult DirectoryRead))))
-> IO
(Either FetchFault (BodyOutcome (StreamResult DirectoryRead)))
forall a b. (a -> b) -> a -> b
$ \Response BodyReader
response ->
if Status -> Int
statusCode (Response BodyReader -> Status
forall body. Response body -> Status
responseStatus Response BodyReader
response) Int -> Int -> Bool
forall a. Eq a => a -> a -> Bool
== Int
200
then (StreamResult DirectoryRead
-> BodyOutcome (StreamResult DirectoryRead))
-> Either LimitError (StreamResult DirectoryRead)
-> Either LimitError (BodyOutcome (StreamResult DirectoryRead))
forall a b. (a -> b) -> Either LimitError a -> Either LimitError b
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
fmap (Int
-> StreamResult DirectoryRead
-> BodyOutcome (StreamResult DirectoryRead)
forall a. Int -> a -> BodyOutcome a
SuccessBody Int
200) (Either LimitError (StreamResult DirectoryRead)
-> Either LimitError (BodyOutcome (StreamResult DirectoryRead)))
-> IO (Either LimitError (StreamResult DirectoryRead))
-> IO
(Either LimitError (BodyOutcome (StreamResult DirectoryRead)))
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> TracingPort -> forall a. IO a -> IO a
spanDirectoryDecode TracingPort
tracing ((forall a. IO a -> IO a)
-> (Steps IO DirectoryRead -> IO (Steps IO DirectoryRead))
-> BodyLimit
-> Steps IO DirectoryRead
-> BodyReader
-> IO (Either LimitError (StreamResult DirectoryRead))
forall (n :: * -> *) (m :: * -> *) s.
Monad n =>
(forall a. m a -> n a)
-> (Steps m s -> n (Steps m s))
-> BodyLimit
-> Steps m s
-> n ByteString
-> n (Either LimitError (StreamResult s))
readStepsWithEnd IO a -> IO a
forall a. a -> a
forall a. IO a -> IO a
id Steps IO DirectoryRead -> IO (Steps IO DirectoryRead)
forall (m :: * -> *) a. Monad m => Steps m a -> m (Steps m a)
finishRecordSteps (Int -> BodyLimit
MetadataBodyLimit (Limits -> Int
maxMetadataBytes Limits
limits)) ((Int -> IO ()) -> Steps IO DirectoryRead
forall (m :: * -> *).
Monad m =>
(Int -> m ()) -> Steps m DirectoryRead
directorySteps Int -> IO ()
workspace) (BodyReader -> BodyReader
brRead (Response BodyReader -> BodyReader
forall body. Response body -> body
responseBody Response BodyReader
response)))
else BodyOutcome (StreamResult DirectoryRead)
-> Either LimitError (BodyOutcome (StreamResult DirectoryRead))
forall a b. b -> Either a b
Right (BodyOutcome (StreamResult DirectoryRead)
-> Either LimitError (BodyOutcome (StreamResult DirectoryRead)))
-> (Maybe RetryAfter -> BodyOutcome (StreamResult DirectoryRead))
-> Maybe RetryAfter
-> Either LimitError (BodyOutcome (StreamResult DirectoryRead))
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Int -> Maybe RetryAfter -> BodyOutcome (StreamResult DirectoryRead)
forall a. Int -> Maybe RetryAfter -> BodyOutcome a
UnreadStatus (Status -> Int
statusCode (Response BodyReader -> Status
forall body. Response body -> Status
responseStatus Response BodyReader
response)) (Maybe RetryAfter
-> Either LimitError (BodyOutcome (StreamResult DirectoryRead)))
-> IO (Maybe RetryAfter)
-> IO
(Either LimitError (BodyOutcome (StreamResult DirectoryRead)))
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> ResponseHeaders -> IO (Maybe RetryAfter)
responseRetryAfter (Response BodyReader -> ResponseHeaders
forall body. Response body -> ResponseHeaders
responseHeaders Response BodyReader
response)
)
(sealRequest <$> gemDirectoryRequestUnder base credential)
)
let parsed = Either MetadataError (StreamResult DirectoryRead)
result Either MetadataError (StreamResult DirectoryRead)
-> (StreamResult DirectoryRead
-> Either MetadataError DirectoryRead)
-> Either MetadataError DirectoryRead
forall a b.
Either MetadataError a
-> (a -> Either MetadataError b) -> Either MetadataError b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= (ParseError -> MetadataError)
-> Either ParseError DirectoryRead
-> Either MetadataError DirectoryRead
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first (MetadataError -> ParseError -> MetadataError
forall a b. a -> b -> a
const MetadataError
MetadataUndecodable) (Either ParseError DirectoryRead
-> Either MetadataError DirectoryRead)
-> (StreamResult DirectoryRead -> Either ParseError DirectoryRead)
-> StreamResult DirectoryRead
-> Either MetadataError DirectoryRead
forall b c a. (b -> c) -> (a -> b) -> a -> c
. StreamResult DirectoryRead -> Either ParseError DirectoryRead
forall a. StreamResult a -> Either ParseError a
streamValue Either MetadataError DirectoryRead
-> (DirectoryRead -> Either MetadataError DirectoryRead)
-> Either MetadataError DirectoryRead
forall a b.
Either MetadataError a
-> (a -> Either MetadataError b) -> Either MetadataError b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= MalformedRows
-> DirectoryRead -> Either MetadataError DirectoryRead
keptUnder MalformedRows
policy
origin = RegistryUrl -> Text
registryUrlText (BaseRequest -> RegistryUrl
baseRequestUrl BaseRequest
base)
case parsed of
Left MetadataError
fault -> Handler () -> IO ()
runInIO (Text -> MetadataError -> Handler ()
forall (m :: * -> *).
KatipContext m =>
Text -> MetadataError -> m ()
logDirectoryFailure Text
origin MetadataError
fault)
Right DirectoryRead
directory -> Bool -> IO () -> IO ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
unless ([InvalidEntry] -> Bool
forall a. [a] -> Bool
forall (t :: * -> *) a. Foldable t => t a -> Bool
null (DirectoryRead -> [InvalidEntry]
readDropped DirectoryRead
directory)) (Handler () -> IO ()
runInIO (Text -> [InvalidEntry] -> Handler ()
forall (m :: * -> *).
KatipContext m =>
Text -> [InvalidEntry] -> m ()
logDirectoryInvalidEntries Text
origin (DirectoryRead -> [InvalidEntry]
readDropped DirectoryRead
directory)))
pure (readDirectory <$> parsed)
keptUnder :: MalformedRows -> DirectoryRead -> Either MetadataError DirectoryRead
keptUnder :: MalformedRows
-> DirectoryRead -> Either MetadataError DirectoryRead
keptUnder MalformedRows
policy DirectoryRead
parsed
| [InvalidEntry] -> Bool
forall a. [a] -> Bool
forall (t :: * -> *) a. Foldable t => t a -> Bool
null (DirectoryRead -> [InvalidEntry]
readDropped DirectoryRead
parsed) = DirectoryRead -> Either MetadataError DirectoryRead
forall a b. b -> Either a b
Right DirectoryRead
parsed
| MalformedRows
DropMalformedRows <- MalformedRows
policy, Bool -> Bool
not (ByteString -> Bool
BS.null (GemDirectory -> ByteString
directoryBytes (DirectoryRead -> GemDirectory
readDirectory DirectoryRead
parsed))) = DirectoryRead -> Either MetadataError DirectoryRead
forall a b. b -> Either a b
Right DirectoryRead
parsed
| Bool
otherwise = MetadataError -> Either MetadataError DirectoryRead
forall a b. a -> Either a b
Left MetadataError
MetadataUndecodable
answerCompact :: PackumentDeps -> MemoryTicket -> Request -> ByteString -> Handler RubyResponse
answerCompact :: PackumentDeps
-> MemoryTicket -> Request -> ByteString -> Handler RubyResponse
answerCompact PackumentDeps
deps MemoryTicket
ticket Request
request ByteString
bytes
| PackumentDeps -> ByteString -> Bool
overMetadataBound PackumentDeps
deps ByteString
bytes = RubyResponse -> Handler RubyResponse
forall a. a -> Handler a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (MetadataError -> RubyResponse
directoryFailure (LimitError -> MetadataError
MetadataBoundExceeded (BodyLimit -> LimitError
BodyTooLarge (Int -> BodyLimit
MetadataBodyLimit (Limits -> Int
maxMetadataBytes (PackumentDeps -> Limits
pdLimits PackumentDeps
deps))))))
| Bool
otherwise = PackumentReplies RubyResponse
-> PackumentDeps
-> MemoryTicket
-> Request
-> Digest SHA256
-> ByteString
-> Handler RubyResponse
forall response.
PackumentReplies response
-> PackumentDeps
-> MemoryTicket
-> Request
-> Digest SHA256
-> ByteString
-> Handler response
answerDigestedMetadataBytes PackumentReplies RubyResponse
replies PackumentDeps
deps MemoryTicket
ticket Request
request Digest SHA256
digest ByteString
bytes
where
digest :: Digest SHA256
digest = ByteString -> Digest SHA256
forall ba a.
(ByteArrayAccess ba, HashAlgorithm a) =>
ba -> Digest a
hash ByteString
bytes :: Digest SHA256
reprDigest :: ByteString
reprDigest = ByteString
"sha-256=\"" ByteString -> ByteString -> ByteString
forall a. Semigroup a => a -> a -> a
<> (Base -> Digest SHA256 -> ByteString
forall bin bout.
(ByteArrayAccess bin, ByteArray bout) =>
Base -> bin -> bout
convertToBase Base
Base64 Digest SHA256
digest :: ByteString) ByteString -> ByteString -> ByteString
forall a. Semigroup a => a -> a -> a
<> ByteString
"\""
replies :: PackumentReplies RubyResponse
replies =
PackumentReplies RubyResponse
rubyReplies
{ packumentOk = \ResponseHeaders
headers -> PackumentReplies RubyResponse
-> ResponseHeaders -> LByteString -> RubyResponse
forall response.
PackumentReplies response
-> ResponseHeaders -> LByteString -> response
packumentOk PackumentReplies RubyResponse
rubyReplies ((HeaderName
"Repr-Digest", ByteString
reprDigest) Header -> ResponseHeaders -> ResponseHeaders
forall a. a -> [a] -> [a]
: ResponseHeaders
headers)
, packumentNotModified = \ResponseHeaders
headers -> PackumentReplies RubyResponse -> ResponseHeaders -> RubyResponse
forall response.
PackumentReplies response -> ResponseHeaders -> response
packumentNotModified PackumentReplies RubyResponse
rubyReplies ((HeaderName
"Repr-Digest", ByteString
reprDigest) Header -> ResponseHeaders -> ResponseHeaders
forall a. a -> [a] -> [a]
: ResponseHeaders
headers)
}
overMetadataBound :: PackumentDeps -> ByteString -> Bool
overMetadataBound :: PackumentDeps -> ByteString -> Bool
overMetadataBound PackumentDeps
deps ByteString
bytes = ByteString -> Int
BS.length ByteString
bytes Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Limits -> Int
maxMetadataBytes (PackumentDeps -> Limits
pdLimits PackumentDeps
deps)
directoryFailure :: MetadataError -> RubyResponse
directoryFailure :: MetadataError -> RubyResponse
directoryFailure MetadataError
fault = case MetadataError
fault of
MetadataAuthorisationFailure Int
code -> PackumentReplies RubyResponse
-> Int -> ResponseHeaders -> Refusal -> RubyResponse
forall response.
PackumentReplies response
-> Int -> ResponseHeaders -> Refusal -> response
packumentPrivateRefused PackumentReplies RubyResponse
rubyReplies Int
code [] Refusal
directoryRefusal
MetadataError
MetadataUndecodable -> PackumentReplies RubyResponse
-> ResponseHeaders -> Refusal -> RubyResponse
forall response.
PackumentReplies response -> ResponseHeaders -> Refusal -> response
packumentBadGateway PackumentReplies RubyResponse
rubyReplies [] Refusal
directoryRefusal
MetadataBoundExceeded LimitError
_ -> PackumentReplies RubyResponse
-> ResponseHeaders -> Refusal -> RubyResponse
forall response.
PackumentReplies response -> ResponseHeaders -> Refusal -> response
packumentBadGateway PackumentReplies RubyResponse
rubyReplies [] Refusal
directoryRefusal
MetadataFetch (FetchBoundExceeded LimitError
_) -> PackumentReplies RubyResponse
-> ResponseHeaders -> Refusal -> RubyResponse
forall response.
PackumentReplies response -> ResponseHeaders -> Refusal -> response
packumentBadGateway PackumentReplies RubyResponse
rubyReplies [] Refusal
directoryRefusal
MetadataHttpFailure Int
400 Maybe RetryAfter
_ -> PackumentReplies RubyResponse
-> ResponseHeaders -> Refusal -> RubyResponse
forall response.
PackumentReplies response -> ResponseHeaders -> Refusal -> response
packumentBadGateway PackumentReplies RubyResponse
rubyReplies [] (Maybe HelpMessage -> Text -> Refusal
mkRefusal Maybe HelpMessage
forall a. Maybe a
Nothing Text
"the native directory rejected the read with HTTP 400. CodeArtifact directories support at most 5,000 gem names")
MetadataError
MetadataAbsent -> PackumentReplies RubyResponse
-> ResponseHeaders -> Refusal -> RubyResponse
forall response.
PackumentReplies response -> ResponseHeaders -> Refusal -> response
packumentBadGateway PackumentReplies RubyResponse
rubyReplies [] Refusal
directoryRefusal
MetadataHttpFailure Int
code Maybe RetryAfter
_ | Int
code Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
>= Int
200 Bool -> Bool -> Bool
&& Int
code Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
< Int
400 -> PackumentReplies RubyResponse
-> ResponseHeaders -> Refusal -> RubyResponse
forall response.
PackumentReplies response -> ResponseHeaders -> Refusal -> response
packumentBadGateway PackumentReplies RubyResponse
rubyReplies [] Refusal
directoryRefusal
MetadataError
_ -> PackumentReplies RubyResponse
-> ResponseHeaders -> Refusal -> RubyResponse
forall response.
PackumentReplies response -> ResponseHeaders -> Refusal -> response
packumentUnavailable PackumentReplies RubyResponse
rubyReplies [] Refusal
directoryRefusal
privateDirectoryFailure :: MetadataError -> RubyResponse
privateDirectoryFailure :: MetadataError -> RubyResponse
privateDirectoryFailure MetadataError
fault = case (MetadataError
fault, MetadataError -> Maybe ReadFailure
metadataReadFailure MetadataError
fault) of
(MetadataHttpFailure Int
400 Maybe RetryAfter
_, Maybe ReadFailure
_) -> MetadataError -> RubyResponse
directoryFailure MetadataError
fault
(MetadataError
_, Just (ReadUnavailable Maybe RetryAfter
retry)) -> PackumentReplies RubyResponse
-> ResponseHeaders -> Refusal -> RubyResponse
forall response.
PackumentReplies response -> ResponseHeaders -> Refusal -> response
packumentUnavailable PackumentReplies RubyResponse
rubyReplies (Maybe RetryAfter -> ResponseHeaders
retryAfterHeaders Maybe RetryAfter
retry) Refusal
directoryRefusal
(MetadataError
_, Just ReadFailure
ReadInvalid) -> PackumentReplies RubyResponse
-> ResponseHeaders -> Refusal -> RubyResponse
forall response.
PackumentReplies response -> ResponseHeaders -> Refusal -> response
packumentBadGateway PackumentReplies RubyResponse
rubyReplies [] Refusal
directoryRefusal
(MetadataError
_, Maybe ReadFailure
Nothing) -> MetadataError -> RubyResponse
directoryFailure MetadataError
fault
directoryRefusal :: Refusal
directoryRefusal :: Refusal
directoryRefusal = Maybe HelpMessage -> Text -> Refusal
mkRefusal Maybe HelpMessage
forall a. Maybe a
Nothing Text
"the RubyGems Compact Index response is unavailable"