module Ecluse.Runtime.Pilot.Export (
Published (..),
PublicationStop (..),
StoreFault (..),
RereadFinding (..),
renderPublicationStop,
maxPublicationAttempts,
publishCandidate,
) where
import Amazonka qualified as AWS
import Amazonka.S3 qualified as S3
import Amazonka.S3.Lens qualified as S3L
import Conduit (runResourceT)
import Data.HashMap.Strict qualified as HashMap
import Data.Text qualified as T
import Katip (KatipContext, Severity (..), katipAddContext, logFM, ls, sl)
import Lens.Micro ((.~), (^.))
import System.Directory (getFileSize)
import UnliftIO (MonadUnliftIO)
import UnliftIO.Exception (handle, try)
import Ecluse.Core.Fault (TransportCause (TransportProtocol), TransportFault, renderTransportFault, transportFault)
import Ecluse.Core.Osv.Compile (OsvCandidate, candidateFacts, candidatePath, stampCandidateNonce)
import Ecluse.Core.Osv.Publication
import Ecluse.Core.Osv.Publication.Envelope (decodeEnvelope, encodeEnvelope)
import Ecluse.Core.Telemetry.Record (timedSeconds)
import Ecluse.Core.Telemetry.Span (withOptionalSpan)
import Ecluse.Runtime.Aws.Env (singleAttemptEnv)
import Ecluse.Runtime.Aws.Fault (classifyAwsTransport, serviceStatus)
import Ecluse.Runtime.Aws.S3.Conditional (Capped (..), ConditionalPut (..), ResponseTooLarge (..), WritePrecondition (..), absenceListing, isWriteConflict, listingProvesAbsence)
import OpenTelemetry.Trace.Core (Span, SpanKind (Client), SpanStatus (Error), TracerProvider, addAttribute, setStatus)
data Published = Published
{ Published -> Maybe ObjectRevision
pubRevision :: Maybe ObjectRevision
, Published -> PublicationPlan
pubPlan :: PublicationPlan
}
deriving stock (Published -> Published -> Bool
(Published -> Published -> Bool)
-> (Published -> Published -> Bool) -> Eq Published
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: Published -> Published -> Bool
== :: Published -> Published -> Bool
$c/= :: Published -> Published -> Bool
/= :: Published -> Published -> Bool
Eq, Int -> Published -> ShowS
[Published] -> ShowS
Published -> String
(Int -> Published -> ShowS)
-> (Published -> String)
-> ([Published] -> ShowS)
-> Show Published
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> Published -> ShowS
showsPrec :: Int -> Published -> ShowS
$cshow :: Published -> String
show :: Published -> String
$cshowList :: [Published] -> ShowS
showList :: [Published] -> ShowS
Show)
data PublicationStop
=
PublicationRefused PublicationRefusal
|
PublicationFaulted StoreFault
deriving stock (PublicationStop -> PublicationStop -> Bool
(PublicationStop -> PublicationStop -> Bool)
-> (PublicationStop -> PublicationStop -> Bool)
-> Eq PublicationStop
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: PublicationStop -> PublicationStop -> Bool
== :: PublicationStop -> PublicationStop -> Bool
$c/= :: PublicationStop -> PublicationStop -> Bool
/= :: PublicationStop -> PublicationStop -> Bool
Eq, Int -> PublicationStop -> ShowS
[PublicationStop] -> ShowS
PublicationStop -> String
(Int -> PublicationStop -> ShowS)
-> (PublicationStop -> String)
-> ([PublicationStop] -> ShowS)
-> Show PublicationStop
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> PublicationStop -> ShowS
showsPrec :: Int -> PublicationStop -> ShowS
$cshow :: PublicationStop -> String
show :: PublicationStop -> String
$cshowList :: [PublicationStop] -> ShowS
showList :: [PublicationStop] -> ShowS
Show)
instance Exception PublicationStop where
displayException :: PublicationStop -> String
displayException = Text -> String
forall a. ToString a => a -> String
toString (Text -> String)
-> (PublicationStop -> Text) -> PublicationStop -> String
forall b c a. (b -> c) -> (a -> b) -> a -> c
. PublicationStop -> Text
renderPublicationStop
data StoreFault
=
StoreTransport TransportFault
|
StoreReadDenied
|
StoreAbsenceUnknown
|
StoreListDenied
|
StoreResponseTooLarge Int
|
StoreNoRevision
|
StoreConditionalWriteUnsupported
|
StoreEnvelopeRefused EnvelopeFault
|
StoreConflictsExhausted Int
|
StoreWriteUncertain TransportFault RereadFinding
deriving stock (StoreFault -> StoreFault -> Bool
(StoreFault -> StoreFault -> Bool)
-> (StoreFault -> StoreFault -> Bool) -> Eq StoreFault
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: StoreFault -> StoreFault -> Bool
== :: StoreFault -> StoreFault -> Bool
$c/= :: StoreFault -> StoreFault -> Bool
/= :: StoreFault -> StoreFault -> Bool
Eq, Int -> StoreFault -> ShowS
[StoreFault] -> ShowS
StoreFault -> String
(Int -> StoreFault -> ShowS)
-> (StoreFault -> String)
-> ([StoreFault] -> ShowS)
-> Show StoreFault
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> StoreFault -> ShowS
showsPrec :: Int -> StoreFault -> ShowS
$cshow :: StoreFault -> String
show :: StoreFault -> String
$cshowList :: [StoreFault] -> ShowS
showList :: [StoreFault] -> ShowS
Show)
data RereadFinding
= RereadNoObject
| RereadOtherRevision ObjectRevision
| RereadFailed StoreFault
deriving stock (RereadFinding -> RereadFinding -> Bool
(RereadFinding -> RereadFinding -> Bool)
-> (RereadFinding -> RereadFinding -> Bool) -> Eq RereadFinding
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: RereadFinding -> RereadFinding -> Bool
== :: RereadFinding -> RereadFinding -> Bool
$c/= :: RereadFinding -> RereadFinding -> Bool
/= :: RereadFinding -> RereadFinding -> Bool
Eq, Int -> RereadFinding -> ShowS
[RereadFinding] -> ShowS
RereadFinding -> String
(Int -> RereadFinding -> ShowS)
-> (RereadFinding -> String)
-> ([RereadFinding] -> ShowS)
-> Show RereadFinding
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> RereadFinding -> ShowS
showsPrec :: Int -> RereadFinding -> ShowS
$cshow :: RereadFinding -> String
show :: RereadFinding -> String
$cshowList :: [RereadFinding] -> ShowS
showList :: [RereadFinding] -> ShowS
Show)
maxPublicationAttempts :: Int
maxPublicationAttempts :: Int
maxPublicationAttempts = Int
3
renderPublicationStop :: PublicationStop -> Text
renderPublicationStop :: PublicationStop -> Text
renderPublicationStop = \case
PublicationRefused PublicationRefusal
refusal -> PublicationRefusal -> Text
renderPublicationRefusal PublicationRefusal
refusal
PublicationFaulted StoreFault
fault -> StoreFault -> Text
renderStoreFault StoreFault
fault
renderStoreFault :: StoreFault -> Text
renderStoreFault :: StoreFault -> Text
renderStoreFault = \case
StoreTransport TransportFault
fault -> Text
"the store request failed: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> TransportFault -> Text
renderTransportFault TransportFault
fault
StoreFault
StoreReadDenied -> Text
"the store refused HEAD on an existing object, so its publication metadata cannot be read. Pilot needs s3:GetObject"
StoreFault
StoreAbsenceUnknown -> Text
"the store refused HEAD, and its listing did not prove the key absent. Pilot needs s3:GetObject and prefix-scoped s3:ListBucket"
StoreFault
StoreListDenied -> Text
"the store refused HEAD and the one-key listing, so a missing object and a denied read look alike. Pilot needs s3:GetObject and s3:ListBucket limited to the advisory prefix"
StoreResponseTooLarge Int
bytes -> Text
"the store's response passed its " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Int -> Text
forall b a. (Show a, IsString b) => a -> b
show Int
bytes Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"-byte bound"
StoreFault
StoreNoRevision -> Text
"the store answered HEAD without an ETag or a length"
StoreFault
StoreConditionalWriteUnsupported -> Text
"the store does not implement conditional writes, which guarded publication requires"
StoreEnvelopeRefused EnvelopeFault
fault -> Text
"the new artifact's metadata cannot be written: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> EnvelopeFault -> Text
renderEnvelopeFault EnvelopeFault
fault
StoreConflictsExhausted Int
attempts -> Text
"another writer replaced the object during each of " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Int -> Text
forall b a. (Show a, IsString b) => a -> b
show Int
attempts Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" attempts"
StoreWriteUncertain TransportFault
fault RereadFinding
finding -> Text
"the write failed without proof either way (" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> TransportFault -> Text
renderTransportFault TransportFault
fault Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"), and a reread " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> RereadFinding -> Text
renderFinding RereadFinding
finding
renderFinding :: RereadFinding -> Text
renderFinding :: RereadFinding -> Text
renderFinding = \case
RereadFinding
RereadNoObject -> Text
"found no object"
RereadOtherRevision (ObjectRevision Text
tag) -> Text
"found revision " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
tag Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
", which no attempt of this publication wrote"
RereadFailed StoreFault
fault -> Text
"could not read the object: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> StoreFault -> Text
renderStoreFault StoreFault
fault
publishCandidate :: (MonadUnliftIO m, KatipContext m) => Maybe TracerProvider -> AWS.Env -> ShrinkPolicy -> BaselineCapability -> Maybe Rebaseline -> Text -> Text -> OsvCandidate -> m (Either PublicationStop Published)
publishCandidate :: forall (m :: * -> *).
(MonadUnliftIO m, KatipContext m) =>
Maybe TracerProvider
-> Env
-> ShrinkPolicy
-> BaselineCapability
-> Maybe Rebaseline
-> Text
-> Text
-> OsvCandidate
-> m (Either PublicationStop Published)
publishCandidate Maybe TracerProvider
mTracerProvider Env
env ShrinkPolicy
policy BaselineCapability
capability Maybe Rebaseline
rebaseline Text
bucket Text
key OsvCandidate
candidate =
Maybe TracerProvider
-> SpanKind
-> Text
-> (Maybe Span -> m (Either PublicationStop Published))
-> m (Either PublicationStop Published)
forall (m :: * -> *) a.
MonadUnliftIO m =>
Maybe TracerProvider
-> SpanKind -> Text -> (Maybe Span -> m a) -> m a
withOptionalSpan Maybe TracerProvider
mTracerProvider SpanKind
Client Text
"ecluse.pilot.osv.upload" ((Maybe Span -> m (Either PublicationStop Published))
-> m (Either PublicationStop Published))
-> (Maybe Span -> m (Either PublicationStop Published))
-> m (Either PublicationStop Published)
forall a b. (a -> b) -> a -> b
$ \Maybe Span
mSpan -> do
size <- IO Integer -> m Integer
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (String -> IO Integer
getFileSize (OsvCandidate -> String
candidatePath OsvCandidate
candidate))
stampUploadSpan mSpan bucket key size
(outcome, elapsed) <- timedSeconds (attempt 1 [])
katipAddContext (sl "bucket" bucket <> sl "object_key" key <> sl "bytes" size <> sl "duration_s" elapsed) $
either (reportStop mSpan) reportPublished outcome
pure outcome
where
attempt :: Int -> [PublicationPlan] -> m (Either PublicationStop Published)
attempt Int
number [PublicationPlan]
attempted =
IO (Either StoreFault StoreObservation)
-> m (Either StoreFault StoreObservation)
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (Env -> Text -> Text -> IO (Either StoreFault StoreObservation)
observeStore Env
env Text
bucket Text
key) m (Either StoreFault StoreObservation)
-> (Either StoreFault StoreObservation
-> m (Either PublicationStop Published))
-> m (Either PublicationStop Published)
forall a b. m a -> (a -> m b) -> m b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= \case
Left StoreFault
fault -> Either PublicationStop Published
-> m (Either PublicationStop Published)
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (PublicationStop -> Either PublicationStop Published
forall a b. a -> Either a b
Left (StoreFault -> PublicationStop
PublicationFaulted StoreFault
fault))
Right StoreObservation
observed
| Just Published
landed <- [PublicationPlan] -> StoreObservation -> Maybe Published
ownPublication [PublicationPlan]
attempted StoreObservation
observed -> Either PublicationStop Published
-> m (Either PublicationStop Published)
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Published -> Either PublicationStop Published
forall a b. b -> Either a b
Right Published
landed)
| Bool
otherwise -> do
nonce <- m PublicationNonce
forall (m :: * -> *). MonadIO m => m PublicationNonce
newPublicationNonce
stampCandidateNonce candidate nonce
bytes <- liftIO (getFileSize (candidatePath candidate))
let request = CandidateFacts
-> Integer
-> PublicationNonce
-> Maybe Rebaseline
-> PublicationRequest
PublicationRequest (OsvCandidate -> CandidateFacts
candidateFacts OsvCandidate
candidate) Integer
bytes PublicationNonce
nonce Maybe Rebaseline
rebaseline
case planPublication policy capability request observed of
Left PublicationRefusal
refusal -> Either PublicationStop Published
-> m (Either PublicationStop Published)
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (PublicationStop -> Either PublicationStop Published
forall a b. a -> Either a b
Left (PublicationRefusal -> PublicationStop
PublicationRefused PublicationRefusal
refusal))
Right PublicationPlan
plan -> Int
-> [PublicationPlan]
-> PublicationPlan
-> m (Either PublicationStop Published)
writePlanned Int
number (PublicationPlan
plan PublicationPlan -> [PublicationPlan] -> [PublicationPlan]
forall a. a -> [a] -> [a]
: [PublicationPlan]
attempted) PublicationPlan
plan
writePlanned :: Int
-> [PublicationPlan]
-> PublicationPlan
-> m (Either PublicationStop Published)
writePlanned Int
number [PublicationPlan]
attempted PublicationPlan
plan = case PublishedState -> Either EnvelopeFault [(Text, Text)]
encodeEnvelope (PublicationPlan -> PublishedState
plState PublicationPlan
plan) of
Left EnvelopeFault
fault -> Either PublicationStop Published
-> m (Either PublicationStop Published)
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (PublicationStop -> Either PublicationStop Published
forall a b. a -> Either a b
Left (StoreFault -> PublicationStop
PublicationFaulted (EnvelopeFault -> StoreFault
StoreEnvelopeRefused EnvelopeFault
fault)))
Right [(Text, Text)]
envelope ->
IO (Either ResponseTooLarge (Either Error (Maybe ObjectRevision)))
-> m (Either
ResponseTooLarge (Either Error (Maybe ObjectRevision)))
forall a. IO a -> m a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (Env
-> Text
-> Text
-> String
-> Precondition
-> [(Text, Text)]
-> IO
(Either ResponseTooLarge (Either Error (Maybe ObjectRevision)))
putCandidate Env
env Text
bucket Text
key (OsvCandidate -> String
candidatePath OsvCandidate
candidate) (PublicationPlan -> Precondition
plPrecondition PublicationPlan
plan) [(Text, Text)]
envelope) m (Either ResponseTooLarge (Either Error (Maybe ObjectRevision)))
-> (Either ResponseTooLarge (Either Error (Maybe ObjectRevision))
-> m (Either PublicationStop Published))
-> m (Either PublicationStop Published)
forall a b. m a -> (a -> m b) -> m b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= \case
Right (Right Maybe ObjectRevision
revision) -> Either PublicationStop Published
-> m (Either PublicationStop Published)
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Published -> Either PublicationStop Published
forall a b. b -> Either a b
Right (Maybe ObjectRevision -> PublicationPlan -> Published
Published Maybe ObjectRevision
revision PublicationPlan
plan))
Left (ResponseTooLarge Int
bytes) -> [PublicationPlan]
-> TransportFault -> m (Either PublicationStop Published)
forall {f :: * -> *}.
MonadIO f =>
[PublicationPlan]
-> TransportFault -> f (Either PublicationStop Published)
reconcile [PublicationPlan]
attempted (TransportCause -> Text -> TransportFault
transportFault TransportCause
TransportProtocol (Text
"the response passed its " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Int -> Text
forall b a. (Show a, IsString b) => a -> b
show Int
bytes Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"-byte bound"))
Right (Left Error
err)
| Error -> Bool
isWriteConflict Error
err Bool -> Bool -> Bool
&& Int
number Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
< Int
maxPublicationAttempts -> do
Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
WarningS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"Another writer replaced " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
key Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
" during publication, so Pilot rereads it and decides again"))
Int -> [PublicationPlan] -> m (Either PublicationStop Published)
attempt (Int
number Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) [PublicationPlan]
attempted
| Error -> Bool
isWriteConflict Error
err -> Either PublicationStop Published
-> m (Either PublicationStop Published)
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (PublicationStop -> Either PublicationStop Published
forall a b. a -> Either a b
Left (StoreFault -> PublicationStop
PublicationFaulted (Int -> StoreFault
StoreConflictsExhausted Int
number)))
| Error -> Maybe Int
serviceStatus Error
err Maybe Int -> Maybe Int -> Bool
forall a. Eq a => a -> a -> Bool
== Int -> Maybe Int
forall a. a -> Maybe a
Just Int
501 -> Either PublicationStop Published
-> m (Either PublicationStop Published)
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (PublicationStop -> Either PublicationStop Published
forall a b. a -> Either a b
Left (StoreFault -> PublicationStop
PublicationFaulted StoreFault
StoreConditionalWriteUnsupported))
| Bool
otherwise -> [PublicationPlan]
-> TransportFault -> m (Either PublicationStop Published)
forall {f :: * -> *}.
MonadIO f =>
[PublicationPlan]
-> TransportFault -> f (Either PublicationStop Published)
reconcile [PublicationPlan]
attempted (Error -> TransportFault
classifyAwsTransport Error
err)
reconcile :: [PublicationPlan]
-> TransportFault -> f (Either PublicationStop Published)
reconcile [PublicationPlan]
attempted TransportFault
fault =
IO (Either StoreFault StoreObservation)
-> f (Either StoreFault StoreObservation)
forall a. IO a -> f a
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (Env -> Text -> Text -> IO (Either StoreFault StoreObservation)
observeStore Env
env Text
bucket Text
key) f (Either StoreFault StoreObservation)
-> (Either StoreFault StoreObservation
-> Either PublicationStop Published)
-> f (Either PublicationStop Published)
forall (f :: * -> *) a b. Functor f => f a -> (a -> b) -> f b
<&> \case
Right StoreObservation
observed | Just Published
landed <- [PublicationPlan] -> StoreObservation -> Maybe Published
ownPublication [PublicationPlan]
attempted StoreObservation
observed -> Published -> Either PublicationStop Published
forall a b. b -> Either a b
Right Published
landed
Right StoreObservation
StoreAbsent -> RereadFinding -> Either PublicationStop Published
forall {b}. RereadFinding -> Either PublicationStop b
uncertain RereadFinding
RereadNoObject
Right (StorePresent ObjectRevision
revision Integer
_ Either EnvelopeFault PublishedState
_) -> RereadFinding -> Either PublicationStop Published
forall {b}. RereadFinding -> Either PublicationStop b
uncertain (ObjectRevision -> RereadFinding
RereadOtherRevision ObjectRevision
revision)
Left StoreFault
readFault -> RereadFinding -> Either PublicationStop Published
forall {b}. RereadFinding -> Either PublicationStop b
uncertain (StoreFault -> RereadFinding
RereadFailed StoreFault
readFault)
where
uncertain :: RereadFinding -> Either PublicationStop b
uncertain = PublicationStop -> Either PublicationStop b
forall a b. a -> Either a b
Left (PublicationStop -> Either PublicationStop b)
-> (RereadFinding -> PublicationStop)
-> RereadFinding
-> Either PublicationStop b
forall b c a. (b -> c) -> (a -> b) -> a -> c
. StoreFault -> PublicationStop
PublicationFaulted (StoreFault -> PublicationStop)
-> (RereadFinding -> StoreFault)
-> RereadFinding
-> PublicationStop
forall b c a. (b -> c) -> (a -> b) -> a -> c
. TransportFault -> RereadFinding -> StoreFault
StoreWriteUncertain TransportFault
fault
ownPublication :: [PublicationPlan] -> StoreObservation -> Maybe Published
ownPublication :: [PublicationPlan] -> StoreObservation -> Maybe Published
ownPublication [PublicationPlan]
attempted = \case
StorePresent ObjectRevision
revision Integer
_ (Right PublishedState
stored) ->
Maybe ObjectRevision -> PublicationPlan -> Published
Published (ObjectRevision -> Maybe ObjectRevision
forall a. a -> Maybe a
Just ObjectRevision
revision) (PublicationPlan -> Published)
-> Maybe PublicationPlan -> Maybe Published
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> (PublicationPlan -> Bool)
-> [PublicationPlan] -> Maybe PublicationPlan
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Maybe a
find ((PublicationNonce -> PublicationNonce -> Bool
forall a. Eq a => a -> a -> Bool
== PublishedState -> PublicationNonce
psNonce PublishedState
stored) (PublicationNonce -> Bool)
-> (PublicationPlan -> PublicationNonce) -> PublicationPlan -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. PublishedState -> PublicationNonce
psNonce (PublishedState -> PublicationNonce)
-> (PublicationPlan -> PublishedState)
-> PublicationPlan
-> PublicationNonce
forall b c a. (b -> c) -> (a -> b) -> a -> c
. PublicationPlan -> PublishedState
plState) [PublicationPlan]
attempted
StoreObservation
_ -> Maybe Published
forall a. Maybe a
Nothing
observeStore :: AWS.Env -> Text -> Text -> IO (Either StoreFault StoreObservation)
observeStore :: Env -> Text -> Text -> IO (Either StoreFault StoreObservation)
observeStore Env
env Text
bucket Text
key =
IO (Either StoreFault StoreObservation)
-> IO (Either StoreFault StoreObservation)
forall {b}. IO (Either StoreFault b) -> IO (Either StoreFault b)
capped (IO (Either StoreFault StoreObservation)
-> IO (Either StoreFault StoreObservation))
-> IO (Either StoreFault StoreObservation)
-> IO (Either StoreFault StoreObservation)
forall a b. (a -> b) -> a -> b
$
ResourceT IO (Either Error HeadObjectResponse)
-> IO (Either Error HeadObjectResponse)
forall (m :: * -> *) a. MonadUnliftIO m => ResourceT m a -> m a
runResourceT (Env
-> Capped HeadObject
-> ResourceT IO (Either Error (AWSResponse (Capped HeadObject)))
forall (m :: * -> *) a.
(MonadResource m, AWSRequest a) =>
Env -> a -> m (Either Error (AWSResponse a))
AWS.sendEither Env
env (HeadObject -> Capped HeadObject
forall a. a -> Capped a
Capped (BucketName -> ObjectKey -> HeadObject
S3.newHeadObject (Text -> BucketName
S3.BucketName Text
bucket) (Text -> ObjectKey
S3.ObjectKey Text
key)))) IO (Either Error HeadObjectResponse)
-> (Either Error HeadObjectResponse
-> IO (Either StoreFault StoreObservation))
-> IO (Either StoreFault StoreObservation)
forall a b. IO a -> (a -> IO b) -> IO b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= \case
Right HeadObjectResponse
response -> Either StoreFault StoreObservation
-> IO (Either StoreFault StoreObservation)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (HeadObjectResponse -> Either StoreFault StoreObservation
presentObject HeadObjectResponse
response)
Left Error
err
| Error -> Maybe Int
serviceStatus Error
err Maybe Int -> Maybe Int -> Bool
forall a. Eq a => a -> a -> Bool
== Int -> Maybe Int
forall a. a -> Maybe a
Just Int
404 -> Either StoreFault StoreObservation
-> IO (Either StoreFault StoreObservation)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (StoreObservation -> Either StoreFault StoreObservation
forall a b. b -> Either a b
Right StoreObservation
StoreAbsent)
| Error -> Maybe Int
serviceStatus Error
err Maybe Int -> Maybe Int -> Bool
forall a. Eq a => a -> a -> Bool
== Int -> Maybe Int
forall a. a -> Maybe a
Just Int
403 -> Env -> Text -> Text -> IO (Either StoreFault StoreObservation)
deniedHead Env
env Text
bucket Text
key
| Bool
otherwise -> Either StoreFault StoreObservation
-> IO (Either StoreFault StoreObservation)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (StoreFault -> Either StoreFault StoreObservation
forall a b. a -> Either a b
Left (TransportFault -> StoreFault
StoreTransport (Error -> TransportFault
classifyAwsTransport Error
err)))
where
capped :: IO (Either StoreFault b) -> IO (Either StoreFault b)
capped = (ResponseTooLarge -> IO (Either StoreFault b))
-> IO (Either StoreFault b) -> IO (Either StoreFault b)
forall (m :: * -> *) e a.
(MonadUnliftIO m, Exception e) =>
(e -> m a) -> m a -> m a
handle (\(ResponseTooLarge Int
bytes) -> Either StoreFault b -> IO (Either StoreFault b)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (StoreFault -> Either StoreFault b
forall a b. a -> Either a b
Left (Int -> StoreFault
StoreResponseTooLarge Int
bytes)))
presentObject :: S3.HeadObjectResponse -> Either StoreFault StoreObservation
presentObject :: HeadObjectResponse -> Either StoreFault StoreObservation
presentObject HeadObjectResponse
response = case (HeadObjectResponse
response HeadObjectResponse
-> Getting (Maybe ETag) HeadObjectResponse (Maybe ETag)
-> Maybe ETag
forall s a. s -> Getting a s a -> a
^. Getting (Maybe ETag) HeadObjectResponse (Maybe ETag)
Lens' HeadObjectResponse (Maybe ETag)
S3L.headObjectResponse_eTag, HeadObjectResponse
response HeadObjectResponse
-> Getting (Maybe Integer) HeadObjectResponse (Maybe Integer)
-> Maybe Integer
forall s a. s -> Getting a s a -> a
^. Getting (Maybe Integer) HeadObjectResponse (Maybe Integer)
Lens' HeadObjectResponse (Maybe Integer)
S3L.headObjectResponse_contentLength) of
(Just (S3.ETag ByteString
tag), Just Integer
size) | Bool -> Bool
not (Text -> Bool
T.null (ByteString -> Text
forall a b. ConvertUtf8 a b => b -> a
decodeUtf8 ByteString
tag)) -> StoreObservation -> Either StoreFault StoreObservation
forall a b. b -> Either a b
Right (ObjectRevision
-> Integer
-> Either EnvelopeFault PublishedState
-> StoreObservation
StorePresent (Text -> ObjectRevision
ObjectRevision (ByteString -> Text
forall a b. ConvertUtf8 a b => b -> a
decodeUtf8 ByteString
tag)) Integer
size Either EnvelopeFault PublishedState
stored)
(Maybe ETag, Maybe Integer)
_ -> StoreFault -> Either StoreFault StoreObservation
forall a b. a -> Either a b
Left StoreFault
StoreNoRevision
where
stored :: Either EnvelopeFault PublishedState
stored = [(Text, Text)] -> Either EnvelopeFault PublishedState
decodeEnvelope (HashMap Text Text -> [(Text, Text)]
forall k v. HashMap k v -> [(k, v)]
HashMap.toList (HeadObjectResponse
response HeadObjectResponse
-> Getting
(HashMap Text Text) HeadObjectResponse (HashMap Text Text)
-> HashMap Text Text
forall s a. s -> Getting a s a -> a
^. Getting (HashMap Text Text) HeadObjectResponse (HashMap Text Text)
Lens' HeadObjectResponse (HashMap Text Text)
S3L.headObjectResponse_metadata))
deniedHead :: AWS.Env -> Text -> Text -> IO (Either StoreFault StoreObservation)
deniedHead :: Env -> Text -> Text -> IO (Either StoreFault StoreObservation)
deniedHead Env
env Text
bucket Text
key =
ResourceT IO (Either Error ListObjectsV2Response)
-> IO (Either Error ListObjectsV2Response)
forall (m :: * -> *) a. MonadUnliftIO m => ResourceT m a -> m a
runResourceT (Env
-> Capped ListObjectsV2
-> ResourceT IO (Either Error (AWSResponse (Capped ListObjectsV2)))
forall (m :: * -> *) a.
(MonadResource m, AWSRequest a) =>
Env -> a -> m (Either Error (AWSResponse a))
AWS.sendEither Env
env (ListObjectsV2 -> Capped ListObjectsV2
forall a. a -> Capped a
Capped (Text -> Text -> ListObjectsV2
absenceListing Text
bucket Text
key))) IO (Either Error ListObjectsV2Response)
-> (Either Error ListObjectsV2Response
-> Either StoreFault StoreObservation)
-> IO (Either StoreFault StoreObservation)
forall (f :: * -> *) a b. Functor f => f a -> (a -> b) -> f b
<&> \case
Right ListObjectsV2Response
listing
| ListObjectsV2Response -> Bool
listingProvesAbsence ListObjectsV2Response
listing -> StoreObservation -> Either StoreFault StoreObservation
forall a b. b -> Either a b
Right StoreObservation
StoreAbsent
| (Object -> Bool) -> [Object] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
any ((ObjectKey -> ObjectKey -> Bool
forall a. Eq a => a -> a -> Bool
== Text -> ObjectKey
S3.ObjectKey Text
key) (ObjectKey -> Bool) -> (Object -> ObjectKey) -> Object -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Object -> Getting ObjectKey Object ObjectKey -> ObjectKey
forall s a. s -> Getting a s a -> a
^. Getting ObjectKey Object ObjectKey
Lens' Object ObjectKey
S3L.object_key)) ([Object] -> Maybe [Object] -> [Object]
forall a. a -> Maybe a -> a
fromMaybe [] (ListObjectsV2Response
listing ListObjectsV2Response
-> Getting (Maybe [Object]) ListObjectsV2Response (Maybe [Object])
-> Maybe [Object]
forall s a. s -> Getting a s a -> a
^. Getting (Maybe [Object]) ListObjectsV2Response (Maybe [Object])
Lens' ListObjectsV2Response (Maybe [Object])
S3L.listObjectsV2Response_contents)) -> StoreFault -> Either StoreFault StoreObservation
forall a b. a -> Either a b
Left StoreFault
StoreReadDenied
| Bool
otherwise -> StoreFault -> Either StoreFault StoreObservation
forall a b. a -> Either a b
Left StoreFault
StoreAbsenceUnknown
Left Error
err
| Error -> Maybe Int
serviceStatus Error
err Maybe Int -> Maybe Int -> Bool
forall a. Eq a => a -> a -> Bool
== Int -> Maybe Int
forall a. a -> Maybe a
Just Int
403 -> StoreFault -> Either StoreFault StoreObservation
forall a b. a -> Either a b
Left StoreFault
StoreListDenied
| Bool
otherwise -> StoreFault -> Either StoreFault StoreObservation
forall a b. a -> Either a b
Left (TransportFault -> StoreFault
StoreTransport (Error -> TransportFault
classifyAwsTransport Error
err))
putCandidate :: AWS.Env -> Text -> Text -> FilePath -> Precondition -> [(Text, Text)] -> IO (Either ResponseTooLarge (Either AWS.Error (Maybe ObjectRevision)))
putCandidate :: Env
-> Text
-> Text
-> String
-> Precondition
-> [(Text, Text)]
-> IO
(Either ResponseTooLarge (Either Error (Maybe ObjectRevision)))
putCandidate Env
env Text
bucket Text
key String
path Precondition
precondition [(Text, Text)]
envelope = IO (Either Error (Maybe ObjectRevision))
-> IO
(Either ResponseTooLarge (Either Error (Maybe ObjectRevision)))
forall (m :: * -> *) e a.
(MonadUnliftIO m, Exception e) =>
m a -> m (Either e a)
try (IO (Either Error (Maybe ObjectRevision))
-> IO
(Either ResponseTooLarge (Either Error (Maybe ObjectRevision))))
-> IO (Either Error (Maybe ObjectRevision))
-> IO
(Either ResponseTooLarge (Either Error (Maybe ObjectRevision)))
forall a b. (a -> b) -> a -> b
$ do
body <- ChunkSize -> String -> IO RequestBody
forall (m :: * -> *).
MonadIO m =>
ChunkSize -> String -> m RequestBody
AWS.chunkedFile ChunkSize
1048576 String
path
let native = BucketName -> ObjectKey -> RequestBody -> PutObject
S3.newPutObject (Text -> BucketName
S3.BucketName Text
bucket) (Text -> ObjectKey
S3.ObjectKey Text
key) RequestBody
body PutObject -> (PutObject -> PutObject) -> PutObject
forall a b. a -> (a -> b) -> b
& (HashMap Text Text -> Identity (HashMap Text Text))
-> PutObject -> Identity PutObject
Lens' PutObject (HashMap Text Text)
S3L.putObject_metadata ((HashMap Text Text -> Identity (HashMap Text Text))
-> PutObject -> Identity PutObject)
-> HashMap Text Text -> PutObject -> PutObject
forall s t a b. ASetter s t a b -> b -> s -> t
.~ [(Text, Text)] -> HashMap Text Text
forall k v. (Eq k, Hashable k) => [(k, v)] -> HashMap k v
HashMap.fromList [(Text, Text)]
envelope
result <- runResourceT (AWS.sendEither (singleAttemptEnv env) (Capped (ConditionalPut (writePrecondition precondition) native)))
pure (fmap (\PutObjectResponse
response -> (\(S3.ETag ByteString
tag) -> Text -> ObjectRevision
ObjectRevision (ByteString -> Text
forall a b. ConvertUtf8 a b => b -> a
decodeUtf8 ByteString
tag)) (ETag -> ObjectRevision) -> Maybe ETag -> Maybe ObjectRevision
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> PutObjectResponse
response PutObjectResponse
-> Getting (Maybe ETag) PutObjectResponse (Maybe ETag)
-> Maybe ETag
forall s a. s -> Getting a s a -> a
^. Getting (Maybe ETag) PutObjectResponse (Maybe ETag)
Lens' PutObjectResponse (Maybe ETag)
S3L.putObjectResponse_eTag) result)
writePrecondition :: Precondition -> WritePrecondition
writePrecondition :: Precondition -> WritePrecondition
writePrecondition = \case
Precondition
CreateOnly -> WritePrecondition
IfAbsent
ReplaceRevision (ObjectRevision Text
tag) -> ByteString -> WritePrecondition
IfRevision (Text -> ByteString
forall a b. ConvertUtf8 a b => a -> b
encodeUtf8 Text
tag)
stampUploadSpan :: (MonadIO m) => Maybe Span -> Text -> Text -> Integer -> m ()
stampUploadSpan :: forall (m :: * -> *).
MonadIO m =>
Maybe Span -> Text -> Text -> Integer -> m ()
stampUploadSpan Maybe Span
mSpan Text
bucket Text
key Integer
size =
Maybe Span -> (Span -> m ()) -> m ()
forall (f :: * -> *) a.
Applicative f =>
Maybe a -> (a -> f ()) -> f ()
whenJust Maybe Span
mSpan ((Span -> m ()) -> m ()) -> (Span -> m ()) -> m ()
forall a b. (a -> b) -> a -> b
$ \Span
sp -> do
Span -> Text -> Text -> m ()
forall (m :: * -> *) a.
(MonadIO m, ToAttribute a) =>
Span -> Text -> a -> m ()
addAttribute Span
sp Text
"ecluse.osv.bucket" Text
bucket
Span -> Text -> Text -> m ()
forall (m :: * -> *) a.
(MonadIO m, ToAttribute a) =>
Span -> Text -> a -> m ()
addAttribute Span
sp Text
"ecluse.osv.object_key" Text
key
Span -> Text -> Text -> m ()
forall (m :: * -> *) a.
(MonadIO m, ToAttribute a) =>
Span -> Text -> a -> m ()
addAttribute Span
sp Text
"ecluse.osv.bytes" (Integer -> Text
forall b a. (Show a, IsString b) => a -> b
show Integer
size :: Text)
reportPublished :: (KatipContext m) => Published -> m ()
reportPublished :: forall (m :: * -> *). KatipContext m => Published -> m ()
reportPublished Published
published =
SimpleLogPayload -> m () -> m ()
forall i (m :: * -> *) a.
(LogItem i, KatipContext m) =>
i -> m a -> m a
katipAddContext (Text -> Integer -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"baseline_bytes" (PublishedState -> Integer
psBaselineBytes PublishedState
stored) SimpleLogPayload -> SimpleLogPayload -> SimpleLogPayload
forall a. Semigroup a => a -> a -> a
<> Text -> Integer -> SimpleLogPayload
forall a. ToJSON a => Text -> a -> SimpleLogPayload
sl Text
"epss_credit_bytes" (PublishedState -> Integer
psEpssCreditBytes PublishedState
stored)) (m () -> m ()) -> m () -> m ()
forall a b. (a -> b) -> a -> b
$
Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
InfoS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"Published the advisory artifact: OSV " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> SourceOrder -> Text
renderSourceOrder (PublicationPlan -> SourceOrder
plOsvOrder PublicationPlan
plan) Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
", EPSS " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text -> (SourceOrder -> Text) -> Maybe SourceOrder -> Text
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Text
"unavailable" SourceOrder -> Text
renderSourceOrder (PublicationPlan -> Maybe SourceOrder
plEpssOrder PublicationPlan
plan) Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
", baseline " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Adjustment -> Text
renderAdjustment (PublishedState -> Adjustment
psAdjustment PublishedState
stored)))
where
plan :: PublicationPlan
plan = Published -> PublicationPlan
pubPlan Published
published
stored :: PublishedState
stored = PublicationPlan -> PublishedState
plState PublicationPlan
plan
reportStop :: (KatipContext m) => Maybe Span -> PublicationStop -> m ()
reportStop :: forall (m :: * -> *).
KatipContext m =>
Maybe Span -> PublicationStop -> m ()
reportStop Maybe Span
mSpan PublicationStop
stop = do
Maybe Span -> (Span -> m ()) -> m ()
forall (f :: * -> *) a.
Applicative f =>
Maybe a -> (a -> f ()) -> f ()
whenJust Maybe Span
mSpan ((Span -> m ()) -> m ()) -> (Span -> m ()) -> m ()
forall a b. (a -> b) -> a -> b
$ \Span
sp -> Span -> SpanStatus -> m ()
forall (m :: * -> *). MonadIO m => Span -> SpanStatus -> m ()
setStatus Span
sp (Text -> SpanStatus
Error (Text
"advisory publication stopped: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> PublicationStop -> Text
renderPublicationStop PublicationStop
stop))
Severity -> LogStr -> m ()
forall (m :: * -> *).
(Applicative m, KatipContext m) =>
Severity -> LogStr -> m ()
logFM Severity
ErrorS (Text -> LogStr
forall a. StringConv a Text => a -> LogStr
ls (Text
"Advisory publication stopped: " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> PublicationStop -> Text
renderPublicationStop PublicationStop
stop))