module Ecluse.Runtime.Cve.Sync.S3 (
S3CveSource,
newS3CveSource,
s3CveFetchFor,
s3CveFetch,
OsvDbCapExceeded (..),
cappedAt,
) where
import Conduit (ConduitT, runResourceT, (.|))
import Data.Conduit.Combinators qualified as C
import UnliftIO.Exception (catch, throwIO)
import Amazonka qualified as AWS
import Amazonka.S3 qualified as S3
import Amazonka.S3.Lens qualified as S3L
import Lens.Micro ((^.))
import Ecluse.Core.Cve.Types (DbEtag (..), OsvDbFetchFault (..))
import Ecluse.Core.Stream (boundBytes)
import Ecluse.Runtime.Aws.Env (AwsEndpoint)
import Ecluse.Runtime.Aws.Fault (classifyAwsTransport, foldTransportEscapes, serviceStatus)
import Ecluse.Runtime.Aws.S3 (buildS3Env)
import Ecluse.Runtime.Cve.Sync.Step (CveFetch (..), FetchedObject (..))
newtype OsvDbCapExceeded = OsvDbCapExceeded Int
deriving stock (OsvDbCapExceeded -> OsvDbCapExceeded -> Bool
(OsvDbCapExceeded -> OsvDbCapExceeded -> Bool)
-> (OsvDbCapExceeded -> OsvDbCapExceeded -> Bool)
-> Eq OsvDbCapExceeded
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: OsvDbCapExceeded -> OsvDbCapExceeded -> Bool
== :: OsvDbCapExceeded -> OsvDbCapExceeded -> Bool
$c/= :: OsvDbCapExceeded -> OsvDbCapExceeded -> Bool
/= :: OsvDbCapExceeded -> OsvDbCapExceeded -> Bool
Eq, Int -> OsvDbCapExceeded -> ShowS
[OsvDbCapExceeded] -> ShowS
OsvDbCapExceeded -> String
(Int -> OsvDbCapExceeded -> ShowS)
-> (OsvDbCapExceeded -> String)
-> ([OsvDbCapExceeded] -> ShowS)
-> Show OsvDbCapExceeded
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> OsvDbCapExceeded -> ShowS
showsPrec :: Int -> OsvDbCapExceeded -> ShowS
$cshow :: OsvDbCapExceeded -> String
show :: OsvDbCapExceeded -> String
$cshowList :: [OsvDbCapExceeded] -> ShowS
showList :: [OsvDbCapExceeded] -> ShowS
Show)
instance Exception OsvDbCapExceeded
newtype S3CveSource = S3CveSource
{ S3CveSource -> Text -> Text -> Int -> CveFetch
s3CveFetchFor :: Text -> Text -> Int -> CveFetch
}
newS3CveSource :: Maybe AwsEndpoint -> IO S3CveSource
newS3CveSource :: Maybe AwsEndpoint -> IO S3CveSource
newS3CveSource Maybe AwsEndpoint
mEndpoint = do
awsEnv <- Maybe AwsEndpoint -> IO Env
buildS3Env Maybe AwsEndpoint
mEndpoint
pure (S3CveSource (s3CveFetch awsEnv))
s3CveFetch :: AWS.Env -> Text -> Text -> Int -> CveFetch
s3CveFetch :: Env -> Text -> Text -> Int -> CveFetch
s3CveFetch Env
awsEnv Text
bucket Text
key Int
maxBytes =
CveFetch
{ fetchHead :: IO (Either OsvDbFetchFault (Maybe FetchedObject))
fetchHead = Env
-> Text
-> Text
-> IO (Either OsvDbFetchFault (Maybe FetchedObject))
s3Head Env
awsEnv Text
bucket Text
key
, fetchDownload :: String -> IO (Either OsvDbFetchFault FetchedObject)
fetchDownload = Env
-> Text
-> Text
-> Int
-> String
-> IO (Either OsvDbFetchFault FetchedObject)
s3Download Env
awsEnv Text
bucket Text
key Int
maxBytes
}
s3Head :: AWS.Env -> Text -> Text -> IO (Either OsvDbFetchFault (Maybe FetchedObject))
s3Head :: Env
-> Text
-> Text
-> IO (Either OsvDbFetchFault (Maybe FetchedObject))
s3Head Env
awsEnv Text
bucket Text
key =
ResourceT IO (Either Error HeadObjectResponse)
-> IO (Either Error HeadObjectResponse)
forall (m :: * -> *) a. MonadUnliftIO m => ResourceT m a -> m a
runResourceT (Env
-> HeadObject
-> ResourceT IO (Either Error (AWSResponse HeadObject))
forall (m :: * -> *) a.
(MonadResource m, AWSRequest a) =>
Env -> a -> m (Either Error (AWSResponse a))
AWS.sendEither Env
awsEnv (BucketName -> ObjectKey -> HeadObject
S3.newHeadObject (Text -> BucketName
S3.BucketName Text
bucket) (Text -> ObjectKey
S3.ObjectKey Text
key))) IO (Either Error HeadObjectResponse)
-> (Either Error HeadObjectResponse
-> Either OsvDbFetchFault (Maybe FetchedObject))
-> IO (Either OsvDbFetchFault (Maybe FetchedObject))
forall (f :: * -> *) a b. Functor f => f a -> (a -> b) -> f b
<&> \case
Right HeadObjectResponse
resp ->
let observed :: ETag -> FetchedObject
observed ETag
etag = DbEtag -> Maybe UTCTime -> FetchedObject
FetchedObject (ETag -> DbEtag
dbEtag ETag
etag) (HeadObjectResponse
resp HeadObjectResponse
-> Getting (Maybe UTCTime) HeadObjectResponse (Maybe UTCTime)
-> Maybe UTCTime
forall s a. s -> Getting a s a -> a
^. Getting (Maybe UTCTime) HeadObjectResponse (Maybe UTCTime)
Lens' HeadObjectResponse (Maybe UTCTime)
S3L.headObjectResponse_lastModified)
in Either OsvDbFetchFault (Maybe FetchedObject)
-> (ETag -> Either OsvDbFetchFault (Maybe FetchedObject))
-> Maybe ETag
-> Either OsvDbFetchFault (Maybe FetchedObject)
forall b a. b -> (a -> b) -> Maybe a -> b
maybe (OsvDbFetchFault -> Either OsvDbFetchFault (Maybe FetchedObject)
forall a b. a -> Either a b
Left OsvDbFetchFault
OsvDbNoEtag) (Maybe FetchedObject -> Either OsvDbFetchFault (Maybe FetchedObject)
forall a b. b -> Either a b
Right (Maybe FetchedObject
-> Either OsvDbFetchFault (Maybe FetchedObject))
-> (ETag -> Maybe FetchedObject)
-> ETag
-> Either OsvDbFetchFault (Maybe FetchedObject)
forall b c a. (b -> c) -> (a -> b) -> a -> c
. FetchedObject -> Maybe FetchedObject
forall a. a -> Maybe a
Just (FetchedObject -> Maybe FetchedObject)
-> (ETag -> FetchedObject) -> ETag -> Maybe FetchedObject
forall b c a. (b -> c) -> (a -> b) -> a -> c
. ETag -> FetchedObject
observed) (HeadObjectResponse
resp HeadObjectResponse
-> Getting (Maybe ETag) HeadObjectResponse (Maybe ETag)
-> Maybe ETag
forall s a. s -> Getting a s a -> a
^. Getting (Maybe ETag) HeadObjectResponse (Maybe ETag)
Lens' HeadObjectResponse (Maybe ETag)
S3L.headObjectResponse_eTag)
Left Error
err
| Error -> Maybe Int
serviceStatus Error
err Maybe Int -> Maybe Int -> Bool
forall a. Eq a => a -> a -> Bool
== Int -> Maybe Int
forall a. a -> Maybe a
Just Int
404 -> Maybe FetchedObject -> Either OsvDbFetchFault (Maybe FetchedObject)
forall a b. b -> Either a b
Right Maybe FetchedObject
forall a. Maybe a
Nothing
| Bool
otherwise -> OsvDbFetchFault -> Either OsvDbFetchFault (Maybe FetchedObject)
forall a b. a -> Either a b
Left (TransportFault -> OsvDbFetchFault
OsvDbTransport (Error -> TransportFault
classifyAwsTransport Error
err))
s3Download :: AWS.Env -> Text -> Text -> Int -> FilePath -> IO (Either OsvDbFetchFault FetchedObject)
s3Download :: Env
-> Text
-> Text
-> Int
-> String
-> IO (Either OsvDbFetchFault FetchedObject)
s3Download Env
awsEnv Text
bucket Text
key Int
maxBytes String
dest = IO (Either OsvDbFetchFault FetchedObject)
-> IO (Either OsvDbFetchFault FetchedObject)
foldFetchEscapes (IO (Either OsvDbFetchFault FetchedObject)
-> IO (Either OsvDbFetchFault FetchedObject))
-> (ResourceT IO (Either OsvDbFetchFault FetchedObject)
-> IO (Either OsvDbFetchFault FetchedObject))
-> ResourceT IO (Either OsvDbFetchFault FetchedObject)
-> IO (Either OsvDbFetchFault FetchedObject)
forall b c a. (b -> c) -> (a -> b) -> a -> c
. ResourceT IO (Either OsvDbFetchFault FetchedObject)
-> IO (Either OsvDbFetchFault FetchedObject)
forall (m :: * -> *) a. MonadUnliftIO m => ResourceT m a -> m a
runResourceT (ResourceT IO (Either OsvDbFetchFault FetchedObject)
-> IO (Either OsvDbFetchFault FetchedObject))
-> ResourceT IO (Either OsvDbFetchFault FetchedObject)
-> IO (Either OsvDbFetchFault FetchedObject)
forall a b. (a -> b) -> a -> b
$ do
resp <- Env -> GetObject -> ResourceT IO (AWSResponse GetObject)
forall (m :: * -> *) a.
(MonadResource m, AWSRequest a) =>
Env -> a -> m (AWSResponse a)
AWS.send Env
awsEnv (BucketName -> ObjectKey -> GetObject
S3.newGetObject (Text -> BucketName
S3.BucketName Text
bucket) (Text -> ObjectKey
S3.ObjectKey Text
key))
for_ (resp ^. S3L.getObjectResponse_contentLength) $ \Integer
len ->
Bool -> ResourceT IO () -> ResourceT IO ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when (Integer
len Integer -> Integer -> Bool
forall a. Ord a => a -> a -> Bool
> Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral Int
maxBytes) (OsvDbCapExceeded -> ResourceT IO ()
forall (m :: * -> *) e a. (MonadIO m, Exception e) => e -> m a
throwIO (Int -> OsvDbCapExceeded
OsvDbCapExceeded Int
maxBytes))
AWS.sinkBody (resp ^. S3L.getObjectResponse_body) (cappedAt maxBytes .| C.sinkFile dest)
let fetched ETag
etag = FetchedObject{foEtag :: DbEtag
foEtag = ETag -> DbEtag
dbEtag ETag
etag, foPushedAt :: Maybe UTCTime
foPushedAt = GetObjectResponse
resp GetObjectResponse
-> Getting (Maybe UTCTime) GetObjectResponse (Maybe UTCTime)
-> Maybe UTCTime
forall s a. s -> Getting a s a -> a
^. Getting (Maybe UTCTime) GetObjectResponse (Maybe UTCTime)
Lens' GetObjectResponse (Maybe UTCTime)
S3L.getObjectResponse_lastModified}
pure (maybe (Left OsvDbNoEtag) (Right . fetched) (resp ^. S3L.getObjectResponse_eTag))
foldFetchEscapes :: IO (Either OsvDbFetchFault FetchedObject) -> IO (Either OsvDbFetchFault FetchedObject)
foldFetchEscapes :: IO (Either OsvDbFetchFault FetchedObject)
-> IO (Either OsvDbFetchFault FetchedObject)
foldFetchEscapes IO (Either OsvDbFetchFault FetchedObject)
act =
(TransportFault -> OsvDbFetchFault)
-> IO (Either OsvDbFetchFault FetchedObject)
-> IO (Either OsvDbFetchFault FetchedObject)
forall fault a.
(TransportFault -> fault)
-> IO (Either fault a) -> IO (Either fault a)
foldTransportEscapes TransportFault -> OsvDbFetchFault
OsvDbTransport IO (Either OsvDbFetchFault FetchedObject)
act
IO (Either OsvDbFetchFault FetchedObject)
-> (OsvDbCapExceeded -> IO (Either OsvDbFetchFault FetchedObject))
-> IO (Either OsvDbFetchFault FetchedObject)
forall (m :: * -> *) e a.
(MonadUnliftIO m, Exception e) =>
m a -> (e -> m a) -> m a
`catch` (\(OsvDbCapExceeded Int
n) -> Either OsvDbFetchFault FetchedObject
-> IO (Either OsvDbFetchFault FetchedObject)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (OsvDbFetchFault -> Either OsvDbFetchFault FetchedObject
forall a b. a -> Either a b
Left (Int -> OsvDbFetchFault
OsvDbTooLarge Int
n)))
dbEtag :: S3.ETag -> DbEtag
dbEtag :: ETag -> DbEtag
dbEtag (S3.ETag ByteString
bytes) = Text -> DbEtag
DbEtag (ByteString -> Text
forall a b. ConvertUtf8 a b => b -> a
decodeUtf8 ByteString
bytes)
cappedAt :: (MonadIO m) => Int -> ConduitT ByteString ByteString m ()
cappedAt :: forall (m :: * -> *).
MonadIO m =>
Int -> ConduitT ByteString ByteString m ()
cappedAt Int
maxBytes = Int -> (Int -> m ()) -> ConduitT ByteString ByteString m ()
forall (m :: * -> *).
Monad m =>
Int -> (Int -> m ()) -> ConduitT ByteString ByteString m ()
boundBytes Int
maxBytes (m () -> Int -> m ()
forall a b. a -> b -> a
const (OsvDbCapExceeded -> m ()
forall (m :: * -> *) e a. (MonadIO m, Exception e) => e -> m a
throwIO (Int -> OsvDbCapExceeded
OsvDbCapExceeded Int
maxBytes)))