module Ecluse.Core.Registry.Cargo.Archive (
CargoArchiveMetadata,
inspectCargoArchive,
cargoPublicationMetadata,
cargoPublicationAgreement,
cargoUploadAgreement,
) where
import Data.Aeson (Value)
import Data.ByteString qualified as BS
import Data.Map.Strict qualified as Map
import Data.Set qualified as Set
import Data.Text qualified as T
import Data.Text.Encoding qualified as TE
import UnliftIO.Exception (evaluate)
import Ecluse.Core.Package.Hash (sha256Hex)
import Ecluse.Core.Registry (ParseError (..), parseRefusal)
import Ecluse.Core.Registry.Archive.Inflate (InflateFormat (GzipMembers), withInflatedChunks)
import Ecluse.Core.Registry.Archive.Tar (TarMember (..), TarScan, scanTar, sourceReader, tarContents, tarMembers)
import Ecluse.Core.Registry.CachedDocument (estimateValueBytes)
import Ecluse.Core.Registry.Cargo.Archive.Internal (CargoArchiveMetadata (..))
import Ecluse.Core.Registry.Cargo.Manifest (CargoPublishMetadata, checkPublicationIndex, checkUploadAgreement, decodeManifest, manifestTextPaths, publicationMetadata, publishMetadataValue)
import Ecluse.Core.Registry.Cargo.Project (cargoArchiveStem)
import Ecluse.Core.Registry.Cargo.Wire (cargoTreeCharge)
import Ecluse.Core.Security.Limits (Limits (..))
import Ecluse.Core.Server.Admission.Meter (newChargePair)
import Ecluse.Core.Server.MemoryModel (saturatingCharge)
inspectCargoArchive :: Limits -> (Int -> IO ()) -> Text -> Text -> Maybe Text -> ByteString -> IO (Either ParseError CargoArchiveMetadata)
inspectCargoArchive :: Limits
-> (Int -> IO ())
-> Text
-> Text
-> Maybe Text
-> ByteString
-> IO (Either ParseError CargoArchiveMetadata)
inspectCargoArchive Limits
limits Int -> IO ()
charge Text
name Text
version Maybe Text
checksum ByteString
bytes = do
if Limits -> Int
maxMirrorArtifactBytes Limits
limits Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
<= Int
0 Bool -> Bool -> Bool
|| ByteString -> Int
BS.length ByteString
bytes Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Limits -> Int
maxMirrorArtifactBytes Limits
limits
then Either ParseError CargoArchiveMetadata
-> IO (Either ParseError CargoArchiveMetadata)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> Either ParseError CargoArchiveMetadata
forall a. Text -> Either ParseError a
parseRefusal Text
"Cargo compressed archive exceeds its byte bound")
else
if (Text -> Bool) -> Maybe Text -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
any (\Text
expected -> Text -> Text
T.toLower Text
expected Text -> Text -> Bool
forall a. Eq a => a -> a -> Bool
/= ByteString -> Text
sha256Hex ByteString
bytes) Maybe Text
checksum
then Either ParseError CargoArchiveMetadata
-> IO (Either ParseError CargoArchiveMetadata)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> Either ParseError CargoArchiveMetadata
forall a. Text -> Either ParseError a
parseRefusal Text
"Cargo compressed archive disagrees with its admitted SHA-256")
else do
let root :: Text
root = Text -> Text -> Text
cargoArchiveStem Text
name Text
version
result <- Limits
-> (Int -> IO ())
-> Text
-> Set Text
-> ByteString
-> IO (Either ParseError TarScan)
archivePass Limits
limits Int -> IO ()
charge Text
root (Text -> Set Text
forall a. a -> Set a
Set.singleton Text
"Cargo.toml") ByteString
bytes
case result >>= archiveFacts root of
Left ParseError
fault -> Either ParseError CargoArchiveMetadata
-> IO (Either ParseError CargoArchiveMetadata)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (ParseError -> Either ParseError CargoArchiveMetadata
forall a b. a -> Either a b
Left ParseError
fault)
Right (ByteString
manifestBytes, Set Text
inventory, Int
retained) -> do
Int -> IO ()
charge (Int
retained Int -> Int -> Int
forall a. Num a => a -> a -> a
+ ByteString -> Int
manifestMemory ByteString
manifestBytes)
parsed <- Either ParseError CargoManifest
-> IO (Either ParseError CargoManifest)
forall (m :: * -> *) a. MonadIO m => a -> m a
evaluate (Limits
-> Text -> Text -> ByteString -> Either ParseError CargoManifest
decodeManifest Limits
limits Text
name Text
version ByteString
manifestBytes)
pure $ (\CargoManifest
manifest -> CargoManifest
-> Set Text
-> ByteString
-> Text
-> Int
-> Int
-> CargoArchiveMetadata
CargoArchiveMetadata CargoManifest
manifest Set Text
inventory ByteString
bytes Text
root (Int
retained Int -> Int -> Int
forall a. Num a => a -> a -> a
+ ByteString -> Int
manifestMemory ByteString
manifestBytes) (ByteString -> Int
manifestMemory ByteString
manifestBytes)) <$> parsed
cargoPublicationMetadata :: Limits -> (Int -> IO ()) -> CargoArchiveMetadata -> IO (Either ParseError CargoPublishMetadata)
cargoPublicationMetadata :: Limits
-> (Int -> IO ())
-> CargoArchiveMetadata
-> IO (Either ParseError CargoPublishMetadata)
cargoPublicationMetadata Limits
limits Int -> IO ()
charge CargoArchiveMetadata
metadata =
case CargoManifest -> Set Text -> Either ParseError [Text]
manifestTextPaths (CargoArchiveMetadata -> CargoManifest
caManifest CargoArchiveMetadata
metadata) (CargoArchiveMetadata -> Set Text
caInventory CargoArchiveMetadata
metadata) of
Left ParseError
fault -> Either ParseError CargoPublishMetadata
-> IO (Either ParseError CargoPublishMetadata)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (ParseError -> Either ParseError CargoPublishMetadata
forall a b. a -> Either a b
Left ParseError
fault)
Right [] -> Map Text ByteString -> IO (Either ParseError CargoPublishMetadata)
finish Map Text ByteString
forall k a. Map k a
Map.empty
Right [Text]
paths -> do
result <- Limits
-> (Int -> IO ())
-> Text
-> Set Text
-> ByteString
-> IO (Either ParseError TarScan)
archivePass Limits
limits (Int -> IO ()
charge (Int -> IO ()) -> (Int -> Int) -> Int -> IO ()
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
retained)) (CargoArchiveMetadata -> Text
caRoot CargoArchiveMetadata
metadata) ([Text] -> Set Text
forall a. Ord a => [a] -> Set a
Set.fromList [Text]
paths) (CargoArchiveMetadata -> ByteString
caSource CargoArchiveMetadata
metadata)
case result of
Left ParseError
fault -> Either ParseError CargoPublishMetadata
-> IO (Either ParseError CargoPublishMetadata)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (ParseError -> Either ParseError CargoPublishMetadata
forall a b. a -> Either a b
Left ParseError
fault)
Right TarScan
scan -> Map Text ByteString -> IO (Either ParseError CargoPublishMetadata)
finish ((Text -> Text) -> Map Text ByteString -> Map Text ByteString
forall k2 k1 a. Ord k2 => (k1 -> k2) -> Map k1 a -> Map k2 a
Map.mapKeys Text -> Text
dropArchiveRoot (TarScan -> Map Text ByteString
tarContents TarScan
scan))
where
retained :: Int
retained = CargoArchiveMetadata -> Int
caRetainedBytes CargoArchiveMetadata
metadata
finish :: Map Text ByteString -> IO (Either ParseError CargoPublishMetadata)
finish Map Text ByteString
content = do
let textBytes :: Int
textBytes = [Int] -> Int
forall a (f :: * -> *). (Foldable f, Num a) => f a -> a
sum ((ByteString -> Int) -> [ByteString] -> [Int]
forall a b. (a -> b) -> [a] -> [b]
map ByteString -> Int
BS.length (Map Text ByteString -> [ByteString]
forall k a. Map k a -> [a]
Map.elems Map Text ByteString
content))
Int -> IO ()
charge (Int
retained Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
65536 Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
8 Int -> Int -> Int
forall a. Num a => a -> a -> a
* Int
textBytes Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
512 Int -> Int -> Int
forall a. Num a => a -> a -> a
* Map Text ByteString -> Int
forall k a. Map k a -> Int
Map.size Map Text ByteString
content)
Either ParseError CargoPublishMetadata
-> IO (Either ParseError CargoPublishMetadata)
forall (m :: * -> *) a. MonadIO m => a -> m a
evaluate (CargoManifest
-> Map Text ByteString -> Either ParseError CargoPublishMetadata
publicationMetadata (CargoArchiveMetadata -> CargoManifest
caManifest CargoArchiveMetadata
metadata) Map Text ByteString
content)
cargoPublicationAgreement :: Limits -> (Int -> IO ()) -> CargoArchiveMetadata -> Value -> CargoPublishMetadata -> IO (Either ParseError CargoPublishMetadata)
cargoPublicationAgreement :: Limits
-> (Int -> IO ())
-> CargoArchiveMetadata
-> Value
-> CargoPublishMetadata
-> IO (Either ParseError CargoPublishMetadata)
cargoPublicationAgreement Limits
limits Int -> IO ()
charge CargoArchiveMetadata
archive Value
source CargoPublishMetadata
metadata = do
let inputs :: [Value]
inputs = [Value
source, CargoPublishMetadata -> Value
publishMetadataValue CargoPublishMetadata
metadata]
workspace :: Integer
workspace = [Integer] -> Integer
forall a (f :: * -> *). (Foldable f, Num a) => f a -> a
sum ((Value -> Integer) -> [Value] -> [Integer]
forall a b. (a -> b) -> [a] -> [b]
map (Int -> Integer
forall a. Integral a => a -> Integer
toInteger (Int -> Integer) -> (Value -> Int) -> Value -> Integer
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Integer -> Int
cargoTreeCharge (Integer -> Int) -> (Value -> Integer) -> Value -> Int
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Int64 -> Integer
forall a. Integral a => a -> Integer
toInteger (Int64 -> Integer) -> (Value -> Int64) -> Value -> Integer
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Value -> Int64
estimateValueBytes) [Value]
inputs)
retained :: Integer
retained = Int -> Integer
forall a. Integral a => a -> Integer
toInteger (CargoArchiveMetadata -> Int
caRetainedBytes CargoArchiveMetadata
archive) Integer -> Integer -> Integer
forall a. Num a => a -> a -> a
+ Int -> Integer
forall a. Integral a => a -> Integer
toInteger (CargoArchiveMetadata -> Int
caManifestWorkspace CargoArchiveMetadata
archive)
Int -> IO ()
charge (Integer -> Int
saturatingCharge (Integer
retained Integer -> Integer -> Integer
forall a. Num a => a -> a -> a
+ Integer
workspace))
Either ParseError CargoPublishMetadata
-> IO (Either ParseError CargoPublishMetadata)
forall (m :: * -> *) a. MonadIO m => a -> m a
evaluate (Limits
-> CargoPublishMetadata
-> Value
-> Either ParseError CargoPublishMetadata
checkPublicationIndex Limits
limits CargoPublishMetadata
metadata Value
source)
cargoUploadAgreement :: Limits -> (Int -> IO ()) -> Int -> Text -> Value -> CargoArchiveMetadata -> IO (Either ParseError ())
cargoUploadAgreement :: Limits
-> (Int -> IO ())
-> Int
-> Text
-> Value
-> CargoArchiveMetadata
-> IO (Either ParseError ())
cargoUploadAgreement Limits
limits Int -> IO ()
charge Int
uploadWorkspace Text
registry Value
upload CargoArchiveMetadata
metadata = do
Int -> IO ()
charge (Integer -> Int
saturatingCharge ([Integer] -> Integer
forall a (f :: * -> *). (Foldable f, Num a) => f a -> a
sum ((Int -> Integer) -> [Int] -> [Integer]
forall a b. (a -> b) -> [a] -> [b]
map Int -> Integer
forall a. Integral a => a -> Integer
toInteger [CargoArchiveMetadata -> Int
caRetainedBytes CargoArchiveMetadata
metadata, CargoArchiveMetadata -> Int
caManifestWorkspace CargoArchiveMetadata
metadata, Int
uploadWorkspace])))
Either ParseError () -> IO (Either ParseError ())
forall (m :: * -> *) a. MonadIO m => a -> m a
evaluate (Limits -> Text -> CargoManifest -> Value -> Either ParseError ()
checkUploadAgreement Limits
limits Text
registry (CargoArchiveMetadata -> CargoManifest
caManifest CargoArchiveMetadata
metadata) Value
upload)
archiveFacts :: Text -> TarScan -> Either ParseError (ByteString, Set Text, Int)
archiveFacts :: Text -> TarScan -> Either ParseError (ByteString, Set Text, Int)
archiveFacts Text
root TarScan
scan = do
case Text -> Map Text TarMember -> Maybe TarMember
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Text
root (TarScan -> Map Text TarMember
tarMembers TarScan
scan) of
Maybe TarMember
Nothing -> () -> Either ParseError ()
forall a b. b -> Either a b
Right ()
Just TarMember
Directory -> () -> Either ParseError ()
forall a b. b -> Either a b
Right ()
Maybe TarMember
_ -> Text -> Either ParseError ()
forall a. Text -> Either ParseError a
parseRefusal Text
"Cargo archive root is not a directory"
[TarMember]
-> (TarMember -> Either ParseError ()) -> Either ParseError ()
forall (t :: * -> *) (f :: * -> *) a b.
(Foldable t, Applicative f) =>
t a -> (a -> f b) -> f ()
for_ (Map Text TarMember -> [TarMember]
forall k a. Map k a -> [a]
Map.elems (TarScan -> Map Text TarMember
tarMembers TarScan
scan)) ((TarMember -> Either ParseError ()) -> Either ParseError ())
-> (TarMember -> Either ParseError ()) -> Either ParseError ()
forall a b. (a -> b) -> a -> b
$ \case
Link Text
_ -> Text -> Either ParseError ()
forall a. Text -> Either ParseError a
parseRefusal Text
"Cargo packaged archive contains an unflattened link"
TarMember
_ -> () -> Either ParseError ()
forall a b. b -> Either a b
Right ()
let prefix :: Text
prefix = Text
root Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"/"
files :: Set Text
files = [Text] -> Set Text
forall a. Ord a => [a] -> Set a
Set.fromList [Text -> Text
dropArchiveRoot Text
path | (Text
path, TarMember
Regular) <- Map Text TarMember -> [(Text, TarMember)]
forall k a. Map k a -> [(k, a)]
Map.toList (TarScan -> Map Text TarMember
tarMembers TarScan
scan)]
retained :: Int
retained = Int
65536 Int -> Int -> Int
forall a. Num a => a -> a -> a
+ [Int] -> Int
forall a (f :: * -> *). (Foldable f, Num a) => f a -> a
sum [Int
256 Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
4 Int -> Int -> Int
forall a. Num a => a -> a -> a
* ByteString -> Int
BS.length (Text -> ByteString
TE.encodeUtf8 Text
path) | Text
path <- Set Text -> [Text]
forall a. Set a -> [a]
Set.toList Set Text
files]
[Text] -> (Text -> Either ParseError ()) -> Either ParseError ()
forall (t :: * -> *) (f :: * -> *) a b.
(Foldable t, Applicative f) =>
t a -> (a -> f b) -> f ()
for_ (Set Text -> [Text]
forall a. Set a -> [a]
Set.toList Set Text
files) ((Text -> Either ParseError ()) -> Either ParseError ())
-> (Text -> Either ParseError ()) -> Either ParseError ()
forall a b. (a -> b) -> a -> b
$ \Text
path -> do
let prefixes :: [Text]
prefixes = Int -> [Text] -> [Text]
forall a. Int -> [a] -> [a]
drop Int
1 ((Text -> Text -> Text) -> Text -> [Text] -> [Text]
forall b a. (b -> a -> b) -> b -> [a] -> [b]
scanl (\Text
held Text
part -> if Text -> Bool
T.null Text
held then Text
part else Text
held Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"/" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
part) Text
"" (HasCallStack => Text -> Text -> [Text]
Text -> Text -> [Text]
T.splitOn Text
"/" Text
path))
Bool -> Either ParseError () -> Either ParseError ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when ((Text -> Bool) -> [Text] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
any (Text -> Set Text -> Bool
forall a. Ord a => a -> Set a -> Bool
`Set.member` Set Text
files) (Int -> [Text] -> [Text]
forall a. Int -> [a] -> [a]
take ([Text] -> Int
forall a. [a] -> Int
forall (t :: * -> *) a. Foldable t => t a -> Int
length [Text]
prefixes Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1) [Text]
prefixes)) (Text -> Either ParseError ()
forall a. Text -> Either ParseError a
parseRefusal Text
"Cargo archive file traverses another regular file")
manifest <- Either ParseError ByteString
-> (ByteString -> Either ParseError ByteString)
-> Maybe ByteString
-> Either ParseError ByteString
forall b a. b -> (a -> b) -> Maybe a -> b
maybe (Text -> Either ParseError ByteString
forall a. Text -> Either ParseError a
parseRefusal Text
"Cargo archive lacks its effective Cargo.toml") ByteString -> Either ParseError ByteString
forall a b. b -> Either a b
Right (Text -> Map Text ByteString -> Maybe ByteString
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup (Text
prefix Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"Cargo.toml") (TarScan -> Map Text ByteString
tarContents TarScan
scan))
pure (manifest, files, retained)
dropArchiveRoot :: Text -> Text
dropArchiveRoot :: Text -> Text
dropArchiveRoot = Int -> Text -> Text
T.drop Int
1 (Text -> Text) -> (Text -> Text) -> Text -> Text
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Text, Text) -> Text
forall a b. (a, b) -> b
snd ((Text, Text) -> Text) -> (Text -> (Text, Text)) -> Text -> Text
forall b c a. (b -> c) -> (a -> b) -> a -> c
. HasCallStack => Text -> Text -> (Text, Text)
Text -> Text -> (Text, Text)
T.breakOn Text
"/"
archivePass :: Limits -> (Int -> IO ()) -> Text -> Set Text -> ByteString -> IO (Either ParseError TarScan)
archivePass :: Limits
-> (Int -> IO ())
-> Text
-> Set Text
-> ByteString
-> IO (Either ParseError TarScan)
archivePass Limits
limits Int -> IO ()
charge Text
root Set Text
selected ByteString
bytes = do
(decoderCharge, retainedCharge) <- (Int -> IO ()) -> IO (Int -> IO (), Int -> IO ())
newChargePair Int -> IO ()
charge
scanTar limits retainedCharge select $ \ConduitT () ByteString TarDecode () -> IO a
consume ->
InflateFormat
-> (Int -> IO ())
-> ByteString
-> (IO (Either ParseError ByteString) -> IO a)
-> IO a
forall a.
InflateFormat
-> (Int -> IO ())
-> ByteString
-> (IO (Either ParseError ByteString) -> IO a)
-> IO a
withInflatedChunks (Int -> InflateFormat
GzipMembers (Limits -> Int
compressedMemberCeiling Limits
limits)) Int -> IO ()
decoderCharge ByteString
bytes (ConduitT () ByteString TarDecode () -> IO a
consume (ConduitT () ByteString TarDecode () -> IO a)
-> (IO (Either ParseError ByteString)
-> ConduitT () ByteString TarDecode ())
-> IO (Either ParseError ByteString)
-> IO a
forall b c a. (b -> c) -> (a -> b) -> a -> c
. IO (Either ParseError ByteString)
-> ConduitT () ByteString TarDecode ()
sourceReader)
where
select :: [Text] -> Either ParseError Bool
select [Text]
parts = case [Text]
parts of
Text
actualRoot : [Text]
relative | Text
actualRoot Text -> Text -> Bool
forall a. Eq a => a -> a -> Bool
== Text
root -> Bool -> Either ParseError Bool
forall a b. b -> Either a b
Right (Text -> Set Text -> Bool
forall a. Ord a => a -> Set a -> Bool
Set.member (Text -> [Text] -> Text
T.intercalate Text
"/" [Text]
relative) Set Text
selected)
[Text]
_ -> Text -> Either ParseError Bool
forall a. Text -> Either ParseError a
parseRefusal Text
"Cargo archive member disagrees with its admitted package root"
manifestMemory :: ByteString -> Int
manifestMemory :: ByteString -> Int
manifestMemory = Integer -> Int
cargoTreeCharge (Integer -> Int) -> (ByteString -> Integer) -> ByteString -> Int
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Int -> Integer
forall a. Integral a => a -> Integer
toInteger (Int -> Integer) -> (ByteString -> Int) -> ByteString -> Integer
forall b c a. (b -> c) -> (a -> b) -> a -> c
. ByteString -> Int
BS.length