module Ecluse.Core.Registry.RubyGems.Archive (
GemArchive,
inspectGemArchive,
gemArchiveBytes,
gemArchiveName,
gemArchiveSpecification,
gemArchiveMetadata,
gemArchiveInventory,
gemArchiveSha256,
checkGemArchive,
checkRawEnvelope,
) where
import Conduit
import Control.Monad.Except (throwError)
import Data.ByteString qualified as BS
import Data.ByteString.Lazy qualified as BSL
import Data.Map.Strict qualified as Map
import Data.Set qualified as Set
import Data.Text qualified as T
import Text.Libyaml qualified as Y
import Text.Show (showString, shows, showsPrec)
import UnliftIO.Exception (evaluate)
import Ecluse.Core.Package (PackageName)
import Ecluse.Core.Package.Hash (HashAlg (SHA512), computeDigest, hexDigestText, sha256Hex)
import Ecluse.Core.Registry (ParseError (..), parseRefusal)
import Ecluse.Core.Registry.Archive.Inflate (InflateFormat (GzipMembers), inflateBounded, withInflatedChunks)
import Ecluse.Core.Registry.Archive.Tar (TarMember (..), TarScan, scanTar, sourceReader, tarContents, tarMembers, tarRawNames)
import Ecluse.Core.Registry.RubyGems.Archive.Metadata
import Ecluse.Core.Registry.RubyGems.Archive.Yaml
import Ecluse.Core.Registry.RubyGems.CompactSpec (GemSpecification (..), specificationCoordinate)
import Ecluse.Core.Registry.RubyGems.Project
import Ecluse.Core.Security.Limits (Limits (..))
import Ecluse.Core.Server.Admission.Meter (newChargePair)
data GemArchive = GemArchive
{ GemArchive -> ByteString
gemArchiveBytes :: ByteString
, GemArchive -> GemMetadata
gemArchiveMetadata :: GemMetadata
, GemArchive -> Set Text
gemArchiveInventory :: Set Text
, GemArchive -> Text
gemArchiveSha256 :: Text
, GemArchive -> Bool
gemArchiveRawNames :: !Bool
}
deriving stock (GemArchive -> GemArchive -> Bool
(GemArchive -> GemArchive -> Bool)
-> (GemArchive -> GemArchive -> Bool) -> Eq GemArchive
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: GemArchive -> GemArchive -> Bool
== :: GemArchive -> GemArchive -> Bool
$c/= :: GemArchive -> GemArchive -> Bool
/= :: GemArchive -> GemArchive -> Bool
Eq)
instance Show GemArchive where
showsPrec :: Int -> GemArchive -> ShowS
showsPrec Int
_ GemArchive
archive = String -> ShowS
showString String
"GemArchive " ShowS -> ShowS -> ShowS
forall b c a. (b -> c) -> (a -> b) -> a -> c
. PackageName -> ShowS
forall a. Show a => a -> ShowS
shows (GemArchive -> PackageName
gemArchiveName GemArchive
archive)
inspectGemArchive :: Limits -> (Int -> IO ()) -> ByteString -> IO (Either ParseError GemArchive)
inspectGemArchive :: Limits
-> (Int -> IO ())
-> ByteString
-> IO (Either ParseError GemArchive)
inspectGemArchive Limits
limits Int -> IO ()
charge ByteString
bytes
| ByteString -> Int
BS.length ByteString
bytes Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Limits -> Int
maxMirrorArtifactBytes Limits
limits = Either ParseError GemArchive -> IO (Either ParseError GemArchive)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Text -> Either ParseError GemArchive
forall a. Text -> Either ParseError a
parseRefusal Text
"gem archive exceeds its byte bound")
| Bool
otherwise = ExceptT ParseError IO GemArchive
-> IO (Either ParseError GemArchive)
forall e (m :: * -> *) a. ExceptT e m a -> m (Either e a)
runExceptT (ExceptT ParseError IO GemArchive
-> IO (Either ParseError GemArchive))
-> ExceptT ParseError IO GemArchive
-> IO (Either ParseError GemArchive)
forall a b. (a -> b) -> a -> b
$ do
(!rawNames, (metadataGzip, dataGzip, checksumsGzip)) <- IO (Either ParseError (Bool, (ByteString, ByteString, ByteString)))
-> ExceptT
ParseError IO (Bool, (ByteString, ByteString, ByteString))
forall e (m :: * -> *) a. m (Either e a) -> ExceptT e m a
ExceptT (Limits
-> (Int -> IO ())
-> ByteString
-> IO
(Either ParseError (Bool, (ByteString, ByteString, ByteString)))
envelopeMembers Limits
limits Int -> IO ()
charge ByteString
bytes)
let pay Int
extra = Int -> IO ()
charge (ByteString -> Int
BS.length ByteString
bytes Int -> Int -> Int
forall a. Num a => a -> a -> a
+ ByteString -> Int
BS.length ByteString
metadataGzip Int -> Int -> Int
forall a. Num a => a -> a -> a
+ ByteString -> Int
BS.length ByteString
dataGzip Int -> Int -> Int
forall a. Num a => a -> a -> a
+ ByteString -> Int
BS.length ByteString
checksumsGzip Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
extra)
(raw, sums) <- ExceptT (inflateMembers limits pay metadataGzip checksumsGzip)
let nativePay Int
extra = Int -> IO ()
pay (Int
128 Int -> Int -> Int
forall a. Num a => a -> a -> a
* (ByteString -> Int
BS.length ByteString
raw Int -> Int -> Int
forall a. Num a => a -> a -> a
+ ByteString -> Int
BS.length ByteString
sums) Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
extra)
metadata <- ExceptT (decodeMembers limits nativePay metadataGzip dataGzip raw sums)
inventory <- ExceptT (dataInventory limits nativePay (BS.length raw + BS.length sums) dataGzip)
unless (inventory == Set.fromList (metadataFiles metadata)) (throwError (ParseError "gem data inventory disagrees with metadata files"))
digest <- evaluate (sha256Hex bytes)
evaluate (GemArchive bytes metadata inventory digest rawNames)
envelopeMembers :: Limits -> (Int -> IO ()) -> ByteString -> IO (Either ParseError (Bool, (ByteString, ByteString, ByteString)))
envelopeMembers :: Limits
-> (Int -> IO ())
-> ByteString
-> IO
(Either ParseError (Bool, (ByteString, ByteString, ByteString)))
envelopeMembers Limits
limits Int -> IO ()
charge ByteString
bytes = (Either ParseError TarScan
-> (TarScan
-> Either ParseError (Bool, (ByteString, ByteString, ByteString)))
-> Either ParseError (Bool, (ByteString, ByteString, ByteString))
forall a b.
Either ParseError a
-> (a -> Either ParseError b) -> Either ParseError b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= \TarScan
scan -> (TarScan -> Bool
tarRawNames TarScan
scan,) ((ByteString, ByteString, ByteString)
-> (Bool, (ByteString, ByteString, ByteString)))
-> Either ParseError (ByteString, ByteString, ByteString)
-> Either ParseError (Bool, (ByteString, ByteString, ByteString))
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> TarScan -> Either ParseError (ByteString, ByteString, ByteString)
envelope TarScan
scan) (Either ParseError TarScan
-> Either ParseError (Bool, (ByteString, ByteString, ByteString)))
-> IO (Either ParseError TarScan)
-> IO
(Either ParseError (Bool, (ByteString, ByteString, ByteString)))
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> Limits
-> (Int -> IO ())
-> ([Text] -> Either ParseError Bool)
-> (forall a.
(ConduitT () ByteString (ExceptT ParseError IO) () -> IO a)
-> IO a)
-> IO (Either ParseError TarScan)
scanTar Limits
limits{maxMetadataBytes = maxMirrorArtifactBytes limits} Int -> IO ()
charge (\[Text]
parts -> Bool -> Either ParseError Bool
forall a b. b -> Either a b
Right ([Text]
parts [Text] -> [[Text]] -> Bool
forall (f :: * -> *) a.
(Foldable f, DisallowElem f, Eq a) =>
a -> f a -> Bool
`elem` [[Text
"metadata.gz"], [Text
"data.tar.gz"], [Text
"checksums.yaml.gz"]])) (\ConduitT () ByteString (ExceptT ParseError IO) () -> IO a
consume -> ConduitT () ByteString (ExceptT ParseError IO) () -> IO a
consume (ByteString -> ConduitT () ByteString (ExceptT ParseError IO) ()
forall (m :: * -> *) o i. Monad m => o -> ConduitT i o m ()
yield ByteString
bytes))
envelope :: TarScan -> Either ParseError (ByteString, ByteString, ByteString)
envelope :: TarScan -> Either ParseError (ByteString, ByteString, ByteString)
envelope TarScan
scan = do
let members :: Map Text TarMember
members = TarScan -> Map Text TarMember
tarMembers TarScan
scan
mandatory :: [Text]
mandatory = [Text
"metadata.gz", Text
"data.tar.gz", Text
"checksums.yaml.gz"]
allowed :: [Text]
allowed = [Text]
mandatory [Text] -> [Text] -> [Text]
forall a. Semigroup a => a -> a -> a
<> (Text -> Text) -> [Text] -> [Text]
forall a b. (a -> b) -> [a] -> [b]
map (Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
".sig") [Text]
mandatory
Bool -> Either ParseError () -> Either ParseError ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
unless ((Text -> Bool) -> [Text] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
all (Text -> [Text] -> Bool
forall (f :: * -> *) a.
(Foldable f, DisallowElem f, Eq a) =>
a -> f a -> Bool
`elem` [Text]
allowed) (Map Text TarMember -> [Text]
forall k a. Map k a -> [k]
Map.keys Map Text TarMember
members) Bool -> Bool -> Bool
&& (TarMember -> Bool) -> [TarMember] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
all (TarMember -> TarMember -> Bool
forall a. Eq a => a -> a -> Bool
== TarMember
Regular) (Map Text TarMember -> [TarMember]
forall k a. Map k a -> [a]
Map.elems Map Text TarMember
members)) (Text -> Either ParseError ()
forall a. Text -> Either ParseError a
parseRefusal Text
"unsupported gem envelope member")
(Text -> Either ParseError ()) -> [Text] -> Either ParseError ()
forall (t :: * -> *) (f :: * -> *) a b.
(Foldable t, Applicative f) =>
(a -> f b) -> t a -> f ()
traverse_ (\Text
name -> Bool -> Either ParseError () -> Either ParseError ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
unless (Text -> Map Text TarMember -> Bool
forall k a. Ord k => k -> Map k a -> Bool
Map.member Text
name Map Text TarMember
members) (Text -> Either ParseError ()
forall a. Text -> Either ParseError a
parseRefusal Text
"missing required gem envelope member")) [Text]
mandatory
let content :: Text -> Either ParseError ByteString
content Text
name = ParseError -> Maybe ByteString -> Either ParseError ByteString
forall l r. l -> Maybe r -> Either l r
maybeToRight (Text -> ParseError
ParseError Text
"missing gem member payload") (Text -> Map Text ByteString -> Maybe ByteString
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Text
name (TarScan -> Map Text ByteString
tarContents TarScan
scan))
(,,) (ByteString
-> ByteString
-> ByteString
-> (ByteString, ByteString, ByteString))
-> Either ParseError ByteString
-> Either
ParseError
(ByteString -> ByteString -> (ByteString, ByteString, ByteString))
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> Text -> Either ParseError ByteString
content Text
"metadata.gz" Either
ParseError
(ByteString -> ByteString -> (ByteString, ByteString, ByteString))
-> Either ParseError ByteString
-> Either
ParseError (ByteString -> (ByteString, ByteString, ByteString))
forall a b.
Either ParseError (a -> b)
-> Either ParseError a -> Either ParseError b
forall (f :: * -> *) a b. Applicative f => f (a -> b) -> f a -> f b
<*> Text -> Either ParseError ByteString
content Text
"data.tar.gz" Either
ParseError (ByteString -> (ByteString, ByteString, ByteString))
-> Either ParseError ByteString
-> Either ParseError (ByteString, ByteString, ByteString)
forall a b.
Either ParseError (a -> b)
-> Either ParseError a -> Either ParseError b
forall (f :: * -> *) a b. Applicative f => f (a -> b) -> f a -> f b
<*> Text -> Either ParseError ByteString
content Text
"checksums.yaml.gz"
inflateMembers :: Limits -> (Int -> IO ()) -> ByteString -> ByteString -> IO (Either ParseError (ByteString, ByteString))
inflateMembers :: Limits
-> (Int -> IO ())
-> ByteString
-> ByteString
-> IO (Either ParseError (ByteString, ByteString))
inflateMembers Limits
limits Int -> IO ()
pay ByteString
metadataGzip ByteString
checksumsGzip = do
metadataBytes <- (Int -> IO ()) -> ByteString -> IO (Either ParseError ByteString)
inflate Int -> IO ()
pay ByteString
metadataGzip
checksumBytes <- inflate (\Int
extra -> Int -> IO ()
pay ((ParseError -> Int)
-> (ByteString -> Int) -> Either ParseError ByteString -> Int
forall a c b. (a -> c) -> (b -> c) -> Either a b -> c
either (Int -> ParseError -> Int
forall a b. a -> b -> a
const Int
0) ((Int
3 Int -> Int -> Int
forall a. Num a => a -> a -> a
*) (Int -> Int) -> (ByteString -> Int) -> ByteString -> Int
forall b c a. (b -> c) -> (a -> b) -> a -> c
. ByteString -> Int
BS.length) Either ParseError ByteString
metadataBytes Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
extra)) checksumsGzip
pure ((,) <$> metadataBytes <*> checksumBytes)
where
inflate :: (Int -> IO ()) -> ByteString -> IO (Either ParseError ByteString)
inflate = InflateFormat
-> Text
-> Int
-> (Int -> IO ())
-> ByteString
-> IO (Either ParseError ByteString)
inflateBounded (Int -> InflateFormat
GzipMembers Int
1) Text
"gem compressed metadata exceeds its inflated byte bound" (Limits -> Int
maxMetadataBytes Limits
limits)
decodeMembers :: Limits -> (Int -> IO ()) -> ByteString -> ByteString -> ByteString -> ByteString -> IO (Either ParseError GemMetadata)
decodeMembers :: Limits
-> (Int -> IO ())
-> ByteString
-> ByteString
-> ByteString
-> ByteString
-> IO (Either ParseError GemMetadata)
decodeMembers Limits
limits Int -> IO ()
nativePay ByteString
metadataGzip ByteString
dataGzip ByteString
raw ByteString
sums = do
parsed <- Limits
-> (Int -> IO ()) -> ByteString -> IO (Either ParseError YamlValue)
decodeYaml Limits
limits Int -> IO ()
nativePay ByteString
raw
checksums <- decodeYaml limits nativePay sums
pure ((parsed >>= parseGemMetadata) <* (checksums >>= verifyChecksums metadataGzip dataGzip))
dataInventory :: Limits -> (Int -> IO ()) -> Int -> ByteString -> IO (Either ParseError (Set Text))
dataInventory :: Limits
-> (Int -> IO ())
-> Int
-> ByteString
-> IO (Either ParseError (Set Text))
dataInventory Limits
limits Int -> IO ()
nativePay Int
decoded ByteString
dataGzip = do
(decoderCharge, treeCharge) <- (Int -> IO ()) -> IO (Int -> IO (), Int -> IO ())
newChargePair Int -> IO ()
nativePay
let innerLimits = Limits
limits{maxMirrorArtifactBytes = maxMirrorArtifactBytes limits - decoded}
inner <- scanTar innerLimits treeCharge (const (Right False)) (\ConduitT () ByteString (ExceptT ParseError IO) () -> IO a
consume -> InflateFormat
-> (Int -> IO ())
-> ByteString
-> (IO (Either ParseError ByteString) -> IO a)
-> IO a
forall a.
InflateFormat
-> (Int -> IO ())
-> ByteString
-> (IO (Either ParseError ByteString) -> IO a)
-> IO a
withInflatedChunks (Int -> InflateFormat
GzipMembers Int
1) Int -> IO ()
decoderCharge ByteString
dataGzip (ConduitT () ByteString (ExceptT ParseError IO) () -> IO a
consume (ConduitT () ByteString (ExceptT ParseError IO) () -> IO a)
-> (IO (Either ParseError ByteString)
-> ConduitT () ByteString (ExceptT ParseError IO) ())
-> IO (Either ParseError ByteString)
-> IO a
forall b c a. (b -> c) -> (a -> b) -> a -> c
. IO (Either ParseError ByteString)
-> ConduitT () ByteString (ExceptT ParseError IO) ()
sourceReader))
pure (Map.keysSet . Map.filter (/= Directory) . tarMembers <$> inner)
verifyChecksums :: ByteString -> ByteString -> YamlValue -> Either ParseError ()
verifyChecksums :: ByteString -> ByteString -> YamlValue -> Either ParseError ()
verifyChecksums ByteString
metadataBytes ByteString
dataBytes YamlValue
value = do
YamlValue -> Either ParseError ()
plainMapping YamlValue
value
algorithms <- YamlValue -> Either ParseError (Map Text YamlValue)
yamlMapping YamlValue
value
unless (Map.member "SHA256" algorithms && all (`elem` ["SHA256", "SHA512"]) (Map.keys algorithms)) (parseRefusal "missing or unsupported gem checksum algorithm")
for_ (Map.toList algorithms) $ \(Text
algorithm, YamlValue
entries) -> do
YamlValue -> Either ParseError ()
plainMapping YamlValue
entries
digests <- YamlValue -> Either ParseError (Map Text YamlValue)
yamlMapping YamlValue
entries
unless (Map.keys digests == ["data.tar.gz", "metadata.gz"]) (parseRefusal "gem checksum member inventory disagrees")
for_ [("metadata.gz", metadataBytes), ("data.tar.gz", dataBytes)] $ \(Text
name, ByteString
bytes) -> do
expected <- Text -> Map Text YamlValue -> Either ParseError YamlValue
yamlField Text
name Map Text YamlValue
digests Either ParseError YamlValue
-> (YamlValue -> Either ParseError Text) -> Either ParseError Text
forall a b.
Either ParseError a
-> (a -> Either ParseError b) -> Either ParseError b
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= YamlValue -> Either ParseError Text
yamlText
digest <- maybeToRight (ParseError "SHA512 computation is unavailable") (computeDigest SHA512)
let actual = if Text
algorithm Text -> Text -> Bool
forall a. Eq a => a -> a -> Bool
== Text
"SHA256" then ByteString -> Text
sha256Hex ByteString
bytes else ByteString -> Text
hexDigestText (LByteString -> ByteString
digest (ByteString -> LByteString
BSL.fromStrict ByteString
bytes))
unless (T.toLower expected == actual) (parseRefusal "gem member checksum disagrees")
plainMapping :: YamlValue -> Either ParseError ()
plainMapping :: YamlValue -> Either ParseError ()
plainMapping YamlValue
value = Bool -> Either ParseError () -> Either ParseError ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
unless (YamlValue -> Tag
yamlTag YamlValue
value Tag -> [Tag] -> Bool
forall (f :: * -> *) a.
(Foldable f, DisallowElem f, Eq a) =>
a -> f a -> Bool
`elem` [Tag
Y.NoTag, Tag
Y.MapTag]) (Text -> Either ParseError ()
forall a. Text -> Either ParseError a
parseRefusal Text
"gem checksum mapping has an unsupported native tag")
gemArchiveSpecification :: GemArchive -> GemSpecification
gemArchiveSpecification :: GemArchive -> GemSpecification
gemArchiveSpecification = GemMetadata -> GemSpecification
metadataSpecification (GemMetadata -> GemSpecification)
-> (GemArchive -> GemMetadata) -> GemArchive -> GemSpecification
forall b c a. (b -> c) -> (a -> b) -> a -> c
. GemArchive -> GemMetadata
gemArchiveMetadata
gemArchiveName :: GemArchive -> PackageName
gemArchiveName :: GemArchive -> PackageName
gemArchiveName = GemSpecification -> PackageName
specificationName (GemSpecification -> PackageName)
-> (GemArchive -> GemSpecification) -> GemArchive -> PackageName
forall b c a. (b -> c) -> (a -> b) -> a -> c
. GemArchive -> GemSpecification
gemArchiveSpecification
checkGemArchive :: GemCoordinate -> Text -> GemArchive -> Either ParseError GemArchive
checkGemArchive :: GemCoordinate -> Text -> GemArchive -> Either ParseError GemArchive
checkGemArchive GemCoordinate
coordinate Text
checksum GemArchive
archive = do
actual <- GemSpecification -> Either ParseError GemCoordinate
specificationCoordinate (GemArchive -> GemSpecification
gemArchiveSpecification GemArchive
archive)
unless (actual == coordinate) (parseRefusal "gem archive identity disagrees with its admitted coordinate")
unless (T.toLower checksum == gemArchiveSha256 archive) (parseRefusal "gem archive disagrees with its admitted SHA-256")
pure archive
checkRawEnvelope :: GemArchive -> Either ParseError GemArchive
checkRawEnvelope :: GemArchive -> Either ParseError GemArchive
checkRawEnvelope GemArchive
archive = do
Bool -> Either ParseError () -> Either ParseError ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
unless (GemArchive -> Bool
gemArchiveRawNames GemArchive
archive) (Text -> Either ParseError ()
forall a. Text -> Either ParseError a
parseRefusal Text
"gem envelope carries an extended or prefixed tar header")
GemArchive -> Either ParseError GemArchive
forall a. a -> Either ParseError a
forall (f :: * -> *) a. Applicative f => a -> f a
pure GemArchive
archive