module Ecluse.Core.Registry.RubyGems.Directory.Internal (
GemDirectory (..),
DirectoryRead (..),
directoryNameCeiling,
directoryStepsWithin,
applyChanges,
) where
import Data.Aeson (Value (Null))
import Data.ByteString qualified as BS
import Data.ByteString.Char8 qualified as B8
import Data.Map.Strict qualified as Map
import Data.Set qualified as Set
import Data.Text.Encoding qualified as TE
import Ecluse.Core.Package (InvalidEntry, InvalidEntryKind (InvalidVersionListing), mkInvalidEntry)
import Ecluse.Core.Package.Entry (EntryKey (RecordEntry))
import Ecluse.Core.Registry.JsonStream (Steps (..))
import Ecluse.Core.Registry.Metadata.RecordStream (recordFragmentCharge, recordSteps)
import Ecluse.Core.Registry.RubyGems.Project (projectName)
import Ecluse.Core.Security (LimitError (TooManyVersions), withinCeiling)
import Ecluse.Core.Server.MemoryModel (decodedDocumentCharge, saturatingCharge)
import Ecluse.Core.Strict (strictElements)
newtype GemDirectory = GemDirectory {GemDirectory -> ByteString
directoryBytes :: ByteString}
deriving stock (GemDirectory -> GemDirectory -> Bool
(GemDirectory -> GemDirectory -> Bool)
-> (GemDirectory -> GemDirectory -> Bool) -> Eq GemDirectory
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: GemDirectory -> GemDirectory -> Bool
== :: GemDirectory -> GemDirectory -> Bool
$c/= :: GemDirectory -> GemDirectory -> Bool
/= :: GemDirectory -> GemDirectory -> Bool
Eq, Int -> GemDirectory -> ShowS
[GemDirectory] -> ShowS
GemDirectory -> String
(Int -> GemDirectory -> ShowS)
-> (GemDirectory -> String)
-> ([GemDirectory] -> ShowS)
-> Show GemDirectory
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> GemDirectory -> ShowS
showsPrec :: Int -> GemDirectory -> ShowS
$cshow :: GemDirectory -> String
show :: GemDirectory -> String
$cshowList :: [GemDirectory] -> ShowS
showList :: [GemDirectory] -> ShowS
Show)
data DirectoryRead = DirectoryRead {DirectoryRead -> GemDirectory
readDirectory :: GemDirectory, DirectoryRead -> [InvalidEntry]
readDropped :: [InvalidEntry]}
deriving stock (DirectoryRead -> DirectoryRead -> Bool
(DirectoryRead -> DirectoryRead -> Bool)
-> (DirectoryRead -> DirectoryRead -> Bool) -> Eq DirectoryRead
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: DirectoryRead -> DirectoryRead -> Bool
== :: DirectoryRead -> DirectoryRead -> Bool
$c/= :: DirectoryRead -> DirectoryRead -> Bool
/= :: DirectoryRead -> DirectoryRead -> Bool
Eq, Int -> DirectoryRead -> ShowS
[DirectoryRead] -> ShowS
DirectoryRead -> String
(Int -> DirectoryRead -> ShowS)
-> (DirectoryRead -> String)
-> ([DirectoryRead] -> ShowS)
-> Show DirectoryRead
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> DirectoryRead -> ShowS
showsPrec :: Int -> DirectoryRead -> ShowS
$cshow :: DirectoryRead -> String
show :: DirectoryRead -> String
$cshowList :: [DirectoryRead] -> ShowS
showList :: [DirectoryRead] -> ShowS
Show)
directoryNameCeiling :: Int
directoryNameCeiling :: Int
directoryNameCeiling = Int
2_000_000
data Scan = Scan
{ :: Bool
, Scan -> Int
scanOrdinal :: Int
, Scan -> Int
scanRetained :: Int
, Scan -> Int
scanWorkspace :: Int
, Scan -> [InvalidEntry]
scanDropped :: [InvalidEntry]
, Scan -> Int
scanDroppedCharge :: Int
, Scan -> Map ByteString ByteString
scanRows :: Map ByteString ByteString
}
directoryStepsWithin :: (Monad m) => Int -> (Int -> m ()) -> Steps m DirectoryRead
directoryStepsWithin :: forall (m :: * -> *).
Monad m =>
Int -> (Int -> m ()) -> Steps m DirectoryRead
directoryStepsWithin Int
nameCeiling Int -> m ()
pay = (Scan -> Int -> ByteString -> m Int)
-> (Scan -> Int -> ByteString -> Either LimitError Scan)
-> (Scan -> Steps m DirectoryRead)
-> Scan
-> Steps m DirectoryRead
forall (m :: * -> *) s a.
Monad m =>
(s -> Int -> ByteString -> m Int)
-> (s -> Int -> ByteString -> Either LimitError s)
-> (s -> Steps m a)
-> s
-> Steps m a
recordSteps Scan -> Int -> ByteString -> m Int
reserve Scan -> Int -> ByteString -> Either LimitError Scan
forall {p}. Scan -> p -> ByteString -> Either LimitError Scan
consume Scan -> Steps m DirectoryRead
forall {m :: * -> *}. Scan -> Steps m DirectoryRead
complete (Bool
-> Int
-> Int
-> Int
-> [InvalidEntry]
-> Int
-> Map ByteString ByteString
-> Scan
Scan Bool
False Int
0 Int
0 Int
0 [] Int
0 Map ByteString ByteString
forall k a. Map k a
Map.empty)
where
reserve :: Scan -> Int -> ByteString -> m Int
reserve Scan
scan Int
previous ByteString
fragment = do
let cost :: Int
cost = Int -> ByteString -> Int
recordFragmentCharge Int
previous ByteString
fragment
Int -> m ()
pay (Integer -> Int
saturatingCharge (Integer
2 Integer -> Integer -> Integer
forall a. Num a => a -> a -> a
* Int -> Integer
forall a. Integral a => a -> Integer
toInteger (Scan -> Int
scanRetained Scan
scan) Integer -> Integer -> Integer
forall a. Num a => a -> a -> a
+ Int -> Integer
forall a. Integral a => a -> Integer
toInteger (Scan -> Int
scanWorkspace Scan
scan) Integer -> Integer -> Integer
forall a. Num a => a -> a -> a
+ Int -> Integer
forall a. Integral a => a -> Integer
toInteger (Scan -> Int
scanDroppedCharge Scan
scan) Integer -> Integer -> Integer
forall a. Num a => a -> a -> a
+ Int -> Integer
forall a. Integral a => a -> Integer
toInteger Int
cost))
Int -> m Int
forall a. a -> m a
forall (f :: * -> *) a. Applicative f => a -> f a
pure Int
cost
consume :: Scan -> p -> ByteString -> Either LimitError Scan
consume Scan
scan p
_ ByteString
bytes
| Bool -> Bool
not (Scan -> Bool
scanHeaderDone Scan
scan) = Scan -> Either LimitError Scan
forall a b. b -> Either a b
Right Scan
next{scanHeaderDone = bytes == "---"}
| ByteString -> Bool
BS.null ByteString
bytes = Scan -> Either LimitError Scan
forall a b. b -> Either a b
Right Scan
next
| Bool
otherwise = case ByteString -> Maybe (ByteString, ByteString)
decodeRow ByteString
bytes of
Maybe (ByteString, ByteString)
Nothing -> Scan -> Either LimitError Scan
forall a b. b -> Either a b
Right Scan
next{scanDropped = droppedRow (scanOrdinal scan) : scanDropped scan, scanDroppedCharge = scanDroppedCharge scan + 256}
Just (ByteString
name, ByteString
changes) -> do
let previous :: Maybe ByteString
previous = ByteString -> Map ByteString ByteString -> Maybe ByteString
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup ByteString
name (Scan -> Map ByteString ByteString
scanRows Scan
scan)
versions :: ByteString
versions = ByteString
-> (ByteString -> ByteString) -> Maybe ByteString -> ByteString
forall b a. b -> (a -> b) -> Maybe a -> b
maybe (ByteString -> ByteString
initialVersions ByteString
changes) (ByteString -> ByteString -> ByteString
`applyChanges` ByteString
changes) Maybe ByteString
previous
rows :: Map ByteString ByteString
rows = ByteString
-> ByteString
-> Map ByteString ByteString
-> Map ByteString ByteString
forall k a. Ord k => k -> a -> Map k a -> Map k a
Map.insert (ByteString -> ByteString
BS.copy ByteString
name) ByteString
versions (Scan -> Map ByteString ByteString
scanRows Scan
scan)
weight :: Int
weight = Scan -> Int
scanRetained Scan
scan Int -> Int -> Int
forall a. Num a => a -> a -> a
+ ByteString -> Int
BS.length ByteString
name Int -> Int -> Int
forall a. Num a => a -> a -> a
+ ByteString -> Int
BS.length ByteString
versions Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
256 Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int -> (ByteString -> Int) -> Maybe ByteString -> Int
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Int
0 (\ByteString
old -> ByteString -> Int
BS.length ByteString
name Int -> Int -> Int
forall a. Num a => a -> a -> a
+ ByteString -> Int
BS.length ByteString
old Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
256) Maybe ByteString
previous
(Int -> Int -> LimitError) -> Int -> Int -> Either LimitError ()
withinCeiling Int -> Int -> LimitError
TooManyVersions Int
nameCeiling (Map ByteString ByteString -> Int
forall k a. Map k a -> Int
Map.size Map ByteString ByteString
rows)
Scan -> Either LimitError Scan
forall a b. b -> Either a b
Right Scan
next{scanRetained = weight, scanWorkspace = max (scanWorkspace scan) (decodedDocumentCharge versions), scanRows = rows}
where
next :: Scan
next = Scan
scan{scanOrdinal = scanOrdinal scan + 1}
complete :: Scan -> Steps m DirectoryRead
complete Scan
scan
| Bool -> Bool
not (Scan -> Bool
scanHeaderDone Scan
scan) = Text -> Steps m DirectoryRead
forall (m :: * -> *) s. Text -> Steps m s
Failed Text
"RubyGems directory has no header delimiter"
| Bool
otherwise = DirectoryRead -> Steps m DirectoryRead
forall (m :: * -> *) s. s -> Steps m s
Finished (GemDirectory -> [InvalidEntry] -> DirectoryRead
DirectoryRead (Map ByteString ByteString -> GemDirectory
packRows (Scan -> Map ByteString ByteString
scanRows Scan
scan)) ([InvalidEntry] -> [InvalidEntry]
forall (t :: * -> *) a. Foldable t => t a -> t a
strictElements ([InvalidEntry] -> [InvalidEntry]
forall a. [a] -> [a]
reverse (Scan -> [InvalidEntry]
scanDropped Scan
scan))))
packRows :: Map ByteString ByteString -> GemDirectory
packRows :: Map ByteString ByteString -> GemDirectory
packRows Map ByteString ByteString
rows = ByteString -> GemDirectory
GemDirectory ([ByteString] -> ByteString
BS.concat [ByteString
name ByteString -> ByteString -> ByteString
forall a. Semigroup a => a -> a -> a
<> ByteString
" " ByteString -> ByteString -> ByteString
forall a. Semigroup a => a -> a -> a
<> ByteString
versions ByteString -> ByteString -> ByteString
forall a. Semigroup a => a -> a -> a
<> ByteString
" \n" | (ByteString
name, ByteString
versions) <- Map ByteString ByteString -> [(ByteString, ByteString)]
forall k a. Map k a -> [(k, a)]
Map.toAscList Map ByteString ByteString
rows, Bool -> Bool
not (ByteString -> Bool
BS.null ByteString
versions)])
droppedRow :: Int -> InvalidEntry
droppedRow :: Int -> InvalidEntry
droppedRow Int
ordinal = InvalidEntryKind -> Text -> Value -> Text -> InvalidEntry
mkInvalidEntry InvalidEntryKind
InvalidVersionListing (EntryKey -> Text
forall b a. (Show a, IsString b) => a -> b
show (Int -> EntryKey
RecordEntry Int
ordinal)) Value
Null Text
"malformed RubyGems directory row"
decodeRow :: ByteString -> Maybe (ByteString, ByteString)
decodeRow :: ByteString -> Maybe (ByteString, ByteString)
decodeRow ByteString
bytes = do
firstSpace <- Word8 -> ByteString -> Maybe Int
BS.elemIndex Word8
32 ByteString
bytes
let name = Int -> ByteString -> ByteString
BS.take Int
firstSpace ByteString
bytes
rest = Int -> ByteString -> ByteString
BS.drop (Int
firstSpace Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) ByteString
bytes
lastSpace <- BS.elemIndex 32 rest
let versions = Int -> ByteString -> ByteString
BS.take Int
lastSpace ByteString
rest
hint = Int -> ByteString -> ByteString
BS.drop (Int
lastSpace Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) ByteString
rest
text <- rightToMaybe (TE.decodeUtf8' name)
_ <- rightToMaybe (projectName text)
guard (BS.null hint || (BS.length hint == 32 && BS.all hexadecimal hint))
guard (not (BS.null versions) && all validToken (B8.split ',' versions))
pure (name, versions)
where
hexadecimal :: a -> Bool
hexadecimal a
c = (a
c a -> a -> Bool
forall a. Ord a => a -> a -> Bool
>= a
48 Bool -> Bool -> Bool
&& a
c a -> a -> Bool
forall a. Ord a => a -> a -> Bool
<= a
57) Bool -> Bool -> Bool
|| (a
c a -> a -> Bool
forall a. Ord a => a -> a -> Bool
>= a
97 Bool -> Bool -> Bool
&& a
c a -> a -> Bool
forall a. Ord a => a -> a -> Bool
<= a
102) Bool -> Bool -> Bool
|| (a
c a -> a -> Bool
forall a. Ord a => a -> a -> Bool
>= a
65 Bool -> Bool -> Bool
&& a
c a -> a -> Bool
forall a. Ord a => a -> a -> Bool
<= a
70)
validToken :: ByteString -> Bool
validToken ByteString
token = Bool -> Bool
not (ByteString -> Bool
BS.null ByteString
token) Bool -> Bool -> Bool
&& (Word8 -> Bool) -> ByteString -> Bool
BS.all (\Word8
c -> (Word8
c Word8 -> Word8 -> Bool
forall a. Ord a => a -> a -> Bool
>= Word8
48 Bool -> Bool -> Bool
&& Word8
c Word8 -> Word8 -> Bool
forall a. Ord a => a -> a -> Bool
<= Word8
57) Bool -> Bool -> Bool
|| (Word8
c Word8 -> Word8 -> Bool
forall a. Ord a => a -> a -> Bool
>= Word8
65 Bool -> Bool -> Bool
&& Word8
c Word8 -> Word8 -> Bool
forall a. Ord a => a -> a -> Bool
<= Word8
90) Bool -> Bool -> Bool
|| (Word8
c Word8 -> Word8 -> Bool
forall a. Ord a => a -> a -> Bool
>= Word8
97 Bool -> Bool -> Bool
&& Word8
c Word8 -> Word8 -> Bool
forall a. Ord a => a -> a -> Bool
<= Word8
122) Bool -> Bool -> Bool
|| Word8
c Word8 -> Word8 -> Bool
forall a. Eq a => a -> a -> Bool
== Word8
45 Bool -> Bool -> Bool
|| Word8
c Word8 -> Word8 -> Bool
forall a. Eq a => a -> a -> Bool
== Word8
46 Bool -> Bool -> Bool
|| Word8
c Word8 -> Word8 -> Bool
forall a. Eq a => a -> a -> Bool
== Word8
95) ByteString
token Bool -> Bool -> Bool
&& ByteString
token ByteString -> ByteString -> Bool
forall a. Eq a => a -> a -> Bool
/= ByteString
"-"
initialVersions :: ByteString -> ByteString
initialVersions :: ByteString -> ByteString
initialVersions ByteString
changes
| Int -> ByteString -> ByteString
BS.take Int
1 ByteString
changes ByteString -> ByteString -> Bool
forall a. Eq a => a -> a -> Bool
== ByteString
"-" Bool -> Bool -> Bool
|| Bool -> Bool
not (ByteString -> Bool
BS.null ((ByteString, ByteString) -> ByteString
forall a b. (a, b) -> b
snd (ByteString -> ByteString -> (ByteString, ByteString)
BS.breakSubstring ByteString
",-" ByteString
changes))) = ByteString -> ByteString -> ByteString
applyChanges ByteString
"" ByteString
changes
| Bool
otherwise = ByteString -> ByteString
BS.copy ByteString
changes
applyChanges :: ByteString -> ByteString -> ByteString
applyChanges :: ByteString -> ByteString -> ByteString
applyChanges ByteString
previous ByteString
changes = ByteString -> [ByteString] -> ByteString
B8.intercalate ByteString
"," (Set ByteString -> [ByteString]
forall a. Set a -> [a]
Set.toAscList ((Set ByteString -> ByteString -> Set ByteString)
-> Set ByteString -> [ByteString] -> Set ByteString
forall b a. (b -> a -> b) -> b -> [a] -> b
forall (t :: * -> *) b a.
Foldable t =>
(b -> a -> b) -> b -> t a -> b
foldl' Set ByteString -> ByteString -> Set ByteString
apply Set ByteString
initial (Char -> ByteString -> [ByteString]
B8.split Char
',' ByteString
changes)))
where
initial :: Set ByteString
initial = if ByteString -> Bool
BS.null ByteString
previous then Set ByteString
forall a. Set a
Set.empty else [ByteString] -> Set ByteString
forall a. Ord a => [a] -> Set a
Set.fromList (Char -> ByteString -> [ByteString]
B8.split Char
',' ByteString
previous)
apply :: Set ByteString -> ByteString -> Set ByteString
apply Set ByteString
seen ByteString
token = case ByteString -> Maybe (Word8, ByteString)
BS.uncons ByteString
token of
Just (Word8
45, ByteString
removed) -> ByteString -> Set ByteString -> Set ByteString
forall a. Ord a => a -> Set a -> Set a
Set.delete ByteString
removed Set ByteString
seen
Maybe (Word8, ByteString)
_ -> ByteString -> Set ByteString -> Set ByteString
forall a. Ord a => a -> Set a -> Set a
Set.insert (ByteString -> ByteString
BS.copy ByteString
token) Set ByteString
seen