module Ecluse.Core.Registry.PyPI.StreamingProjection (
PyPIProjection,
emptyProjection,
collectField,
keepsFile,
claimsEnvelope,
retainedEntries,
finishProjection,
packedFileStep,
finishPackedProjection,
) where
import Control.Monad.ST (ST)
import Data.Aeson (Value)
import Data.Aeson.Key qualified as Key
import Data.Aeson.KeyMap qualified as KeyMap
import Data.Aeson.Types (parseEither)
import Ecluse.Core.Package (InvalidEntry, InvalidEntryKind (InvalidVersionListing), PackageInfo, PackageName, mkInvalidEntry)
import Ecluse.Core.Package.Entry (EntryKey (ArrayEntry))
import Ecluse.Core.Registry.Json.Packed (DocTable, Packed)
import Ecluse.Core.Registry.Json.Writer (Writer, decodeWhole, discard, sealValue)
import Ecluse.Core.Registry.Metadata (MetadataError (MetadataUndecodable))
import Ecluse.Core.Registry.Metadata.EntryFloor (RetainedEntries (RetainedEntries))
import Ecluse.Core.Registry.Metadata.Projection (validateNameAgreement)
import Ecluse.Core.Registry.PyPI.Document (PackedSimple, SimpleDocument, packedSimple, simpleDocument, urlHole)
import Ecluse.Core.Registry.PyPI.FileWriter (FileValue (..))
import Ecluse.Core.Registry.PyPI.Project (FilenameMemo, ProjectedFile, filenameMemo, projectFile, projectName, projectSimpleIndex, projectedEntryKey, readCoordinate)
import Ecluse.Core.Registry.PyPI.Streaming (PyPIField (..))
import Ecluse.Core.Registry.PyPI.Wire (IndexFile (ifEntryKey, ifFilename, ifUrl), checkApiVersion, decodeIndexFiles)
import Ecluse.Core.Registry.ServedDocument (artifactUrlPacked)
import Ecluse.Core.Security (LimitError, Limits, checkVersionCountOf)
data PyPIProjection = PyPIProjection
{ PyPIProjection -> KeyMap Value
projectedEnvelope :: KeyMap.KeyMap Value
, PyPIProjection -> [(ProjectedFile, FilePayload)]
projectedFiles :: [(ProjectedFile, FilePayload)]
, PyPIProjection -> FilenameMemo
projectedMemo :: FilenameMemo
, PyPIProjection -> [InvalidEntry]
projectedFileDrops :: [InvalidEntry]
, PyPIProjection -> [InvalidEntry]
projectedVersionDrops :: [InvalidEntry]
, PyPIProjection -> Int
projectedReleases :: Int
, PyPIProjection -> Int
projectedOthers :: Int
, PyPIProjection -> Bool
projectedShape :: Bool
, PyPIProjection -> Bool
projectedFilesSeen :: Bool
, PyPIProjection -> Bool
projectedFilesActive :: Bool
, PyPIProjection -> Bool
projectedVersionsSeen :: Bool
, PyPIProjection -> Bool
projectedVersionsActive :: Bool
}
data FilePayload = TreeFile Value | PackedFile Packed
emptyProjection :: PackageName -> PyPIProjection
emptyProjection :: PackageName -> PyPIProjection
emptyProjection PackageName
name = KeyMap Value
-> [(ProjectedFile, FilePayload)]
-> FilenameMemo
-> [InvalidEntry]
-> [InvalidEntry]
-> Int
-> Int
-> Bool
-> Bool
-> Bool
-> Bool
-> Bool
-> PyPIProjection
PyPIProjection KeyMap Value
forall a. Monoid a => a
mempty [] (PackageName -> FilenameMemo
filenameMemo PackageName
name) [] [] Int
0 Int
0 Bool
True Bool
False Bool
False Bool
False Bool
False
keepsFile :: PyPIProjection -> Bool
keepsFile :: PyPIProjection -> Bool
keepsFile = PyPIProjection -> Bool
projectedFilesActive
claimsEnvelope :: PyPIProjection -> Text -> Bool
claimsEnvelope :: PyPIProjection -> Text -> Bool
claimsEnvelope PyPIProjection
acc Text
key = Key -> KeyMap Value -> Bool
forall a. Key -> KeyMap a -> Bool
KeyMap.member (Text -> Key
Key.fromText Text
key) (PyPIProjection -> KeyMap Value
projectedEnvelope PyPIProjection
acc)
retainedEntries :: PyPIProjection -> RetainedEntries
retainedEntries :: PyPIProjection -> RetainedEntries
retainedEntries PyPIProjection
acc = Int -> Int -> RetainedEntries
RetainedEntries (PyPIProjection -> Int
projectedReleases PyPIProjection
acc) (PyPIProjection -> Int
projectedOthers PyPIProjection
acc)
collectField :: Limits -> PyPIProjection -> PyPIField -> Either LimitError PyPIProjection
collectField :: Limits
-> PyPIProjection -> PyPIField -> Either LimitError PyPIProjection
collectField Limits
limits PyPIProjection
acc = \case
PyPIField
IgnoredField -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc
EnvelopeField Text
key Value
value
| Key -> KeyMap Value -> Bool
forall a. Key -> KeyMap a -> Bool
KeyMap.member (Text -> Key
Key.fromText Text
key) (PyPIProjection -> KeyMap Value
projectedEnvelope PyPIProjection
acc) -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc
| Bool
otherwise -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc{projectedEnvelope = KeyMap.insert (Key.fromText key) value (projectedEnvelope acc)}
EnvelopeItem Text
_ -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc{projectedOthers = projectedOthers acc + 1}
FilesShape Bool
valid -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc{projectedFilesSeen = True, projectedFilesActive = not (projectedFilesSeen acc), projectedShape = projectedShape acc && (projectedFilesSeen acc || valid)}
VersionsShape Bool
valid -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc{projectedVersionsSeen = True, projectedVersionsActive = not (projectedVersionsSeen acc), projectedShape = projectedShape acc && (projectedVersionsSeen acc || valid)}
InvalidVersionField Int
position Value
value
| PyPIProjection -> Bool
projectedVersionsActive PyPIProjection
acc ->
PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc{projectedVersionDrops = mkInvalidEntry InvalidVersionListing (show position) value "expected a version string" : projectedVersionDrops acc, projectedOthers = projectedOthers acc + 1}
| Bool
otherwise -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc
FileField Int
position Maybe Value
raw
| PyPIProjection -> Bool
projectedFilesActive PyPIProjection
acc -> do
Limits -> Int -> Either LimitError ()
checkVersionCountOf Limits
limits (Int
position Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1)
PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right (PyPIProjection
-> (Value -> PyPIProjection) -> Maybe Value -> PyPIProjection
forall b a. b -> (a -> b) -> Maybe a -> b
maybe PyPIProjection
acc (\Value
value -> Int -> Value -> PyPIProjection -> PyPIProjection
retainFile Int
position Value
value PyPIProjection
acc) Maybe Value
raw)
| Bool
otherwise -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc
retainFile :: Int -> Value -> PyPIProjection -> PyPIProjection
retainFile :: Int -> Value -> PyPIProjection -> PyPIProjection
retainFile Int
position Value
value PyPIProjection
current = (PyPIProjection -> IndexFile -> PyPIProjection)
-> PyPIProjection -> [IndexFile] -> PyPIProjection
forall b a. (b -> a -> b) -> b -> [a] -> b
forall (t :: * -> *) b a.
Foldable t =>
(b -> a -> b) -> b -> t a -> b
foldl' (FilePayload -> PyPIProjection -> IndexFile -> PyPIProjection
retainFileFact (Value -> FilePayload
TreeFile Value
value)) PyPIProjection
withDrops [IndexFile]
files
where
([IndexFile]
files, [InvalidEntry]
drops) = [(Int, Value)] -> ([IndexFile], [InvalidEntry])
decodeIndexFiles [(Int
position, Value
value)]
withDrops :: PyPIProjection
withDrops = PyPIProjection
current{projectedFileDrops = reverse drops <> projectedFileDrops current, projectedReleases = projectedReleases current + length drops}
retainFileFact :: FilePayload -> PyPIProjection -> IndexFile -> PyPIProjection
retainFileFact :: FilePayload -> PyPIProjection -> IndexFile -> PyPIProjection
retainFileFact FilePayload
served PyPIProjection
current IndexFile
file =
let (Maybe FileCoordinate
coordinate, FilenameMemo
memo) = FilenameMemo -> Text -> (Maybe FileCoordinate, FilenameMemo)
readCoordinate (PyPIProjection -> FilenameMemo
projectedMemo PyPIProjection
current) (IndexFile -> Text
ifFilename IndexFile
file)
!projected :: ProjectedFile
projected = IndexFile -> Maybe FileCoordinate -> ProjectedFile
projectFile IndexFile
file Maybe FileCoordinate
coordinate
in PyPIProjection
current{projectedFiles = (projected, served) : projectedFiles current, projectedMemo = memo, projectedReleases = projectedReleases current + 1}
packedFileStep :: Writer st -> Limits -> PyPIProjection -> Int -> FileValue -> (Either LimitError PyPIProjection -> ST st r) -> ST st r
packedFileStep :: forall st r.
Writer st
-> Limits
-> PyPIProjection
-> Int
-> FileValue
-> (Either LimitError PyPIProjection -> ST st r)
-> ST st r
packedFileStep Writer st
writer Limits
limits PyPIProjection
acc Int
position FileValue
value Either LimitError PyPIProjection -> ST st r
next
| Bool -> Bool
not (PyPIProjection -> Bool
projectedFilesActive PyPIProjection
acc) = Writer st -> ST st ()
forall st. Writer st -> ST st ()
discard Writer st
writer ST st () -> ST st r -> ST st r
forall a b. ST st a -> ST st b -> ST st b
forall (m :: * -> *) a b. Monad m => m a -> m b -> m b
>> Either LimitError PyPIProjection -> ST st r
next (PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc)
| Left LimitError
fault <- Limits -> Int -> Either LimitError ()
checkVersionCountOf Limits
limits (Int
position Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) = Writer st -> ST st ()
forall st. Writer st -> ST st ()
discard Writer st
writer ST st () -> ST st r -> ST st r
forall a b. ST st a -> ST st b -> ST st b
forall (m :: * -> *) a b. Monad m => m a -> m b -> m b
>> Either LimitError PyPIProjection -> ST st r
next (LimitError -> Either LimitError PyPIProjection
forall a b. a -> Either a b
Left LimitError
fault)
| Bool
otherwise = do
packed <- Writer st -> [Text] -> ST st Packed
forall st. Writer st -> [Text] -> ST st Packed
sealValue Writer st
writer [Text]
urlHole
case value of
FileValue (Just IndexFile
file) -> do
let !indexed :: IndexFile
indexed = IndexFile
file{ifEntryKey = ArrayEntry position}
!served :: Packed
served = Maybe Text -> Packed -> Packed
artifactUrlPacked (Text -> Maybe Text
forall a. a -> Maybe a
Just (IndexFile -> Text
ifUrl IndexFile
file)) Packed
packed
Either LimitError PyPIProjection -> ST st r
next (PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right (PyPIProjection -> Either LimitError PyPIProjection)
-> PyPIProjection -> Either LimitError PyPIProjection
forall a b. (a -> b) -> a -> b
$! FilePayload -> PyPIProjection -> IndexFile -> PyPIProjection
retainFileFact (Packed -> FilePayload
PackedFile Packed
served) PyPIProjection
acc IndexFile
indexed)
FileValue
_ -> do
diagnostic <- Writer st -> Packed -> ST st Value
forall st. Writer st -> Packed -> ST st Value
decodeWhole Writer st
writer Packed
packed
next (Right $! retainFile position diagnostic acc)
finishProjection :: PackageName -> PyPIProjection -> Either MetadataError (PackageInfo, SimpleDocument)
finishProjection :: PackageName
-> PyPIProjection
-> Either MetadataError (PackageInfo, SimpleDocument)
finishProjection PackageName
requested PyPIProjection
acc = do
info <- PackageName -> PyPIProjection -> Either MetadataError PackageInfo
finishPackage PackageName
requested PyPIProjection
acc
pure (info, simpleDocument (projectedEnvelope acc) (servedFiles (projectedFiles acc)))
finishPackedProjection :: PackageName -> DocTable -> PyPIProjection -> Either MetadataError (PackageInfo, PackedSimple)
finishPackedProjection :: PackageName
-> DocTable
-> PyPIProjection
-> Either MetadataError (PackageInfo, PackedSimple)
finishPackedProjection PackageName
requested DocTable
table PyPIProjection
acc = do
info <- PackageName -> PyPIProjection -> Either MetadataError PackageInfo
finishPackage PackageName
requested PyPIProjection
acc
pure (info, packedSimple (projectedEnvelope acc) table (foldl' serve [] (projectedFiles acc)))
where
serve :: [(EntryKey, Packed)]
-> (ProjectedFile, FilePayload) -> [(EntryKey, Packed)]
serve [(EntryKey, Packed)]
served (ProjectedFile
file, PackedFile Packed
payload) = let !key :: EntryKey
key = ProjectedFile -> EntryKey
projectedEntryKey ProjectedFile
file in (EntryKey
key, Packed
payload) (EntryKey, Packed) -> [(EntryKey, Packed)] -> [(EntryKey, Packed)]
forall a. a -> [a] -> [a]
: [(EntryKey, Packed)]
served
serve [(EntryKey, Packed)]
served (ProjectedFile, FilePayload)
_ = [(EntryKey, Packed)]
served
finishPackage :: PackageName -> PyPIProjection -> Either MetadataError PackageInfo
finishPackage :: PackageName -> PyPIProjection -> Either MetadataError PackageInfo
finishPackage PackageName
requested PyPIProjection
acc = do
(String -> MetadataError)
-> Either String () -> Either MetadataError ()
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first (MetadataError -> String -> MetadataError
forall a b. a -> b -> a
const MetadataError
MetadataUndecodable) ((KeyMap Value -> Parser ()) -> KeyMap Value -> Either String ()
forall a b. (a -> Parser b) -> a -> Either String b
parseEither KeyMap Value -> Parser ()
checkApiVersion (PyPIProjection -> KeyMap Value
projectedEnvelope PyPIProjection
acc))
reported <- (Text -> Either ParseError PackageName)
-> PackageName -> Maybe Value -> Either MetadataError PackageName
validateNameAgreement Text -> Either ParseError PackageName
projectName PackageName
requested (Key -> KeyMap Value -> Maybe Value
forall v. Key -> KeyMap v -> Maybe v
KeyMap.lookup Key
"name" (PyPIProjection -> KeyMap Value
projectedEnvelope PyPIProjection
acc))
unless (projectedShape acc) (Left MetadataUndecodable)
let invalid = [InvalidEntry] -> [InvalidEntry]
forall a. [a] -> [a]
reverse (PyPIProjection -> [InvalidEntry]
projectedFileDrops PyPIProjection
acc) [InvalidEntry] -> [InvalidEntry] -> [InvalidEntry]
forall a. Semigroup a => a -> a -> a
<> [InvalidEntry] -> [InvalidEntry]
forall a. [a] -> [a]
reverse (PyPIProjection -> [InvalidEntry]
projectedVersionDrops PyPIProjection
acc)
pure (projectSimpleIndex reported invalid (map fst (projectedFiles acc)))
servedFiles :: [(ProjectedFile, FilePayload)] -> [(EntryKey, Value)]
servedFiles :: [(ProjectedFile, FilePayload)] -> [(EntryKey, Value)]
servedFiles = ([(EntryKey, Value)]
-> (ProjectedFile, FilePayload) -> [(EntryKey, Value)])
-> [(EntryKey, Value)]
-> [(ProjectedFile, FilePayload)]
-> [(EntryKey, Value)]
forall b a. (b -> a -> b) -> b -> [a] -> b
forall (t :: * -> *) b a.
Foldable t =>
(b -> a -> b) -> b -> t a -> b
foldl' [(EntryKey, Value)]
-> (ProjectedFile, FilePayload) -> [(EntryKey, Value)]
serve []
where
serve :: [(EntryKey, Value)]
-> (ProjectedFile, FilePayload) -> [(EntryKey, Value)]
serve [(EntryKey, Value)]
served (ProjectedFile
file, TreeFile Value
value) = let !key :: EntryKey
key = ProjectedFile -> EntryKey
projectedEntryKey ProjectedFile
file in (EntryKey
key, Value
value) (EntryKey, Value) -> [(EntryKey, Value)] -> [(EntryKey, Value)]
forall a. a -> [a] -> [a]
: [(EntryKey, Value)]
served
serve [(EntryKey, Value)]
served (ProjectedFile, FilePayload)
_ = [(EntryKey, Value)]
served