-- SPDX-FileCopyrightText: 2026 Alexandra de Wit
--
-- SPDX-License-Identifier: MIT

-- | Join supported PyPI fields while preserving source positions and existing read limits.
module Ecluse.Core.Registry.PyPI.StreamingProjection (
    PyPIProjection,
    emptyProjection,
    collectField,
    keepsFile,
    claimsEnvelope,
    retainedEntries,
    finishProjection,
    packedFileStep,
    finishPackedProjection,
) where

import Control.Monad.ST (ST)
import Data.Aeson (Value)
import Data.Aeson.Key qualified as Key
import Data.Aeson.KeyMap qualified as KeyMap
import Data.Aeson.Types (parseEither)

import Ecluse.Core.Package (InvalidEntry, InvalidEntryKind (InvalidVersionListing), PackageInfo, PackageName, mkInvalidEntry)
import Ecluse.Core.Package.Entry (EntryKey (ArrayEntry))
import Ecluse.Core.Registry.Json.Packed (DocTable, Packed)
import Ecluse.Core.Registry.Json.Writer (Writer, decodeWhole, discard, sealValue)
import Ecluse.Core.Registry.Metadata (MetadataError (MetadataUndecodable))
import Ecluse.Core.Registry.Metadata.EntryFloor (RetainedEntries (RetainedEntries))
import Ecluse.Core.Registry.Metadata.Projection (validateNameAgreement)
import Ecluse.Core.Registry.PyPI.Document (PackedSimple, SimpleDocument, packedSimple, simpleDocument, urlHole)
import Ecluse.Core.Registry.PyPI.FileWriter (FileValue (..))
import Ecluse.Core.Registry.PyPI.Project (FilenameMemo, ProjectedFile, filenameMemo, projectFile, projectName, projectSimpleIndex, projectedEntryKey, readCoordinate)
import Ecluse.Core.Registry.PyPI.Streaming (PyPIField (..))
import Ecluse.Core.Registry.PyPI.Wire (IndexFile (ifEntryKey, ifFilename, ifUrl), checkApiVersion, decodeIndexFiles)
import Ecluse.Core.Registry.ServedDocument (artifactUrlPacked)
import Ecluse.Core.Security (LimitError, Limits, checkVersionCountOf)

-- | Pending files wait for the reported name so member order cannot change the package identity.
data PyPIProjection = PyPIProjection
    { PyPIProjection -> KeyMap Value
projectedEnvelope :: KeyMap.KeyMap Value
    , PyPIProjection -> [(ProjectedFile, FilePayload)]
projectedFiles :: [(ProjectedFile, FilePayload)]
    , PyPIProjection -> FilenameMemo
projectedMemo :: FilenameMemo
    , PyPIProjection -> [InvalidEntry]
projectedFileDrops :: [InvalidEntry]
    , PyPIProjection -> [InvalidEntry]
projectedVersionDrops :: [InvalidEntry]
    , PyPIProjection -> Int
projectedReleases :: Int
    , PyPIProjection -> Int
projectedOthers :: Int
    , PyPIProjection -> Bool
projectedShape :: Bool
    , PyPIProjection -> Bool
projectedFilesSeen :: Bool
    , PyPIProjection -> Bool
projectedFilesActive :: Bool
    , PyPIProjection -> Bool
projectedVersionsSeen :: Bool
    , PyPIProjection -> Bool
projectedVersionsActive :: Bool
    }

data FilePayload = TreeFile Value | PackedFile Packed

-- | Start one project's source without retaining any input chunks.
emptyProjection :: PackageName -> PyPIProjection
emptyProjection :: PackageName -> PyPIProjection
emptyProjection PackageName
name = KeyMap Value
-> [(ProjectedFile, FilePayload)]
-> FilenameMemo
-> [InvalidEntry]
-> [InvalidEntry]
-> Int
-> Int
-> Bool
-> Bool
-> Bool
-> Bool
-> Bool
-> PyPIProjection
PyPIProjection KeyMap Value
forall a. Monoid a => a
mempty [] (PackageName -> FilenameMemo
filenameMemo PackageName
name) [] [] Int
0 Int
0 Bool
True Bool
False Bool
False Bool
False Bool
False

-- | Whether a file read now would be retained: files in the first files array.
keepsFile :: PyPIProjection -> Bool
keepsFile :: PyPIProjection -> Bool
keepsFile = PyPIProjection -> Bool
projectedFilesActive

-- | Whether the envelope holds an earlier occurrence of the key, so the walk drops a repeat as it reads it.
claimsEnvelope :: PyPIProjection -> Text -> Bool
claimsEnvelope :: PyPIProjection -> Text -> Bool
claimsEnvelope PyPIProjection
acc Text
key = Key -> KeyMap Value -> Bool
forall a. Key -> KeyMap a -> Bool
KeyMap.member (Text -> Key
Key.fromText Text
key) (PyPIProjection -> KeyMap Value
projectedEnvelope PyPIProjection
acc)

-- | The files and file drops, and the version drops and envelope array items, the projection holds.
retainedEntries :: PyPIProjection -> RetainedEntries
retainedEntries :: PyPIProjection -> RetainedEntries
retainedEntries PyPIProjection
acc = Int -> Int -> RetainedEntries
RetainedEntries (PyPIProjection -> Int
projectedReleases PyPIProjection
acc) (PyPIProjection -> Int
projectedOthers PyPIProjection
acc)

-- | Take one field. A file refuses the read at the first file position past the version ceiling.
collectField :: Limits -> PyPIProjection -> PyPIField -> Either LimitError PyPIProjection
collectField :: Limits
-> PyPIProjection -> PyPIField -> Either LimitError PyPIProjection
collectField Limits
limits PyPIProjection
acc = \case
    PyPIField
IgnoredField -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc
    EnvelopeField Text
key Value
value
        | Key -> KeyMap Value -> Bool
forall a. Key -> KeyMap a -> Bool
KeyMap.member (Text -> Key
Key.fromText Text
key) (PyPIProjection -> KeyMap Value
projectedEnvelope PyPIProjection
acc) -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc
        | Bool
otherwise -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc{projectedEnvelope = KeyMap.insert (Key.fromText key) value (projectedEnvelope acc)}
    -- The walk reports items only for an envelope key no earlier occurrence has claimed.
    EnvelopeItem Text
_ -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc{projectedOthers = projectedOthers acc + 1}
    FilesShape Bool
valid -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc{projectedFilesSeen = True, projectedFilesActive = not (projectedFilesSeen acc), projectedShape = projectedShape acc && (projectedFilesSeen acc || valid)}
    VersionsShape Bool
valid -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc{projectedVersionsSeen = True, projectedVersionsActive = not (projectedVersionsSeen acc), projectedShape = projectedShape acc && (projectedVersionsSeen acc || valid)}
    InvalidVersionField Int
position Value
value
        | PyPIProjection -> Bool
projectedVersionsActive PyPIProjection
acc ->
            PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc{projectedVersionDrops = mkInvalidEntry InvalidVersionListing (show position) value "expected a version string" : projectedVersionDrops acc, projectedOthers = projectedOthers acc + 1}
        | Bool
otherwise -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc
    FileField Int
position Maybe Value
raw
        | PyPIProjection -> Bool
projectedFilesActive PyPIProjection
acc -> do
            Limits -> Int -> Either LimitError ()
checkVersionCountOf Limits
limits (Int
position Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1)
            PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right (PyPIProjection
-> (Value -> PyPIProjection) -> Maybe Value -> PyPIProjection
forall b a. b -> (a -> b) -> Maybe a -> b
maybe PyPIProjection
acc (\Value
value -> Int -> Value -> PyPIProjection -> PyPIProjection
retainFile Int
position Value
value PyPIProjection
acc) Maybe Value
raw)
        | Bool
otherwise -> PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc

-- The served file, or its invalid-entry diagnostic, under its source position.
retainFile :: Int -> Value -> PyPIProjection -> PyPIProjection
retainFile :: Int -> Value -> PyPIProjection -> PyPIProjection
retainFile Int
position Value
value PyPIProjection
current = (PyPIProjection -> IndexFile -> PyPIProjection)
-> PyPIProjection -> [IndexFile] -> PyPIProjection
forall b a. (b -> a -> b) -> b -> [a] -> b
forall (t :: * -> *) b a.
Foldable t =>
(b -> a -> b) -> b -> t a -> b
foldl' (FilePayload -> PyPIProjection -> IndexFile -> PyPIProjection
retainFileFact (Value -> FilePayload
TreeFile Value
value)) PyPIProjection
withDrops [IndexFile]
files
  where
    ([IndexFile]
files, [InvalidEntry]
drops) = [(Int, Value)] -> ([IndexFile], [InvalidEntry])
decodeIndexFiles [(Int
position, Value
value)]
    withDrops :: PyPIProjection
withDrops = PyPIProjection
current{projectedFileDrops = reverse drops <> projectedFileDrops current, projectedReleases = projectedReleases current + length drops}

retainFileFact :: FilePayload -> PyPIProjection -> IndexFile -> PyPIProjection
retainFileFact :: FilePayload -> PyPIProjection -> IndexFile -> PyPIProjection
retainFileFact FilePayload
served PyPIProjection
current IndexFile
file =
    let (Maybe FileCoordinate
coordinate, FilenameMemo
memo) = FilenameMemo -> Text -> (Maybe FileCoordinate, FilenameMemo)
readCoordinate (PyPIProjection -> FilenameMemo
projectedMemo PyPIProjection
current) (IndexFile -> Text
ifFilename IndexFile
file)
        !projected :: ProjectedFile
projected = IndexFile -> Maybe FileCoordinate -> ProjectedFile
projectFile IndexFile
file Maybe FileCoordinate
coordinate
     in PyPIProjection
current{projectedFiles = (projected, served) : projectedFiles current, projectedMemo = memo, projectedReleases = projectedReleases current + 1}

-- | Refuse at the first file position past the version ceiling. Decode a whole file only for its invalid-entry diagnostic.
packedFileStep :: Writer st -> Limits -> PyPIProjection -> Int -> FileValue -> (Either LimitError PyPIProjection -> ST st r) -> ST st r
packedFileStep :: forall st r.
Writer st
-> Limits
-> PyPIProjection
-> Int
-> FileValue
-> (Either LimitError PyPIProjection -> ST st r)
-> ST st r
packedFileStep Writer st
writer Limits
limits PyPIProjection
acc Int
position FileValue
value Either LimitError PyPIProjection -> ST st r
next
    | Bool -> Bool
not (PyPIProjection -> Bool
projectedFilesActive PyPIProjection
acc) = Writer st -> ST st ()
forall st. Writer st -> ST st ()
discard Writer st
writer ST st () -> ST st r -> ST st r
forall a b. ST st a -> ST st b -> ST st b
forall (m :: * -> *) a b. Monad m => m a -> m b -> m b
>> Either LimitError PyPIProjection -> ST st r
next (PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right PyPIProjection
acc)
    | Left LimitError
fault <- Limits -> Int -> Either LimitError ()
checkVersionCountOf Limits
limits (Int
position Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) = Writer st -> ST st ()
forall st. Writer st -> ST st ()
discard Writer st
writer ST st () -> ST st r -> ST st r
forall a b. ST st a -> ST st b -> ST st b
forall (m :: * -> *) a b. Monad m => m a -> m b -> m b
>> Either LimitError PyPIProjection -> ST st r
next (LimitError -> Either LimitError PyPIProjection
forall a b. a -> Either a b
Left LimitError
fault)
    | Bool
otherwise = do
        packed <- Writer st -> [Text] -> ST st Packed
forall st. Writer st -> [Text] -> ST st Packed
sealValue Writer st
writer [Text]
urlHole
        case value of
            FileValue (Just IndexFile
file) -> do
                let !indexed :: IndexFile
indexed = IndexFile
file{ifEntryKey = ArrayEntry position}
                    !served :: Packed
served = Maybe Text -> Packed -> Packed
artifactUrlPacked (Text -> Maybe Text
forall a. a -> Maybe a
Just (IndexFile -> Text
ifUrl IndexFile
file)) Packed
packed
                Either LimitError PyPIProjection -> ST st r
next (PyPIProjection -> Either LimitError PyPIProjection
forall a b. b -> Either a b
Right (PyPIProjection -> Either LimitError PyPIProjection)
-> PyPIProjection -> Either LimitError PyPIProjection
forall a b. (a -> b) -> a -> b
$! FilePayload -> PyPIProjection -> IndexFile -> PyPIProjection
retainFileFact (Packed -> FilePayload
PackedFile Packed
served) PyPIProjection
acc IndexFile
indexed)
            FileValue
_ -> do
                diagnostic <- Writer st -> Packed -> ST st Value
forall st. Writer st -> Packed -> ST st Value
decodeWhole Writer st
writer Packed
packed
                next (Right $! retainFile position diagnostic acc)

-- | Check protocol, name and shape, then project the source's retained files.
finishProjection :: PackageName -> PyPIProjection -> Either MetadataError (PackageInfo, SimpleDocument)
finishProjection :: PackageName
-> PyPIProjection
-> Either MetadataError (PackageInfo, SimpleDocument)
finishProjection PackageName
requested PyPIProjection
acc = do
    info <- PackageName -> PyPIProjection -> Either MetadataError PackageInfo
finishPackage PackageName
requested PyPIProjection
acc
    pure (info, simpleDocument (projectedEnvelope acc) (servedFiles (projectedFiles acc)))

-- | Bind the reported name after reading, and retain serving files against the sealed table.
finishPackedProjection :: PackageName -> DocTable -> PyPIProjection -> Either MetadataError (PackageInfo, PackedSimple)
finishPackedProjection :: PackageName
-> DocTable
-> PyPIProjection
-> Either MetadataError (PackageInfo, PackedSimple)
finishPackedProjection PackageName
requested DocTable
table PyPIProjection
acc = do
    info <- PackageName -> PyPIProjection -> Either MetadataError PackageInfo
finishPackage PackageName
requested PyPIProjection
acc
    pure (info, packedSimple (projectedEnvelope acc) table (foldl' serve [] (projectedFiles acc)))
  where
    serve :: [(EntryKey, Packed)]
-> (ProjectedFile, FilePayload) -> [(EntryKey, Packed)]
serve [(EntryKey, Packed)]
served (ProjectedFile
file, PackedFile Packed
payload) = let !key :: EntryKey
key = ProjectedFile -> EntryKey
projectedEntryKey ProjectedFile
file in (EntryKey
key, Packed
payload) (EntryKey, Packed) -> [(EntryKey, Packed)] -> [(EntryKey, Packed)]
forall a. a -> [a] -> [a]
: [(EntryKey, Packed)]
served
    serve [(EntryKey, Packed)]
served (ProjectedFile, FilePayload)
_ = [(EntryKey, Packed)]
served

finishPackage :: PackageName -> PyPIProjection -> Either MetadataError PackageInfo
finishPackage :: PackageName -> PyPIProjection -> Either MetadataError PackageInfo
finishPackage PackageName
requested PyPIProjection
acc = do
    (String -> MetadataError)
-> Either String () -> Either MetadataError ()
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first (MetadataError -> String -> MetadataError
forall a b. a -> b -> a
const MetadataError
MetadataUndecodable) ((KeyMap Value -> Parser ()) -> KeyMap Value -> Either String ()
forall a b. (a -> Parser b) -> a -> Either String b
parseEither KeyMap Value -> Parser ()
checkApiVersion (PyPIProjection -> KeyMap Value
projectedEnvelope PyPIProjection
acc))
    reported <- (Text -> Either ParseError PackageName)
-> PackageName -> Maybe Value -> Either MetadataError PackageName
validateNameAgreement Text -> Either ParseError PackageName
projectName PackageName
requested (Key -> KeyMap Value -> Maybe Value
forall v. Key -> KeyMap v -> Maybe v
KeyMap.lookup Key
"name" (PyPIProjection -> KeyMap Value
projectedEnvelope PyPIProjection
acc))
    unless (projectedShape acc) (Left MetadataUndecodable)
    let invalid = [InvalidEntry] -> [InvalidEntry]
forall a. [a] -> [a]
reverse (PyPIProjection -> [InvalidEntry]
projectedFileDrops PyPIProjection
acc) [InvalidEntry] -> [InvalidEntry] -> [InvalidEntry]
forall a. Semigroup a => a -> a -> a
<> [InvalidEntry] -> [InvalidEntry]
forall a. [a] -> [a]
reverse (PyPIProjection -> [InvalidEntry]
projectedVersionDrops PyPIProjection
acc)
    pure (projectSimpleIndex reported invalid (map fst (projectedFiles acc)))

-- Restore source order with every key evaluated, so the served list holds no decoded file.
servedFiles :: [(ProjectedFile, FilePayload)] -> [(EntryKey, Value)]
servedFiles :: [(ProjectedFile, FilePayload)] -> [(EntryKey, Value)]
servedFiles = ([(EntryKey, Value)]
 -> (ProjectedFile, FilePayload) -> [(EntryKey, Value)])
-> [(EntryKey, Value)]
-> [(ProjectedFile, FilePayload)]
-> [(EntryKey, Value)]
forall b a. (b -> a -> b) -> b -> [a] -> b
forall (t :: * -> *) b a.
Foldable t =>
(b -> a -> b) -> b -> t a -> b
foldl' [(EntryKey, Value)]
-> (ProjectedFile, FilePayload) -> [(EntryKey, Value)]
serve []
  where
    serve :: [(EntryKey, Value)]
-> (ProjectedFile, FilePayload) -> [(EntryKey, Value)]
serve [(EntryKey, Value)]
served (ProjectedFile
file, TreeFile Value
value) = let !key :: EntryKey
key = ProjectedFile -> EntryKey
projectedEntryKey ProjectedFile
file in (EntryKey
key, Value
value) (EntryKey, Value) -> [(EntryKey, Value)] -> [(EntryKey, Value)]
forall a. a -> [a] -> [a]
: [(EntryKey, Value)]
served
    serve [(EntryKey, Value)]
served (ProjectedFile, FilePayload)
_ = [(EntryKey, Value)]
served