-- SPDX-FileCopyrightText: 2026 Alexandra de Wit
--
-- SPDX-License-Identifier: MIT

-- | Simple-index walks share source ordering and validation across tree and packed reads.
module Ecluse.Core.Registry.PyPI.Reader (
    pypiWalk,
    pypiFullWalk,
    fileUniqueFields,
) where

import Control.Monad.ST (ST)
import Data.Aeson (Value (Array, Null, Object, String))
import Data.Aeson.Key qualified as Key
import Data.Aeson.KeyMap qualified as KeyMap
import Data.JsonStream.Lexer (Cursor, Element (..))
import Data.Vector qualified as V

import Ecluse.Core.Registry.Json.Intern (InternTable, nameBytes, nameText)
import Ecluse.Core.Registry.Json.Shape (Members, Mode (..), Shape (..), Trees (..), knownMembers, listedMember, namedMembers, prepareMembers, readShape)
import Ecluse.Core.Registry.Json.Walk (FieldStep, Step, Steps (..), Walk, Walked (..), eachItem, eachMember, pureStep, skipFrom, skipRest, tooDeep, withElement)
import Ecluse.Core.Registry.Json.Walk qualified as Walk
import Ecluse.Core.Registry.PyPI.FileWriter (FileValue, FileWriter)
import Ecluse.Core.Registry.PyPI.PendingSidecar (readPendingSidecar)
import Ecluse.Core.Registry.PyPI.Project (FilenameMemo, filenameMemo)
import Ecluse.Core.Registry.PyPI.Streaming (PyPIField (..), PyPIRead (..), SelectedFile (..), SelectedFileEvent (..), collectSelected, fileScalars, finishSelected, hashNames)
import Ecluse.Core.Security (LimitError)

-- | Members whose values differ in every file, so the table keeps them as read.
fileUniqueFields :: [Text]
fileUniqueFields :: [Text]
fileUniqueFields = [Text
"filename", Text
"url", Text
"hashes", Text
"upload-time", Text
"provenance", Text
"core-metadata"]

-- | Walk an index into trees, retaining the first occurrence and the selected-file skip policy.
pypiWalk :: Int -> PyPIRead -> (s -> PyPIField -> Either LimitError s) -> (s -> Bool) -> (s -> Text -> Bool) -> InternTable -> s -> Cursor -> Step s
{-# INLINE pypiWalk #-}
pypiWalk :: forall s.
Int
-> PyPIRead
-> (s -> PyPIField -> Either LimitError s)
-> (s -> Bool)
-> (s -> Text -> Bool)
-> InternTable
-> s
-> Cursor
-> Step s
pypiWalk Int
depth PyPIRead
mode s -> PyPIField -> Either LimitError s
step s -> Bool
keeps s -> Text -> Bool
claims InternTable
table0 = Int
-> PyPIRead
-> FieldStep s PyPIField (Step s)
-> (s -> Text -> Bool)
-> (InternTable -> s -> Step s)
-> (InternTable
    -> s
    -> Int
    -> Element
    -> Cursor
    -> (s -> InternTable -> Cursor -> Step s)
    -> Step s)
-> InternTable
-> s
-> Cursor
-> Step s
forall r s.
Walk r =>
Int
-> PyPIRead
-> FieldStep s PyPIField r
-> (s -> Text -> Bool)
-> (InternTable -> s -> r)
-> (InternTable
    -> s
    -> Int
    -> Element
    -> Cursor
    -> (s -> InternTable -> Cursor -> r)
    -> r)
-> InternTable
-> s
-> Cursor
-> r
walkIndex Int
depth PyPIRead
mode ((s -> PyPIField -> Either LimitError s)
-> FieldStep s PyPIField (Step s)
forall s field r.
(s -> field -> Either LimitError s) -> FieldStep s field r
pureStep s -> PyPIField -> Either LimitError s
step) s -> Text -> Bool
claims (\InternTable
_ s
acc -> s -> Step s
forall (m :: * -> *) s. s -> Steps m s
Finished s
acc) InternTable
-> s
-> Int
-> Element
-> Cursor
-> (s -> InternTable -> Cursor -> Step s)
-> Step s
forall {r}.
Walk r =>
InternTable
-> s
-> Int
-> Element
-> Cursor
-> (s -> InternTable -> Cursor -> r)
-> r
treeFile InternTable
table0
  where
    fileShape :: Shape
fileShape = Value -> Members -> Shape
ObjectOr Value
Null (Int -> (Members -> Members) -> Members
fileMembers Int
depth (InternTable -> Members -> Members
prepareMembers InternTable
table0))
    treeFile :: InternTable
-> s
-> Int
-> Element
-> Cursor
-> (s -> InternTable -> Cursor -> r)
-> r
treeFile InternTable
table s
acc Int
position Element
element Cursor
rest s -> InternTable -> Cursor -> r
next =
        Trees
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built Trees -> InternTable -> Cursor -> r)
-> r
forall b r.
Build b r =>
b
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built b -> InternTable -> Cursor -> r)
-> r
readShape Trees
Trees Shape
fileShape (if s -> Bool
keeps s
acc then Mode
Share else Mode
Keep) InternTable
table Element
element Cursor
rest ((Built Trees -> InternTable -> Cursor -> r) -> r)
-> (Built Trees -> InternTable -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Built Trees
payload InternTable
held Cursor
after ->
            FieldStep s PyPIField r -> s -> PyPIField -> (s -> r) -> r
forall r s field.
Walk r =>
FieldStep s field r -> s -> field -> (s -> r) -> r
Walk.emit ((s -> PyPIField -> Either LimitError s) -> FieldStep s PyPIField r
forall s field r.
(s -> field -> Either LimitError s) -> FieldStep s field r
pureStep s -> PyPIField -> Either LimitError s
step) s
acc (Int -> Maybe Value -> PyPIField
FileField Int
position (Value -> Maybe Value
forall a. a -> Maybe a
Just Value
Built Trees
payload)) (\s
kept -> s -> InternTable -> Cursor -> r
next s
kept InternTable
held Cursor
after)

-- | Walk a full index with typed reduction beside packing, returning the read's final table.
pypiFullWalk :: FileWriter st -> Int -> FieldStep s PyPIField (ST st (Steps (ST st) (Walked s))) -> (s -> Bool) -> (s -> Text -> Bool) -> (s -> Int -> FileValue -> (Either LimitError s -> ST st (Steps (ST st) (Walked s))) -> ST st (Steps (ST st) (Walked s))) -> InternTable -> s -> Cursor -> ST st (Steps (ST st) (Walked s))
{-# INLINE pypiFullWalk #-}
pypiFullWalk :: forall st s.
FileWriter st
-> Int
-> FieldStep s PyPIField (ST st (Steps (ST st) (Walked s)))
-> (s -> Bool)
-> (s -> Text -> Bool)
-> (s
    -> Int
    -> FileValue
    -> (Either LimitError s -> ST st (Steps (ST st) (Walked s)))
    -> ST st (Steps (ST st) (Walked s)))
-> InternTable
-> s
-> Cursor
-> ST st (Steps (ST st) (Walked s))
pypiFullWalk FileWriter st
writer Int
depth FieldStep s PyPIField (ST st (Steps (ST st) (Walked s)))
step s -> Bool
keeps s -> Text -> Bool
claims s
-> Int
-> FileValue
-> (Either LimitError s -> ST st (Steps (ST st) (Walked s)))
-> ST st (Steps (ST st) (Walked s))
fileStep InternTable
table0 = Int
-> PyPIRead
-> FieldStep s PyPIField (ST st (Steps (ST st) (Walked s)))
-> (s -> Text -> Bool)
-> (InternTable -> s -> ST st (Steps (ST st) (Walked s)))
-> (InternTable
    -> s
    -> Int
    -> Element
    -> Cursor
    -> (s -> InternTable -> Cursor -> ST st (Steps (ST st) (Walked s)))
    -> ST st (Steps (ST st) (Walked s)))
-> InternTable
-> s
-> Cursor
-> ST st (Steps (ST st) (Walked s))
forall r s.
Walk r =>
Int
-> PyPIRead
-> FieldStep s PyPIField r
-> (s -> Text -> Bool)
-> (InternTable -> s -> r)
-> (InternTable
    -> s
    -> Int
    -> Element
    -> Cursor
    -> (s -> InternTable -> Cursor -> r)
    -> r)
-> InternTable
-> s
-> Cursor
-> r
walkIndex Int
depth PyPIRead
FullRead FieldStep s PyPIField (ST st (Steps (ST st) (Walked s)))
step s -> Text -> Bool
claims (\InternTable
table s
acc -> Steps (ST st) (Walked s) -> ST st (Steps (ST st) (Walked s))
forall a. a -> ST st a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Walked s -> Steps (ST st) (Walked s)
forall (m :: * -> *) s. s -> Steps m s
Finished (InternTable -> s -> Walked s
forall s. InternTable -> s -> Walked s
Walked InternTable
table s
acc))) InternTable
-> s
-> Int
-> Element
-> Cursor
-> (s -> InternTable -> Cursor -> ST st (Steps (ST st) (Walked s)))
-> ST st (Steps (ST st) (Walked s))
packedFile InternTable
table0
  where
    fileShape :: Shape
fileShape = Value -> Members -> Shape
ObjectOr Value
Null (Int -> (Members -> Members) -> Members
fileMembers Int
depth (InternTable -> Members -> Members
prepareMembers InternTable
table0))
    packedFile :: InternTable
-> s
-> Int
-> Element
-> Cursor
-> (s -> InternTable -> Cursor -> ST st (Steps (ST st) (Walked s)))
-> ST st (Steps (ST st) (Walked s))
packedFile InternTable
table s
acc Int
position Element
element Cursor
rest s -> InternTable -> Cursor -> ST st (Steps (ST st) (Walked s))
next =
        FileWriter st
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built (FileWriter st)
    -> InternTable -> Cursor -> ST st (Steps (ST st) (Walked s)))
-> ST st (Steps (ST st) (Walked s))
forall b r.
Build b r =>
b
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built b -> InternTable -> Cursor -> r)
-> r
readShape FileWriter st
writer Shape
fileShape (if s -> Bool
keeps s
acc then Mode
Share else Mode
Keep) InternTable
table Element
element Cursor
rest ((Built (FileWriter st)
  -> InternTable -> Cursor -> ST st (Steps (ST st) (Walked s)))
 -> ST st (Steps (ST st) (Walked s)))
-> (Built (FileWriter st)
    -> InternTable -> Cursor -> ST st (Steps (ST st) (Walked s)))
-> ST st (Steps (ST st) (Walked s))
forall a b. (a -> b) -> a -> b
$ \Built (FileWriter st)
payload InternTable
held Cursor
after ->
            s
-> Int
-> FileValue
-> (Either LimitError s -> ST st (Steps (ST st) (Walked s)))
-> ST st (Steps (ST st) (Walked s))
fileStep s
acc Int
position Built (FileWriter st)
FileValue
payload ((LimitError -> ST st (Steps (ST st) (Walked s)))
-> (s -> ST st (Steps (ST st) (Walked s)))
-> Either LimitError s
-> ST st (Steps (ST st) (Walked s))
forall a c b. (a -> c) -> (b -> c) -> Either a b -> c
either LimitError -> ST st (Steps (ST st) (Walked s))
forall r. Walk r => LimitError -> r
Walk.refuse (\s
kept -> s -> InternTable -> Cursor -> ST st (Steps (ST st) (Walked s))
next s
kept InternTable
held Cursor
after))

-- An envelope key the consumer @claims@ holds an earlier occurrence, so a repeat is read and dropped.
walkIndex :: (Walk r) => Int -> PyPIRead -> FieldStep s PyPIField r -> (s -> Text -> Bool) -> (InternTable -> s -> r) -> (InternTable -> s -> Int -> Element -> Cursor -> (s -> InternTable -> Cursor -> r) -> r) -> InternTable -> s -> Cursor -> r
{-# INLINE walkIndex #-}
walkIndex :: forall r s.
Walk r =>
Int
-> PyPIRead
-> FieldStep s PyPIField r
-> (s -> Text -> Bool)
-> (InternTable -> s -> r)
-> (InternTable
    -> s
    -> Int
    -> Element
    -> Cursor
    -> (s -> InternTable -> Cursor -> r)
    -> r)
-> InternTable
-> s
-> Cursor
-> r
walkIndex Int
depth PyPIRead
mode FieldStep s PyPIField r
step s -> Text -> Bool
claims InternTable -> s -> r
finishRead InternTable
-> s
-> Int
-> Element
-> Cursor
-> (s -> InternTable -> Cursor -> r)
-> r
readIndexFile InternTable
table0 s
initial = Cursor -> r
start
  where
    start :: Cursor -> r
start Cursor
tokens
        | Int
depth Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
<= Int
0 = Cursor -> (Element -> Cursor -> r) -> r
forall r. Walk r => Cursor -> (Element -> Cursor -> r) -> r
withElement Cursor
tokens Element -> Cursor -> r
forall r. Walk r => Element -> Cursor -> r
tooDeep
        | Bool
otherwise = Cursor -> (Element -> Cursor -> r) -> r
forall r. Walk r => Cursor -> (Element -> Cursor -> r) -> r
withElement Cursor
tokens ((Element -> Cursor -> r) -> r) -> (Element -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Element
element Cursor
rest -> case Element
element of
            Element
ObjectBegin -> (Walked s -> Name -> Cursor -> (Walked s -> Cursor -> r) -> r)
-> (Walked s -> Cursor -> r) -> Walked s -> Cursor -> r
forall r st.
Walk r =>
(st -> Name -> Cursor -> (st -> Cursor -> r) -> r)
-> (st -> Cursor -> r) -> st -> Cursor -> r
eachMember Walked s -> Name -> Cursor -> (Walked s -> Cursor -> r) -> r
topField (\(Walked InternTable
held s
acc) Cursor
_ -> InternTable -> s -> r
finishRead InternTable
held s
acc) (InternTable -> s -> Walked s
forall s. InternTable -> s -> Walked s
Walked InternTable
table0 s
initial) Cursor
rest
            Element
_ -> Element -> Cursor -> (Cursor -> r) -> r
forall r. Walk r => Element -> Cursor -> (Cursor -> r) -> r
skipFrom Element
element Cursor
rest (r -> Cursor -> r
forall a b. a -> b -> a
const (InternTable -> s -> r
finishRead InternTable
table0 s
initial))
    full :: Bool
full = case PyPIRead
mode of
        PyPIRead
FullRead -> Bool
True
        SelectedRead{} -> Bool
False
        MirrorRead{} -> Bool
True
    emit :: s -> PyPIField -> (s -> r) -> r
emit = FieldStep s PyPIField r -> s -> PyPIField -> (s -> r) -> r
forall r s field.
Walk r =>
FieldStep s field r -> s -> field -> (s -> r) -> r
Walk.emit FieldStep s PyPIField r
step
    topField :: Walked s -> Name -> Cursor -> (Walked s -> Cursor -> r) -> r
topField walked :: Walked s
walked@(Walked InternTable
table s
acc) Name
name Cursor
after Walked s -> Cursor -> r
continue = case Name -> ByteString
nameBytes Name
name of
        ByteString
"name" -> Text -> Shape -> r
envelope Text
"name" (Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1))
        ByteString
"meta" -> Cursor -> (Element -> Cursor -> r) -> r
forall r. Walk r => Cursor -> (Element -> Cursor -> r) -> r
withElement Cursor
after ((Element -> Cursor -> r) -> r) -> (Element -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Element
element Cursor
afterKey -> case Element
element of
            Element
ObjectBegin
                | Int
depth Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Int
1, s -> Text -> Bool
claims s
acc Text
"meta" -> (Fields s -> Name -> Cursor -> (Fields s -> Cursor -> r) -> r)
-> (Fields s -> Cursor -> r) -> Fields s -> Cursor -> r
forall r st.
Walk r =>
(st -> Name -> Cursor -> (st -> Cursor -> r) -> r)
-> (st -> Cursor -> r) -> st -> Cursor -> r
eachMember (Bool
-> InternTable
-> Fields s
-> Name
-> Cursor
-> (Fields s -> Cursor -> r)
-> r
metaMember Bool
False InternTable
table) (\(Fields s
held KeyMap Value
_) -> Walked s -> Cursor -> r
continue (InternTable -> s -> Walked s
forall s. InternTable -> s -> Walked s
Walked InternTable
table s
held)) (s -> KeyMap Value -> Fields s
forall s. s -> KeyMap Value -> Fields s
Fields s
acc KeyMap Value
forall a. Monoid a => a
mempty) Cursor
afterKey
                | Int
depth Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Int
1 -> (Fields s -> Name -> Cursor -> (Fields s -> Cursor -> r) -> r)
-> (Fields s -> Cursor -> r) -> Fields s -> Cursor -> r
forall r st.
Walk r =>
(st -> Name -> Cursor -> (st -> Cursor -> r) -> r)
-> (st -> Cursor -> r) -> st -> Cursor -> r
eachMember (Bool
-> InternTable
-> Fields s
-> Name
-> Cursor
-> (Fields s -> Cursor -> r)
-> r
metaMember Bool
True InternTable
table) (\(Fields s
held KeyMap Value
members) -> Text -> s -> Value -> Cursor -> r
keep Text
"meta" s
held (KeyMap Value -> Value
Object KeyMap Value
members)) (s -> KeyMap Value -> Fields s
forall s. s -> KeyMap Value -> Fields s
Fields s
acc KeyMap Value
forall a. Monoid a => a
mempty) Cursor
afterKey
            Element
_ -> Text -> Shape -> Element -> Cursor -> r
envelopeValue Text
"meta" (Int -> Members -> Shape -> Shape
ObjectWith (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1) Members
metaFields (Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1))) Element
element Cursor
afterKey
        ByteString
"project-status" | Bool
full -> Text -> Shape -> r
envelope Text
"project-status" (Int -> Members -> Shape -> Shape
ObjectWith (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1) Members
statusFields (Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1)))
        ByteString
"alternate-locations" | Bool
full -> Cursor -> (Element -> Cursor -> r) -> r
forall r. Walk r => Cursor -> (Element -> Cursor -> r) -> r
withElement Cursor
after ((Element -> Cursor -> r) -> r) -> (Element -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Element
element Cursor
afterKey -> case Element
element of
            Element
ArrayBegin
                | Int
depth Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Int
1, s -> Text -> Bool
claims s
acc Text
"alternate-locations" -> Int -> InternTable -> Cursor -> (Cursor -> r) -> r
forall {r}.
Walk r =>
Int -> InternTable -> Cursor -> (Cursor -> r) -> r
dropItems (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) InternTable
table Cursor
afterKey (Walked s -> Cursor -> r
continue Walked s
walked)
                | Int
depth Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Int
1 -> Text
-> Int
-> InternTable
-> s
-> Cursor
-> (s -> Value -> Cursor -> r)
-> r
envelopeItems Text
"alternate-locations" (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) InternTable
table s
acc Cursor
afterKey (Text -> s -> Value -> Cursor -> r
keep Text
"alternate-locations")
            Element
_ -> Text -> Shape -> Element -> Cursor -> r
envelopeValue Text
"alternate-locations" (Int -> Shape -> Shape -> Shape
ArrayWith (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1) (Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2)) (Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1))) Element
element Cursor
afterKey
        ByteString
"files" -> (Bool -> PyPIField)
-> ((Walked s -> Cursor -> r) -> Walked s -> Cursor -> r)
-> Walked s
-> Cursor
-> (Walked s -> Cursor -> r)
-> r
shaped Bool -> PyPIField
FilesShape (Walked s -> Cursor -> r) -> Walked s -> Cursor -> r
files Walked s
walked Cursor
after Walked s -> Cursor -> r
continue
        ByteString
"versions" | Bool
full -> (Bool -> PyPIField)
-> ((Walked s -> Cursor -> r) -> Walked s -> Cursor -> r)
-> Walked s
-> Cursor
-> (Walked s -> Cursor -> r)
-> r
shaped Bool -> PyPIField
VersionsShape ((Walked s
 -> Int -> Element -> Cursor -> (Walked s -> Cursor -> r) -> r)
-> (Walked s -> Cursor -> r) -> Walked s -> Cursor -> r
forall r st.
Walk r =>
(st -> Int -> Element -> Cursor -> (st -> Cursor -> r) -> r)
-> (st -> Cursor -> r) -> st -> Cursor -> r
eachItem Walked s
-> Int -> Element -> Cursor -> (Walked s -> Cursor -> r) -> r
version) Walked s
walked Cursor
after Walked s -> Cursor -> r
continue
        ByteString
_ -> Cursor -> (Element -> Cursor -> r) -> r
forall r. Walk r => Cursor -> (Element -> Cursor -> r) -> r
withElement Cursor
after ((Element -> Cursor -> r) -> r) -> (Element -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Element
element Cursor
afterKey -> Element -> Cursor -> (Cursor -> r) -> r
forall r. Walk r => Element -> Cursor -> (Cursor -> r) -> r
skipFrom Element
element Cursor
afterKey (Walked s -> Cursor -> r
continue Walked s
walked)
      where
        envelope :: Text -> Shape -> r
envelope Text
key Shape
shape = Cursor -> (Element -> Cursor -> r) -> r
forall r. Walk r => Cursor -> (Element -> Cursor -> r) -> r
withElement Cursor
after (Text -> Shape -> Element -> Cursor -> r
envelopeValue Text
key Shape
shape)
        envelopeValue :: Text -> Shape -> Element -> Cursor -> r
envelopeValue Text
key Shape
shape Element
element Cursor
afterKey =
            Trees
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built Trees -> InternTable -> Cursor -> r)
-> r
forall b r.
Build b r =>
b
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built b -> InternTable -> Cursor -> r)
-> r
readShape Trees
Trees Shape
shape Mode
Keep InternTable
table Element
element Cursor
afterKey ((Built Trees -> InternTable -> Cursor -> r) -> r)
-> (Built Trees -> InternTable -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Built Trees
value InternTable
_ -> Text -> s -> Value -> Cursor -> r
keep Text
key s
acc Value
Built Trees
value
        keep :: Text -> s -> Value -> Cursor -> r
keep Text
key s
held Value
value Cursor
afterValue = s -> PyPIField -> (s -> r) -> r
emit s
held (Text -> Value -> PyPIField
EnvelopeField Text
key Value
value) (\s
acc' -> Walked s -> Cursor -> r
continue (InternTable -> s -> Walked s
forall s. InternTable -> s -> Walked s
Walked InternTable
table s
acc') Cursor
afterValue)

    -- Meta keeps its listed members, the first of each name, as an object read keeps them. A member
    -- it does not keep, every member of a meta object after the first included, is read and dropped.
    metaMember :: Bool
-> InternTable
-> Fields s
-> Name
-> Cursor
-> (Fields s -> Cursor -> r)
-> r
metaMember Bool
keeping InternTable
table (Fields s
acc KeyMap Value
members) Name
name Cursor
after Fields s -> Cursor -> r
next = Cursor -> (Element -> Cursor -> r) -> r
forall r. Walk r => Cursor -> (Element -> Cursor -> r) -> r
withElement Cursor
after ((Element -> Cursor -> r) -> r) -> (Element -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Element
element Cursor
afterKey -> case Members -> Name -> Maybe (Key, Shape)
listedMember Members
metaFields Name
name of
        Just (Key
key, Shape
shape)
            | Key
key Key -> Key -> Bool
forall a. Eq a => a -> a -> Bool
== Key
"tracks"
            , Element
ArrayBegin <- Element
element
            , Int
depth Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Int
2 ->
                if Key -> Bool
kept Key
key
                    then Text
-> Int
-> InternTable
-> s
-> Cursor
-> (s -> Value -> Cursor -> r)
-> r
envelopeItems Text
"meta" (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
3) InternTable
table s
acc Cursor
afterKey ((s -> Value -> Cursor -> r) -> r)
-> (s -> Value -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \s
counted Value
value -> Fields s -> Cursor -> r
next (s -> KeyMap Value -> Fields s
forall s. s -> KeyMap Value -> Fields s
Fields s
counted (Key -> Value -> KeyMap Value -> KeyMap Value
forall v. Key -> v -> KeyMap v -> KeyMap v
KeyMap.insert Key
key Value
value KeyMap Value
members))
                    else Int -> InternTable -> Cursor -> (Cursor -> r) -> r
forall {r}.
Walk r =>
Int -> InternTable -> Cursor -> (Cursor -> r) -> r
dropItems (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
3) InternTable
table Cursor
afterKey (Fields s -> Cursor -> r
next (s -> KeyMap Value -> Fields s
forall s. s -> KeyMap Value -> Fields s
Fields s
acc KeyMap Value
members))
            | Key -> Bool
kept Key
key -> Trees
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built Trees -> InternTable -> Cursor -> r)
-> r
forall b r.
Build b r =>
b
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built b -> InternTable -> Cursor -> r)
-> r
readShape Trees
Trees Shape
shape Mode
Keep InternTable
table Element
element Cursor
afterKey (\Built Trees
value InternTable
_ -> Fields s -> Cursor -> r
next (s -> KeyMap Value -> Fields s
forall s. s -> KeyMap Value -> Fields s
Fields s
acc (Key -> Value -> KeyMap Value -> KeyMap Value
forall v. Key -> v -> KeyMap v -> KeyMap v
KeyMap.insert Key
key Value
Built Trees
value KeyMap Value
members)))
            | Bool
otherwise -> Trees
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built Trees -> InternTable -> Cursor -> r)
-> r
forall b r.
Build b r =>
b
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built b -> InternTable -> Cursor -> r)
-> r
readShape Trees
Trees Shape
shape Mode
Keep InternTable
table Element
element Cursor
afterKey (\Built Trees
_ InternTable
_ -> Fields s -> Cursor -> r
next (s -> KeyMap Value -> Fields s
forall s. s -> KeyMap Value -> Fields s
Fields s
acc KeyMap Value
members))
        Maybe (Key, Shape)
Nothing -> Element -> Cursor -> (Cursor -> r) -> r
forall r. Walk r => Element -> Cursor -> (Cursor -> r) -> r
skipFrom Element
element Cursor
afterKey (Fields s -> Cursor -> r
next (s -> KeyMap Value -> Fields s
forall s. s -> KeyMap Value -> Fields s
Fields s
acc KeyMap Value
members))
      where
        kept :: Key -> Bool
kept Key
key = Bool
keeping Bool -> Bool -> Bool
&& Bool -> Bool
not (Key -> KeyMap Value -> Bool
forall a. Key -> KeyMap a -> Bool
KeyMap.member Key
key KeyMap Value
members)

    -- Each item reaches the step as it is read, so the entry floor counts it before its array closes.
    envelopeItems :: Text
-> Int
-> InternTable
-> s
-> Cursor
-> (s -> Value -> Cursor -> r)
-> r
envelopeItems Text
key Int
budget InternTable
table s
acc0 Cursor
tokens s -> Value -> Cursor -> r
done = (Items s
 -> Int -> Element -> Cursor -> (Items s -> Cursor -> r) -> r)
-> (Items s -> Cursor -> r) -> Items s -> Cursor -> r
forall r st.
Walk r =>
(st -> Int -> Element -> Cursor -> (st -> Cursor -> r) -> r)
-> (st -> Cursor -> r) -> st -> Cursor -> r
eachItem Items s
-> Int -> Element -> Cursor -> (Items s -> Cursor -> r) -> r
forall {p}.
Items s -> p -> Element -> Cursor -> (Items s -> Cursor -> r) -> r
item (\(Items s
acc [Value]
values) -> s -> Value -> Cursor -> r
done s
acc (Array -> Value
Array ([Value] -> Array
forall a. [a] -> Vector a
V.fromList ([Value] -> [Value]
forall a. [a] -> [a]
reverse [Value]
values)))) (s -> [Value] -> Items s
forall s. s -> [Value] -> Items s
Items s
acc0 []) Cursor
tokens
      where
        item :: Items s -> p -> Element -> Cursor -> (Items s -> Cursor -> r) -> r
item (Items s
acc [Value]
values) p
_ Element
element Cursor
rest Items s -> Cursor -> r
next =
            Trees
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built Trees -> InternTable -> Cursor -> r)
-> r
forall b r.
Build b r =>
b
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built b -> InternTable -> Cursor -> r)
-> r
readShape Trees
Trees (Int -> Shape
Scalar Int
budget) Mode
Keep InternTable
table Element
element Cursor
rest ((Built Trees -> InternTable -> Cursor -> r) -> r)
-> (Built Trees -> InternTable -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Built Trees
value InternTable
_ Cursor
afterValue ->
                s -> PyPIField -> (s -> r) -> r
emit s
acc (Text -> PyPIField
EnvelopeItem Text
key) (\s
counted -> Items s -> Cursor -> r
next (s -> [Value] -> Items s
forall s. s -> [Value] -> Items s
Items s
counted (Value
Built Trees
value Value -> [Value] -> [Value]
forall a. a -> [a] -> [a]
: [Value]
values)) Cursor
afterValue)

    -- A repeated array's items are read as its first occurrence's are, so the nesting limit holds, and none is kept.
    dropItems :: Int -> InternTable -> Cursor -> (Cursor -> r) -> r
dropItems Int
budget InternTable
table Cursor
tokens Cursor -> r
done = (() -> Int -> Element -> Cursor -> (() -> Cursor -> r) -> r)
-> (() -> Cursor -> r) -> () -> Cursor -> r
forall r st.
Walk r =>
(st -> Int -> Element -> Cursor -> (st -> Cursor -> r) -> r)
-> (st -> Cursor -> r) -> st -> Cursor -> r
eachItem () -> Int -> Element -> Cursor -> (() -> Cursor -> r) -> r
forall {r} {p} {p}.
Walk r =>
p -> p -> Element -> Cursor -> (() -> Cursor -> r) -> r
item ((Cursor -> r) -> () -> Cursor -> r
forall a b. a -> b -> a
const Cursor -> r
done) () Cursor
tokens
      where
        item :: p -> p -> Element -> Cursor -> (() -> Cursor -> r) -> r
item p
_ p
_ Element
element Cursor
rest () -> Cursor -> r
next = Trees
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built Trees -> InternTable -> Cursor -> r)
-> r
forall b r.
Build b r =>
b
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built b -> InternTable -> Cursor -> r)
-> r
readShape Trees
Trees (Int -> Shape
Scalar Int
budget) Mode
Keep InternTable
table Element
element Cursor
rest (\Built Trees
_ InternTable
_ -> () -> Cursor -> r
next ())

    -- The first array claims the key and its end yields an ignored field. Null claims it empty.
    shaped :: (Bool -> PyPIField)
-> ((Walked s -> Cursor -> r) -> Walked s -> Cursor -> r)
-> Walked s
-> Cursor
-> (Walked s -> Cursor -> r)
-> r
shaped Bool -> PyPIField
claim (Walked s -> Cursor -> r) -> Walked s -> Cursor -> r
items (Walked InternTable
table s
acc) Cursor
after Walked s -> Cursor -> r
continue = Cursor -> (Element -> Cursor -> r) -> r
forall r. Walk r => Cursor -> (Element -> Cursor -> r) -> r
withElement Cursor
after ((Element -> Cursor -> r) -> r) -> (Element -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Element
element Cursor
rest -> case Element
element of
        Element
ArrayBegin -> s -> PyPIField -> (s -> r) -> r
emit s
acc (Bool -> PyPIField
claim Bool
True) ((s -> r) -> r) -> (s -> r) -> r
forall a b. (a -> b) -> a -> b
$ \s
claimed ->
            (Walked s -> Cursor -> r) -> Walked s -> Cursor -> r
items (\(Walked InternTable
table' s
acc') Cursor
afterArray -> s -> PyPIField -> (s -> r) -> r
emit s
acc' PyPIField
IgnoredField (\s
ended -> Walked s -> Cursor -> r
continue (InternTable -> s -> Walked s
forall s. InternTable -> s -> Walked s
Walked InternTable
table' s
ended) Cursor
afterArray)) (InternTable -> s -> Walked s
forall s. InternTable -> s -> Walked s
Walked InternTable
table s
claimed) Cursor
rest
        JValue Value
Null -> s -> PyPIField -> (s -> r) -> r
emit s
acc (Bool -> PyPIField
claim Bool
True) (\s
claimed -> Walked s -> Cursor -> r
continue (InternTable -> s -> Walked s
forall s. InternTable -> s -> Walked s
Walked InternTable
table s
claimed) Cursor
rest)
        Element
_ -> Element -> Cursor -> (Cursor -> r) -> r
forall r. Walk r => Element -> Cursor -> (Cursor -> r) -> r
skipFrom Element
element Cursor
rest (\Cursor
afterValue -> s -> PyPIField -> (s -> r) -> r
emit s
acc (Bool -> PyPIField
claim Bool
False) (\s
marked -> Walked s -> Cursor -> r
continue (InternTable -> s -> Walked s
forall s. InternTable -> s -> Walked s
Walked InternTable
table s
marked) Cursor
afterValue))

    -- A selected read carries its filename memo from file to file of one array.
    files :: (Walked s -> Cursor -> r) -> Walked s -> Cursor -> r
files = case PyPIRead
mode of
        PyPIRead
FullRead -> (Walked s
 -> Int -> Element -> Cursor -> (Walked s -> Cursor -> r) -> r)
-> (Walked s -> Cursor -> r) -> Walked s -> Cursor -> r
forall r st.
Walk r =>
(st -> Int -> Element -> Cursor -> (st -> Cursor -> r) -> r)
-> (st -> Cursor -> r) -> st -> Cursor -> r
eachItem Walked s
-> Int -> Element -> Cursor -> (Walked s -> Cursor -> r) -> r
file
        SelectedRead PackageName
name Text
wanted -> PackageName
-> Text -> (Walked s -> Cursor -> r) -> Walked s -> Cursor -> r
selected PackageName
name Text
wanted
        MirrorRead PackageName
name Text
wanted -> PackageName
-> Text -> (Walked s -> Cursor -> r) -> Walked s -> Cursor -> r
selected PackageName
name Text
wanted

    selected :: PackageName
-> Text -> (Walked s -> Cursor -> r) -> Walked s -> Cursor -> r
selected PackageName
name Text
wanted Walked s -> Cursor -> r
done (Walked InternTable
table s
acc) =
        (Selecting s
 -> Int -> Element -> Cursor -> (Selecting s -> Cursor -> r) -> r)
-> (Selecting s -> Cursor -> r) -> Selecting s -> Cursor -> r
forall r st.
Walk r =>
(st -> Int -> Element -> Cursor -> (st -> Cursor -> r) -> r)
-> (st -> Cursor -> r) -> st -> Cursor -> r
eachItem (Selection
-> Selecting s
-> Int
-> Element
-> Cursor
-> (Selecting s -> Cursor -> r)
-> r
candidate Selection{selWanted :: Text
selWanted = Text
wanted, selBudget :: Int
selBudget = Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
3, selFields :: Members
selFields = Members
fileFields, selHashes :: Members
selHashes = Members
hashFields}) (\(Selecting InternTable
table' FilenameMemo
_ s
acc') -> Walked s -> Cursor -> r
done (InternTable -> s -> Walked s
forall s. InternTable -> s -> Walked s
Walked InternTable
table' s
acc')) (InternTable -> FilenameMemo -> s -> Selecting s
forall a. InternTable -> FilenameMemo -> a -> Selecting a
Selecting InternTable
table (PackageName -> FilenameMemo
filenameMemo PackageName
name) s
acc)

    file :: Walked s
-> Int -> Element -> Cursor -> (Walked s -> Cursor -> r) -> r
file (Walked InternTable
table s
acc) Int
position Element
element Cursor
rest Walked s -> Cursor -> r
continue
        | Int
depth Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
<= Int
2 = Element -> Cursor -> r
forall r. Walk r => Element -> Cursor -> r
tooDeep Element
element Cursor
rest
        | Bool
otherwise = InternTable
-> s
-> Int
-> Element
-> Cursor
-> (s -> InternTable -> Cursor -> r)
-> r
readIndexFile InternTable
table s
acc Int
position Element
element Cursor
rest (\s
kept InternTable
held Cursor
after -> Walked s -> Cursor -> r
continue (InternTable -> s -> Walked s
forall s. InternTable -> s -> Walked s
Walked InternTable
held s
kept) Cursor
after)

    candidate :: Selection
-> Selecting s
-> Int
-> Element
-> Cursor
-> (Selecting s -> Cursor -> r)
-> r
candidate Selection
selection (Selecting InternTable
table FilenameMemo
memo s
acc) Int
position Element
element Cursor
rest Selecting s -> Cursor -> r
continue
        | Int
depth Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
<= Int
2 = Element -> Cursor -> r
forall r. Walk r => Element -> Cursor -> r
tooDeep Element
element Cursor
rest
        | Bool
otherwise = Selection
-> FilenameMemo
-> InternTable
-> Element
-> Cursor
-> (Maybe Value -> FilenameMemo -> InternTable -> Cursor -> r)
-> r
forall r.
Walk r =>
Selection
-> FilenameMemo
-> InternTable
-> Element
-> Cursor
-> (Maybe Value -> FilenameMemo -> InternTable -> Cursor -> r)
-> r
selectedFile Selection
selection FilenameMemo
memo InternTable
table Element
element Cursor
rest ((Maybe Value -> FilenameMemo -> InternTable -> Cursor -> r) -> r)
-> (Maybe Value -> FilenameMemo -> InternTable -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Maybe Value
payload FilenameMemo
memo' InternTable
table' Cursor
afterValue ->
            s -> PyPIField -> (s -> r) -> r
emit s
acc (Int -> Maybe Value -> PyPIField
FileField Int
position Maybe Value
payload) (\s
acc' -> Selecting s -> Cursor -> r
continue (InternTable -> FilenameMemo -> s -> Selecting s
forall a. InternTable -> FilenameMemo -> a -> Selecting a
Selecting InternTable
table' FilenameMemo
memo' s
acc') Cursor
afterValue)

    version :: Walked s
-> Int -> Element -> Cursor -> (Walked s -> Cursor -> r) -> r
version (Walked InternTable
table s
acc) Int
position Element
element Cursor
rest Walked s -> Cursor -> r
continue =
        Trees
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built Trees -> InternTable -> Cursor -> r)
-> r
forall b r.
Build b r =>
b
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built b -> InternTable -> Cursor -> r)
-> r
readShape Trees
Trees (Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2)) Mode
Keep InternTable
table Element
element Cursor
rest ((Built Trees -> InternTable -> Cursor -> r) -> r)
-> (Built Trees -> InternTable -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Built Trees
value InternTable
_ Cursor
afterValue ->
            let field :: PyPIField
field = case Built Trees
value of
                    String Text
_ -> PyPIField
IgnoredField
                    Built Trees
_ -> Int -> Value -> PyPIField
InvalidVersionField Int
position Value
Built Trees
value
             in s -> PyPIField -> (s -> r) -> r
emit s
acc PyPIField
field (\s
acc' -> Walked s -> Cursor -> r
continue (InternTable -> s -> Walked s
forall s. InternTable -> s -> Walked s
Walked InternTable
table s
acc') Cursor
afterValue)

    fileFields :: Members
fileFields = Int -> (Members -> Members) -> Members
fileMembers Int
depth Members -> Members
forall a. a -> a
id
    hashFields :: Members
hashFields = [Text] -> Shape -> Members
knownMembers [Text]
hashNames (Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
4))
    metaFields :: Members
metaFields = [(Text, Shape)] -> Members
namedMembers ([(Text
"tracks", Int -> Shape -> Shape -> Shape
ArrayWith (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) (Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
3)) (Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2))) | Bool
full] [(Text, Shape)] -> [(Text, Shape)] -> [(Text, Shape)]
forall a. Semigroup a => a -> a -> a
<> [(Text
"api-version", Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2))] [(Text, Shape)] -> [(Text, Shape)] -> [(Text, Shape)]
forall a. Semigroup a => a -> a -> a
<> [(Text
"_last-serial", Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2)) | Bool
full])
    statusFields :: Members
statusFields = [(Text, Shape)] -> Members
namedMembers [(Text
key, Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2)) | Text
key <- [Text
"status", Text
"reason"]]

fileMembers :: Int -> (Members -> Members) -> Members
fileMembers :: Int -> (Members -> Members) -> Members
fileMembers Int
depth Members -> Members
prepare = Members -> Members
prepare ([(Text, Shape)] -> Members
namedMembers ([(Text, Shape)]
digestMembers [(Text, Shape)] -> [(Text, Shape)] -> [(Text, Shape)]
forall a. Semigroup a => a -> a -> a
<> [(Text, Shape)]
scalarMembers))
  where
    digestMembers :: [(Text, Shape)]
digestMembers = [(Text
key, Int -> Members -> Shape -> Shape
ObjectWith (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
3) Members
hashFields (Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
3))) | Text
key <- [Text
"hashes", Text
"core-metadata"]]
    scalarMembers :: [(Text, Shape)]
scalarMembers = [(Text
key, Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
3)) | Text
key <- [Text]
fileScalars]
    hashFields :: Members
hashFields = Members -> Members
prepare ([Text] -> Shape -> Members
knownMembers [Text]
hashNames (Int -> Shape
Scalar (Int
depth Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
4)))

data Selection = Selection
    { Selection -> Text
selWanted :: Text
    , Selection -> Int
selBudget :: Int
    , Selection -> Members
selFields :: Members
    , Selection -> Members
selHashes :: Members
    }

-- Keep the filename memo scoped to the current files array.
data Selecting a = Selecting !InternTable !FilenameMemo a

-- The consumer's state beside an envelope object's members so far.
data Fields s = Fields s (KeyMap.KeyMap Value)

-- The consumer's state beside an envelope array's items so far, newest first.
data Items s = Items s [Value]

-- After a rejecting filename, only the lexer checks the remaining members.
selectedFile :: (Walk r) => Selection -> FilenameMemo -> InternTable -> Element -> Cursor -> (Maybe Value -> FilenameMemo -> InternTable -> Cursor -> r) -> r
{-# INLINEABLE selectedFile #-}
selectedFile :: forall r.
Walk r =>
Selection
-> FilenameMemo
-> InternTable
-> Element
-> Cursor
-> (Maybe Value -> FilenameMemo -> InternTable -> Cursor -> r)
-> r
selectedFile Selection{selWanted :: Selection -> Text
selWanted = Text
wanted, selBudget :: Selection -> Int
selBudget = Int
budget, selFields :: Selection -> Members
selFields = Members
fields, selHashes :: Selection -> Members
selHashes = Members
hashes} FilenameMemo
memo0 InternTable
table0 Element
element Cursor
rest Maybe Value -> FilenameMemo -> InternTable -> Cursor -> r
next = case Element
element of
    Element
ObjectBegin -> (Selecting SelectedFile
 -> Name -> Cursor -> (Selecting SelectedFile -> Cursor -> r) -> r)
-> (Selecting SelectedFile -> Cursor -> r)
-> Selecting SelectedFile
-> Cursor
-> r
forall r st.
Walk r =>
(st -> Name -> Cursor -> (st -> Cursor -> r) -> r)
-> (st -> Cursor -> r) -> st -> Cursor -> r
eachMember Selecting SelectedFile
-> Name -> Cursor -> (Selecting SelectedFile -> Cursor -> r) -> r
visit (\(Selecting InternTable
table FilenameMemo
memo SelectedFile
file) Cursor
after -> Maybe Value -> FilenameMemo -> InternTable -> Cursor -> r
next (SelectedFile -> Maybe Value
finishSelected SelectedFile
file) FilenameMemo
memo InternTable
table Cursor
after) (InternTable
-> FilenameMemo -> SelectedFile -> Selecting SelectedFile
forall a. InternTable -> FilenameMemo -> a -> Selecting a
Selecting InternTable
table0 FilenameMemo
memo0 (Bool -> [(Key, Value)] -> Maybe Value -> Bool -> SelectedFile
CandidateFile Bool
False [] Maybe Value
forall a. Maybe a
Nothing Bool
False)) Cursor
rest
    Element
_ -> Element -> Cursor -> (Cursor -> r) -> r
forall r. Walk r => Element -> Cursor -> (Cursor -> r) -> r
skipFrom Element
element Cursor
rest (Maybe Value -> FilenameMemo -> InternTable -> Cursor -> r
next Maybe Value
forall a. Maybe a
Nothing FilenameMemo
memo0 InternTable
table0)
  where
    collect :: InternTable
-> FilenameMemo
-> SelectedFile
-> SelectedFileEvent
-> Selecting SelectedFile
collect InternTable
table FilenameMemo
memo SelectedFile
file SelectedFileEvent
event = case Text
-> FilenameMemo
-> SelectedFile
-> SelectedFileEvent
-> (SelectedFile, FilenameMemo)
collectSelected Text
wanted FilenameMemo
memo SelectedFile
file SelectedFileEvent
event of
        (SelectedFile
collected, FilenameMemo
remembered) -> InternTable
-> FilenameMemo -> SelectedFile -> Selecting SelectedFile
forall a. InternTable -> FilenameMemo -> a -> Selecting a
Selecting InternTable
table FilenameMemo
remembered SelectedFile
collected
    -- A budget with no level for a hash value reads a rejected file on, so its hashes meet the nesting limit.
    skipsRejected :: Bool
skipsRejected = Int
budget Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Int
1
    -- Selected files keep their own text and do not enter the intern table.
    visit :: Selecting SelectedFile
-> Name -> Cursor -> (Selecting SelectedFile -> Cursor -> r) -> r
visit (Selecting InternTable
table FilenameMemo
memo SelectedFile
file) Name
name Cursor
after Selecting SelectedFile -> Cursor -> r
continue = Cursor -> (Element -> Cursor -> r) -> r
forall r. Walk r => Cursor -> (Element -> Cursor -> r) -> r
withElement Cursor
after ((Element -> Cursor -> r) -> r) -> (Element -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Element
value Cursor
afterKey -> case Name -> ByteString
nameBytes Name
name of
        ByteString
"hashes"
            | Int
budget Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
<= Int
0 -> Element -> Cursor -> r
forall r. Walk r => Element -> Cursor -> r
tooDeep Element
value Cursor
afterKey
            | Element
ObjectBegin <- Element
value ->
                (Selecting SelectedFile
 -> Name -> Cursor -> (Selecting SelectedFile -> Cursor -> r) -> r)
-> (Selecting SelectedFile -> Cursor -> r)
-> Selecting SelectedFile
-> Cursor
-> r
forall r st.
Walk r =>
(st -> Name -> Cursor -> (st -> Cursor -> r) -> r)
-> (st -> Cursor -> r) -> st -> Cursor -> r
eachMember
                    Selecting SelectedFile
-> Name -> Cursor -> (Selecting SelectedFile -> Cursor -> r) -> r
forall {r}.
Walk r =>
Selecting SelectedFile
-> Name -> Cursor -> (Selecting SelectedFile -> Cursor -> r) -> r
hashField
                    (\(Selecting InternTable
table' FilenameMemo
memo' SelectedFile
hashed) Cursor
afterObject -> Selecting SelectedFile -> Cursor -> r
continue (InternTable
-> FilenameMemo
-> SelectedFile
-> SelectedFileEvent
-> Selecting SelectedFile
collect InternTable
table' FilenameMemo
memo' SelectedFile
hashed SelectedFileEvent
HashesEnd) Cursor
afterObject)
                    (InternTable
-> FilenameMemo
-> SelectedFile
-> SelectedFileEvent
-> Selecting SelectedFile
collect InternTable
table FilenameMemo
memo SelectedFile
file SelectedFileEvent
HashesStart)
                    Cursor
afterKey
            | Bool
otherwise -> Trees
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built Trees -> InternTable -> Cursor -> r)
-> r
forall b r.
Build b r =>
b
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built b -> InternTable -> Cursor -> r)
-> r
readShape Trees
Trees (Int -> Shape
Scalar Int
budget) Mode
Keep InternTable
table Element
value Cursor
afterKey ((Built Trees -> InternTable -> Cursor -> r) -> r)
-> (Built Trees -> InternTable -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Built Trees
scalar InternTable
table' Cursor
afterValue ->
                Selecting SelectedFile -> Cursor -> r
continue (InternTable
-> FilenameMemo
-> SelectedFile
-> SelectedFileEvent
-> Selecting SelectedFile
collect InternTable
table' FilenameMemo
memo SelectedFile
file (Value -> SelectedFileEvent
HashesValue Value
Built Trees
scalar)) Cursor
afterValue
        ByteString
"core-metadata" -> Int
-> Members
-> InternTable
-> Element
-> Cursor
-> (Value -> InternTable -> Cursor -> r)
-> r
forall r.
Walk r =>
Int
-> Members
-> InternTable
-> Element
-> Cursor
-> (Value -> InternTable -> Cursor -> r)
-> r
readPendingSidecar Int
budget Members
hashes InternTable
table Element
value Cursor
afterKey ((Value -> InternTable -> Cursor -> r) -> r)
-> (Value -> InternTable -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Value
pending InternTable
table' Cursor
afterValue ->
            Selecting SelectedFile -> Cursor -> r
continue (InternTable
-> FilenameMemo
-> SelectedFile
-> SelectedFileEvent
-> Selecting SelectedFile
collect InternTable
table' FilenameMemo
memo SelectedFile
file (Key -> Value -> SelectedFileEvent
FileScalar Key
"core-metadata" Value
pending)) Cursor
afterValue
        ByteString
_ -> case Members -> Name -> Maybe (Key, Shape)
listedMember Members
fields Name
name of
            Just (Key
key, Shape
shape) -> Trees
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built Trees -> InternTable -> Cursor -> r)
-> r
forall b r.
Build b r =>
b
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built b -> InternTable -> Cursor -> r)
-> r
readShape Trees
Trees Shape
shape Mode
Keep InternTable
table Element
value Cursor
afterKey ((Built Trees -> InternTable -> Cursor -> r) -> r)
-> (Built Trees -> InternTable -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Built Trees
scalar InternTable
table' Cursor
afterValue ->
                case InternTable
-> FilenameMemo
-> SelectedFile
-> SelectedFileEvent
-> Selecting SelectedFile
collect InternTable
table' FilenameMemo
memo SelectedFile
file (Key -> Value -> SelectedFileEvent
FileScalar Key
key Value
Built Trees
scalar) of
                    Selecting InternTable
held FilenameMemo
remembered SelectedFile
RejectedFile | Bool
skipsRejected -> Int -> Cursor -> (Cursor -> r) -> r
forall r. Walk r => Int -> Cursor -> (Cursor -> r) -> r
skipRest Int
1 Cursor
afterValue (Maybe Value -> FilenameMemo -> InternTable -> Cursor -> r
next Maybe Value
forall a. Maybe a
Nothing FilenameMemo
remembered InternTable
held)
                    Selecting SelectedFile
collected -> Selecting SelectedFile -> Cursor -> r
continue Selecting SelectedFile
collected Cursor
afterValue
            Maybe (Key, Shape)
Nothing -> Element -> Cursor -> (Cursor -> r) -> r
forall r. Walk r => Element -> Cursor -> (Cursor -> r) -> r
skipFrom Element
value Cursor
afterKey (Selecting SelectedFile -> Cursor -> r
continue (InternTable
-> FilenameMemo -> SelectedFile -> Selecting SelectedFile
forall a. InternTable -> FilenameMemo -> a -> Selecting a
Selecting InternTable
table FilenameMemo
memo SelectedFile
file))
    hashField :: Selecting SelectedFile
-> Name -> Cursor -> (Selecting SelectedFile -> Cursor -> r) -> r
hashField (Selecting InternTable
table FilenameMemo
memo SelectedFile
file) Name
name Cursor
after Selecting SelectedFile -> Cursor -> r
continue = Cursor -> (Element -> Cursor -> r) -> r
forall r. Walk r => Cursor -> (Element -> Cursor -> r) -> r
withElement Cursor
after ((Element -> Cursor -> r) -> r) -> (Element -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Element
value Cursor
afterKey ->
        Trees
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built Trees -> InternTable -> Cursor -> r)
-> r
forall b r.
Build b r =>
b
-> Shape
-> Mode
-> InternTable
-> Element
-> Cursor
-> (Built b -> InternTable -> Cursor -> r)
-> r
readShape Trees
Trees (Int -> Shape
Scalar (Int
budget Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1)) Mode
Keep InternTable
table Element
value Cursor
afterKey ((Built Trees -> InternTable -> Cursor -> r) -> r)
-> (Built Trees -> InternTable -> Cursor -> r) -> r
forall a b. (a -> b) -> a -> b
$ \Built Trees
scalar InternTable
table' Cursor
afterValue ->
            Selecting SelectedFile -> Cursor -> r
continue (InternTable
-> FilenameMemo
-> SelectedFile
-> SelectedFileEvent
-> Selecting SelectedFile
collect InternTable
table' FilenameMemo
memo SelectedFile
file (Key -> Value -> SelectedFileEvent
HashField (Key -> ((Key, Shape) -> Key) -> Maybe (Key, Shape) -> Key
forall b a. b -> (a -> b) -> Maybe a -> b
maybe (Text -> Key
Key.fromText (Name -> Text
nameText Name
name)) (Key, Shape) -> Key
forall a b. (a, b) -> a
fst (Members -> Name -> Maybe (Key, Shape)
listedMember Members
hashes Name
name)) Value
Built Trees
scalar)) Cursor
afterValue