module Ecluse.Core.Registry.PyPI.Mirror (
pypiMirrorDocuments,
pypiStateFormat,
) where
import Data.Aeson (Value (Object, String), object, (.:), (.=))
import Data.Aeson.KeyMap qualified as KeyMap
import Data.Map.Strict qualified as Map
import Data.Primitive.SmallArray (smallArrayFromList)
import Ecluse.Core.Ecosystem (Ecosystem (PyPI))
import Ecluse.Core.Mirror.Documents (MirrorDocuments (..), restoreFragments)
import Ecluse.Core.Mirror.State (MirrorVersion, recordEnvelope)
import Ecluse.Core.Mirror.State.Codec (forceSourceEntries, parseSourceEntries, sourceEntries, sourceEntriesCharge)
import Ecluse.Core.Mirror.State.Format (MirrorStateFormat (..), ProjectEnvelope (..))
import Ecluse.Core.Mirror.State.Weight (fragmentShellAllowance, sourceEntriesWeight, valueBackingExcess)
import Ecluse.Core.Package (Artifact (..), PackageDetails (pkgArtifacts, pkgName), pkgEcosystem)
import Ecluse.Core.Package.Entry (EntryKey (ArrayEntry))
import Ecluse.Core.Registry.CachedDocument (pypiPacked, pypiSimpleCached)
import Ecluse.Core.Registry.Json.Packed (docTable)
import Ecluse.Core.Registry.Json.Writer (packTree)
import Ecluse.Core.Registry.Metadata (VersionDoc (..))
import Ecluse.Core.Registry.MirrorUpload (MirrorReadData (PyPIMirrorRead))
import Ecluse.Core.Registry.MirrorUpload.Internal (restoreMirrorFragment)
import Ecluse.Core.Registry.PyPI.Document (packedSimple, simpleDocument, simpleEnvelope, simpleFiles, simpleSourceCompanions, urlHole)
import Ecluse.Core.Server.Cache.VersionWeight.Fields (retainedRawWeight)
pypiMirrorDocuments :: MirrorDocuments
pypiMirrorDocuments :: MirrorDocuments
pypiMirrorDocuments =
MirrorDocuments
{ mdRestore :: MirrorVersion -> [Artifact] -> Maybe VersionDoc
mdRestore = MirrorVersion -> [Artifact] -> Maybe VersionDoc
restoreSimpleVersion
, mdOrder :: [(Text, VersionDoc)] -> [(Text, VersionDoc)]
mdOrder = [(Text, VersionDoc)] -> [(Text, VersionDoc)]
forall a. a -> a
id
, mdEntryKey :: Int -> EntryKey
mdEntryKey = Int -> EntryKey
ArrayEntry
, mdRekey :: (EntryKey -> EntryKey) -> CachedDoc -> Maybe CachedDoc
mdRekey = \EntryKey -> EntryKey
rekey CachedDoc
raw -> do
simple <- (SimpleDocument -> CachedDoc, CachedDoc -> Maybe SimpleDocument)
-> CachedDoc -> Maybe SimpleDocument
forall a b. (a, b) -> b
snd (SimpleDocument -> CachedDoc, CachedDoc -> Maybe SimpleDocument)
pypiSimpleCached CachedDoc
raw
pure (fst pypiSimpleCached (simpleDocument (simpleEnvelope simple) [(rekey key, value) | (key, value) <- simpleFiles simple]))
, mdAssemble :: ProjectMirrorRecord -> [VersionDoc] -> CachedDoc
mdAssemble = \ProjectMirrorRecord
record [VersionDoc]
docs ->
let files :: [(EntryKey, Value)]
files = [[(EntryKey, Value)]] -> [(EntryKey, Value)]
forall (t :: * -> *) a. Foldable t => t [a] -> [a]
concat [SimpleDocument -> [(EntryKey, Value)]
simpleFiles SimpleDocument
document | VersionDoc
doc <- [VersionDoc]
docs, Just CachedDoc
raw <- [VersionDoc -> Maybe CachedDoc
vdRaw VersionDoc
doc], Just SimpleDocument
document <- [(SimpleDocument -> CachedDoc, CachedDoc -> Maybe SimpleDocument)
-> CachedDoc -> Maybe SimpleDocument
forall a b. (a, b) -> b
snd (SimpleDocument -> CachedDoc, CachedDoc -> Maybe SimpleDocument)
pypiSimpleCached CachedDoc
raw]]
in (PackedSimple -> CachedDoc, CachedDoc -> Maybe PackedSimple)
-> PackedSimple -> CachedDoc
forall a b. (a, b) -> a
fst (PackedSimple -> CachedDoc, CachedDoc -> Maybe PackedSimple)
pypiPacked (Object -> DocTable -> [(EntryKey, Packed)] -> PackedSimple
packedSimple (ProjectMirrorRecord -> Object
recordEnvelope ProjectMirrorRecord
record) (SmallArray Text -> DocTable
docTable ([Text] -> SmallArray Text
forall a. [a] -> SmallArray a
smallArrayFromList [])) [(EntryKey
key, [Text] -> Value -> Packed
packTree [Text]
urlHole Value
value) | (EntryKey
key, Value
value) <- [(EntryKey, Value)]
files])
}
pypiStateFormat :: MirrorStateFormat
pypiStateFormat :: MirrorStateFormat
pypiStateFormat =
MirrorStateFormat
{ msfKind :: Text
msfKind = Text
"pypi"
, msfEncodeSource :: MirrorReadData -> Maybe Value
msfEncodeSource = \case
PyPIMirrorRead SimpleDocument
document -> Value -> Maybe Value
forall a. a -> Maybe a
Just ([Pair] -> Value
object [Key
"kind" Key -> Text -> Pair
forall v. ToJSON v => Key -> v -> Pair
forall e kv v. (KeyValue e kv, ToJSON v) => Key -> v -> kv
.= (Text
"pypi" :: Text), Key
"envelope" Key -> Object -> Pair
forall v. ToJSON v => Key -> v -> Pair
forall e kv v. (KeyValue e kv, ToJSON v) => Key -> v -> kv
.= SimpleDocument -> Object
simpleEnvelope SimpleDocument
document, Key
"entries" Key -> [Value] -> Pair
forall v. ToJSON v => Key -> v -> Pair
forall e kv v. (KeyValue e kv, ToJSON v) => Key -> v -> kv
.= [(EntryKey, Value)] -> [Value]
sourceEntries (SimpleDocument -> [(EntryKey, Value)]
simpleFiles SimpleDocument
document)])
MirrorReadData
_ -> Maybe Value
forall a. Maybe a
Nothing
, msfDecodeSource :: Object -> Parser MirrorReadData
msfDecodeSource = \Object
value -> do
envelope <- Object
value Object -> Key -> Parser Object
forall a. FromJSON a => Object -> Key -> Parser a
.: Key
"envelope"
PyPIMirrorRead . simpleDocument envelope <$> parseSourceEntries "file entry" value
, msfDecodeCharge :: Value -> Integer
msfDecodeCharge = Value -> Integer
sourceEntriesCharge
, msfForceSource :: MirrorReadData -> ()
msfForceSource = \case
PyPIMirrorRead SimpleDocument
document -> Object -> Object
forall a. NFData a => a -> a
force (SimpleDocument -> Object
simpleEnvelope SimpleDocument
document) Object -> () -> ()
forall a b. a -> b -> b
`seq` [(EntryKey, Value)] -> ()
forceSourceEntries (SimpleDocument -> [(EntryKey, Value)]
simpleFiles SimpleDocument
document)
MirrorReadData
_ -> ()
, msfEnvelope :: ProjectEnvelope
msfEnvelope =
ProjectEnvelope
{ envelopeSplit :: MirrorReadData -> (Object, MirrorReadData)
envelopeSplit = \case
PyPIMirrorRead SimpleDocument
document -> (SimpleDocument -> Object
simpleEnvelope SimpleDocument
document, SimpleDocument -> MirrorReadData
PyPIMirrorRead (Object -> [(EntryKey, Value)] -> SimpleDocument
simpleDocument Object
forall a. Monoid a => a
mempty (SimpleDocument -> [(EntryKey, Value)]
simpleFiles SimpleDocument
document)))
MirrorReadData
other -> (Object
forall a. Monoid a => a
mempty, MirrorReadData
other)
, envelopeCheck :: Object -> PackageDetails -> MirrorReadData -> Either Text ()
envelopeCheck = \Object
envelope PackageDetails
details -> \case
PyPIMirrorRead SimpleDocument
document -> (PublishFault -> Text) -> Either PublishFault () -> Either Text ()
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first (Text -> PublishFault -> Text
forall a b. a -> b -> a
const Text
"invalid project envelope identity") (Either PublishFault (NonEmpty ExpectedMirrorFile)
-> Either PublishFault ()
forall (f :: * -> *) a. Functor f => f a -> f ()
void (PackageDetails
-> MirrorReadData
-> Either PublishFault (NonEmpty ExpectedMirrorFile)
restoreMirrorFragment PackageDetails
details (SimpleDocument -> MirrorReadData
PyPIMirrorRead (Object -> [(EntryKey, Value)] -> SimpleDocument
simpleDocument Object
envelope (SimpleDocument -> [(EntryKey, Value)]
simpleFiles SimpleDocument
document)))))
MirrorReadData
_ -> Text -> Either Text ()
forall a b. a -> Either a b
Left Text
"invalid project envelope identity"
}
,
msfFragmentShell :: PackageDetails -> Integer
msfFragmentShell = \PackageDetails
details -> if PackageName -> Ecosystem
pkgEcosystem (PackageDetails -> PackageName
pkgName PackageDetails
details) Ecosystem -> Ecosystem -> Bool
forall a. Eq a => a -> a -> Bool
== Ecosystem
PyPI then Integer
768 else Integer
fragmentShellAllowance
, msfSourceWeight :: MirrorReadData -> Integer
msfSourceWeight = \case
PyPIMirrorRead SimpleDocument
document -> CachedDoc -> Integer
retainedRawWeight ((SimpleDocument -> CachedDoc, CachedDoc -> Maybe SimpleDocument)
-> SimpleDocument -> CachedDoc
forall a b. (a, b) -> a
fst (SimpleDocument -> CachedDoc, CachedDoc -> Maybe SimpleDocument)
pypiSimpleCached SimpleDocument
document) Integer -> Integer -> Integer
forall a. Num a => a -> a -> a
+ Value -> Integer
valueBackingExcess (Object -> Value
Object (SimpleDocument -> Object
simpleEnvelope SimpleDocument
document)) Integer -> Integer -> Integer
forall a. Num a => a -> a -> a
+ [(EntryKey, Value)] -> Integer
sourceEntriesWeight (SimpleDocument -> [(EntryKey, Value)]
simpleFiles SimpleDocument
document)
MirrorReadData
_ -> Integer
0
, msfCompanions :: CompanionRequirement
msfCompanions = CompanionRequirement
simpleSourceCompanions
}
restoreSimpleVersion :: MirrorVersion -> [Artifact] -> Maybe VersionDoc
restoreSimpleVersion :: MirrorVersion -> [Artifact] -> Maybe VersionDoc
restoreSimpleVersion MirrorVersion
mirror [Artifact]
current = do
fragments@((firstDetails, (), _) :| _) <- (Int -> EntryKey)
-> (MirrorReadData -> Maybe ((), Map EntryKey Value))
-> MirrorVersion
-> [Artifact]
-> Maybe
(NonEmpty (PackageDetails, (), NonEmpty (Artifact, Value)))
forall source entry.
(Int -> EntryKey)
-> (MirrorReadData -> Maybe (source, Map EntryKey entry))
-> MirrorVersion
-> [Artifact]
-> Maybe
(NonEmpty (PackageDetails, source, NonEmpty (Artifact, entry)))
restoreFragments Int -> EntryKey
ArrayEntry MirrorReadData -> Maybe ((), Map EntryKey Value)
simpleEntries MirrorVersion
mirror [Artifact]
current
let pairs = ((PackageDetails, (), NonEmpty (Artifact, Value))
-> [(Artifact, Value)])
-> NonEmpty (PackageDetails, (), NonEmpty (Artifact, Value))
-> [(Artifact, Value)]
forall (t :: * -> *) a b. Foldable t => (a -> [b]) -> t a -> [b]
concatMap (\(PackageDetails
_, ()
_, NonEmpty (Artifact, Value)
restored) -> NonEmpty (Artifact, Value) -> [(Artifact, Value)]
forall a. NonEmpty a -> [a]
forall (t :: * -> *) a. Foldable t => t a -> [a]
toList NonEmpty (Artifact, Value)
restored) NonEmpty (PackageDetails, (), NonEmpty (Artifact, Value))
fragments
files <- traverse currentUrl pairs
artifacts <- nonEmpty (map fst pairs)
pure (VersionDoc firstDetails{pkgArtifacts = artifacts} (Just (fst pypiSimpleCached (simpleDocument mempty files))))
where
simpleEntries :: MirrorReadData -> Maybe ((), Map EntryKey Value)
simpleEntries = \case
PyPIMirrorRead SimpleDocument
document -> ((), Map EntryKey Value) -> Maybe ((), Map EntryKey Value)
forall a. a -> Maybe a
Just ((), [(EntryKey, Value)] -> Map EntryKey Value
forall k a. Ord k => [(k, a)] -> Map k a
Map.fromList (SimpleDocument -> [(EntryKey, Value)]
simpleFiles SimpleDocument
document))
MirrorReadData
_ -> Maybe ((), Map EntryKey Value)
forall a. Maybe a
Nothing
currentUrl :: (Artifact, Value) -> Maybe (EntryKey, Value)
currentUrl (Artifact
artifact, Value
entry) = case Value
entry of
Object Object
fields -> (EntryKey, Value) -> Maybe (EntryKey, Value)
forall a. a -> Maybe a
Just (Artifact -> EntryKey
artEntryKey Artifact
artifact, Object -> Value
Object (Key -> Value -> Object -> Object
forall v. Key -> v -> KeyMap v -> KeyMap v
KeyMap.insert Key
"url" (Text -> Value
String (Artifact -> Text
artUrl Artifact
artifact)) Object
fields))
Value
_ -> Maybe (EntryKey, Value)
forall a. Maybe a
Nothing