{-# OPTIONS_GHC -fno-full-laziness #-}
module Ecluse.Core.Registry.Npm.Metadata (
npmReadWith,
prepareNpmRead,
npmChargeFactors,
npmPackumentWalk,
NpmFullRead,
npmFullWalk,
npmFullTable,
projectNpmStream,
projectNpmPacked,
selectNpmRead,
selectNpmVersionDoc,
) where
import Control.Monad.ST (ST, stToIO)
import Data.Aeson (Value (Object))
import Data.Aeson.Key qualified as Key
import Data.Aeson.KeyMap qualified as KeyMap
import Data.JsonStream.Lexer (Cursor)
import Data.Map.Strict qualified as Map
import Data.Primitive.PrimVar (PrimVar)
import Ecluse.Core.Package (PackageInfo (..), PackageName, renderPackageName)
import Ecluse.Core.Package.Filter (enforceArtifactLocations, enforceArtifactLocationsOf)
import Ecluse.Core.Registry.CachedDocument (CachedDoc, npmCached, npmPacked)
import Ecluse.Core.Registry.Json.Intern (InternTable, Interned (..), decodedName, internName, tableTexts)
import Ecluse.Core.Registry.Json.Packed (docTable)
import Ecluse.Core.Registry.Json.Shape (Trees (..))
import Ecluse.Core.Registry.Json.Walk (Step, Steps, Walked (..), pureStep, readJsonWalk)
import Ecluse.Core.Registry.JsonStream (StreamResult (..))
import Ecluse.Core.Registry.Metadata (MetadataError, VersionDoc (..), VersionRead (..))
import Ecluse.Core.Registry.Metadata.EntryFloor (countingEntries, readFlooredWalk)
import Ecluse.Core.Registry.Metadata.Fetch.Preparation (PrepareRead (PrepareRead))
import Ecluse.Core.Registry.Metadata.Fetch.Types (DocumentWalk, EcosystemRead (..))
import Ecluse.Core.Registry.Metadata.Projection (streamError)
import Ecluse.Core.Registry.Npm.Document (PackedPackument)
import Ecluse.Core.Registry.Npm.Reader (PackumentRead (..), npmWalk, releaseUniqueFields)
import Ecluse.Core.Registry.Npm.Request (MetadataForm (Full), metadataRequestUnder, npmArtifactHosts, packageUrl)
import Ecluse.Core.Registry.Npm.StreamingProjection (PackedRead, TreeRead, emptyPackedRead, emptyTreeRead, finishPacked, finishTree, keepsPackedRelease, keepsTreeRelease, keptEntries, packedStep, retainedEntries, treeStep)
import Ecluse.Core.Registry.Npm.VersionWriter (VersionWriter, newVersionWriter)
import Ecluse.Core.Registry.Origin (OriginClient (ocChargeFullReadWorkspace))
import Ecluse.Core.Registry.ServedDocument (objectField)
import Ecluse.Core.Security (AllowedHostPorts, Limits, ecosystemArtifactAuthorities, maxNestingDepth)
import Ecluse.Core.Server.Admission.Types (ChargeFactors (..))
import Ecluse.Core.Version (Version, renderVersion)
prepareNpmRead :: PrepareRead
prepareNpmRead :: PrepareRead
prepareNpmRead = (OriginClient -> IO (Either MetadataError EcosystemRead))
-> PrepareRead
PrepareRead (Either MetadataError EcosystemRead
-> IO (Either MetadataError EcosystemRead)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Either MetadataError EcosystemRead
-> IO (Either MetadataError EcosystemRead))
-> (OriginClient -> Either MetadataError EcosystemRead)
-> OriginClient
-> IO (Either MetadataError EcosystemRead)
forall b c a. (b -> c) -> (a -> b) -> a -> c
. EcosystemRead -> Either MetadataError EcosystemRead
forall a b. b -> Either a b
Right (EcosystemRead -> Either MetadataError EcosystemRead)
-> (OriginClient -> EcosystemRead)
-> OriginClient
-> Either MetadataError EcosystemRead
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Int -> IO ()) -> EcosystemRead
npmReadWith ((Int -> IO ()) -> EcosystemRead)
-> (OriginClient -> Int -> IO ()) -> OriginClient -> EcosystemRead
forall b c a. (b -> c) -> (a -> b) -> a -> c
. OriginClient -> Int -> IO ()
ocChargeFullReadWorkspace)
npmReadWith :: (Int -> IO ()) -> EcosystemRead
npmReadWith :: (Int -> IO ()) -> EcosystemRead
npmReadWith Int -> IO ()
pay =
EcosystemRead
{ erRequest :: BaseRequest
-> Maybe ClientCredential
-> PackageName
-> Either UrlFormationError Request
erRequest = \BaseRequest
base Maybe ClientCredential
token -> BaseRequest
-> Maybe ClientCredential
-> MetadataForm
-> PackageName
-> Either UrlFormationError Request
metadataRequestUnder BaseRequest
base Maybe ClientCredential
token MetadataForm
Full
, erUniqueFields :: [Text]
erUniqueFields = [Text]
releaseUniqueFields
, erWalkFull :: Limits -> PackageName -> Text -> DocumentWalk NpmFullRead
erWalkFull = (Int -> IO ())
-> Limits -> PackageName -> Text -> DocumentWalk NpmFullRead
readNpmFull Int -> IO ()
pay
, erFinishFull :: Limits
-> PackageName
-> Text
-> StreamResult NpmFullRead
-> Either MetadataError (PackageInfo, CachedDoc)
erFinishFull = Limits
-> PackageName
-> Text
-> StreamResult NpmFullRead
-> Either MetadataError (PackageInfo, CachedDoc)
finishNpmFull
, erRetainedEntries :: NpmFullRead -> RetainedEntries
erRetainedEntries = \(Walked InternTable
_ PackedRead
acc) -> PackedRead -> RetainedEntries
keptEntries PackedRead
acc
, erWalkSelected :: Limits -> PackageName -> Version -> DocumentWalk (Walked TreeRead)
erWalkSelected = \Limits
limits PackageName
name Version
version -> Limits
-> PackageName -> PackumentRead -> DocumentWalk (Walked TreeRead)
readNpmPackument Limits
limits PackageName
name (Text -> PackumentRead
OneRelease (Version -> Text
renderVersion Version
version))
, erFinishSelected :: Limits
-> PackageName
-> Text
-> Version
-> StreamResult (Walked TreeRead)
-> Either MetadataError VersionRead
erFinishSelected = Limits
-> PackageName
-> Text
-> Version
-> StreamResult (Walked TreeRead)
-> Either MetadataError VersionRead
finishNpmVersion
}
npmChargeFactors :: ChargeFactors
npmChargeFactors :: ChargeFactors
npmChargeFactors = ChargeFactors{cfFullReadPermille :: Int
cfFullReadPermille = Int
1100, cfKeptPermille :: Int
cfKeptPermille = Int
900, cfOutputPermille :: Int
cfOutputPermille = Int
2000}
readNpmPackument :: Limits -> PackageName -> PackumentRead -> DocumentWalk (Walked TreeRead)
readNpmPackument :: Limits
-> PackageName -> PackumentRead -> DocumentWalk (Walked TreeRead)
readNpmPackument Limits
limits PackageName
name PackumentRead
mode BodyLimit
bound InternTable
table = BodyLimit
-> (Cursor -> Step (Walked TreeRead))
-> IO ByteString
-> IO (Either LimitError (StreamResult (Walked TreeRead)))
forall (m :: * -> *) s.
Monad m =>
BodyLimit
-> (Cursor -> Step s)
-> m ByteString
-> m (Either LimitError (StreamResult s))
readJsonWalk BodyLimit
bound (Limits
-> PackageName
-> PackumentRead
-> InternTable
-> Cursor
-> Step (Walked TreeRead)
npmPackumentWalk Limits
limits PackageName
name PackumentRead
mode InternTable
table)
npmPackumentWalk :: Limits -> PackageName -> PackumentRead -> InternTable -> Cursor -> Step (Walked TreeRead)
npmPackumentWalk :: Limits
-> PackageName
-> PackumentRead
-> InternTable
-> Cursor
-> Step (Walked TreeRead)
npmPackumentWalk Limits
limits PackageName
name PackumentRead
mode InternTable
table = Trees
-> Int
-> PackumentRead
-> FieldStep
TreeRead (NpmFieldOf (Built Trees)) (Step (Walked TreeRead))
-> (TreeRead -> Text -> Bool)
-> InternTable
-> TreeRead
-> Cursor
-> Step (Walked TreeRead)
forall b r s.
(Build b r, Result r ~ Walked s) =>
b
-> Int
-> PackumentRead
-> FieldStep s (NpmFieldOf (Built b)) r
-> (s -> Text -> Bool)
-> InternTable
-> s
-> Cursor
-> r
npmWalk Trees
Trees (Limits -> Int
maxNestingDepth Limits
limits) PackumentRead
mode ((TreeRead -> NpmFieldOf Value -> Either LimitError TreeRead)
-> FieldStep TreeRead (NpmFieldOf Value) (Step (Walked TreeRead))
forall s field r.
(s -> field -> Either LimitError s) -> FieldStep s field r
pureStep (Limits
-> PackageName
-> TreeRead
-> NpmFieldOf Value
-> Either LimitError TreeRead
treeStep Limits
limits PackageName
name)) TreeRead -> Text -> Bool
keepsTreeRelease InternTable
table TreeRead
emptyTreeRead
type NpmFullRead = Walked PackedRead
readNpmFull :: (Int -> IO ()) -> Limits -> PackageName -> Text -> DocumentWalk NpmFullRead
readNpmFull :: (Int -> IO ())
-> Limits -> PackageName -> Text -> DocumentWalk NpmFullRead
readNpmFull Int -> IO ()
pay Limits
limits PackageName
name Text
base BodyLimit
bound InternTable
table0 IO ByteString
readChunk = do
(table, writer) <- ST RealWorld (InternTable, VersionWriter RealWorld)
-> IO (InternTable, VersionWriter RealWorld)
forall a. ST RealWorld a -> IO a
stToIO (Text
-> PackageName
-> InternTable
-> ST RealWorld (InternTable, VersionWriter RealWorld)
forall st.
Text
-> PackageName
-> InternTable
-> ST st (InternTable, VersionWriter st)
npmFullTable Text
base PackageName
name InternTable
table0)
readFlooredWalk pay (cfFullReadPermille npmChargeFactors) bound (\PrimVar RealWorld Int
owed -> PrimVar RealWorld Int
-> VersionWriter RealWorld
-> Limits
-> PackageName
-> InternTable
-> Cursor
-> ST RealWorld (Steps (ST RealWorld) NpmFullRead)
forall st.
PrimVar st Int
-> VersionWriter st
-> Limits
-> PackageName
-> InternTable
-> Cursor
-> ST st (Steps (ST st) NpmFullRead)
npmFullWalk PrimVar RealWorld Int
owed VersionWriter RealWorld
writer Limits
limits PackageName
name InternTable
table) readChunk
npmFullTable :: Text -> PackageName -> InternTable -> ST st (InternTable, VersionWriter st)
npmFullTable :: forall st.
Text
-> PackageName
-> InternTable
-> ST st (InternTable, VersionWriter st)
npmFullTable Text
base PackageName
name InternTable
table0 = (InternTable
seeded,) (VersionWriter st -> (InternTable, VersionWriter st))
-> ST st (VersionWriter st)
-> ST st (InternTable, VersionWriter st)
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> Maybe (Entry, Entry) -> ST st (VersionWriter st)
forall st. Maybe (Entry, Entry) -> ST st (VersionWriter st)
newVersionWriter ((Entry, Entry) -> Maybe (Entry, Entry)
forall a. a -> Maybe a
Just (Entry
authorKey, Entry
pointer))
where
Interned Entry
authorKey InternTable
withKey = Name -> InternTable -> Interned
internName (Text -> Name
decodedName Text
"author") InternTable
table0
Interned Entry
pointer InternTable
seeded = Name -> InternTable -> Interned
internName (Text -> Name
decodedName (Text -> PackageName -> Text
authorPointer Text
base PackageName
name)) InternTable
withKey
npmFullWalk :: PrimVar st Int -> VersionWriter st -> Limits -> PackageName -> InternTable -> Cursor -> ST st (Steps (ST st) NpmFullRead)
npmFullWalk :: forall st.
PrimVar st Int
-> VersionWriter st
-> Limits
-> PackageName
-> InternTable
-> Cursor
-> ST st (Steps (ST st) NpmFullRead)
npmFullWalk PrimVar st Int
owed VersionWriter st
writer Limits
limits PackageName
name InternTable
table = VersionWriter st
-> Int
-> PackumentRead
-> FieldStep
PackedRead
(NpmFieldOf (Built (VersionWriter st)))
(ST st (Steps (ST st) NpmFullRead))
-> (PackedRead -> Text -> Bool)
-> InternTable
-> PackedRead
-> Cursor
-> ST st (Steps (ST st) NpmFullRead)
forall b r s.
(Build b r, Result r ~ Walked s) =>
b
-> Int
-> PackumentRead
-> FieldStep s (NpmFieldOf (Built b)) r
-> (s -> Text -> Bool)
-> InternTable
-> s
-> Cursor
-> r
npmWalk VersionWriter st
writer (Limits -> Int
maxNestingDepth Limits
limits) PackumentRead
WholePackument FieldStep
PackedRead
(NpmFieldOf (Built (VersionWriter st)))
(ST st (Steps (ST st) NpmFullRead))
PackedRead
-> NpmFieldOf VersionValue
-> (Either LimitError PackedRead
-> ST st (Steps (ST st) NpmFullRead))
-> ST st (Steps (ST st) NpmFullRead)
forall {r}.
PackedRead
-> NpmFieldOf VersionValue
-> (Either LimitError PackedRead -> ST st r)
-> ST st r
step PackedRead -> Text -> Bool
keepsPackedRelease InternTable
table PackedRead
emptyPackedRead
where
step :: PackedRead
-> NpmFieldOf VersionValue
-> (Either LimitError PackedRead -> ST st r)
-> ST st r
step PackedRead
acc NpmFieldOf VersionValue
field Either LimitError PackedRead -> ST st r
next = VersionWriter st
-> Limits
-> PackageName
-> PackedRead
-> NpmFieldOf VersionValue
-> (Either LimitError PackedRead -> ST st r)
-> ST st r
forall st r.
VersionWriter st
-> Limits
-> PackageName
-> PackedRead
-> NpmFieldOf VersionValue
-> (Either LimitError PackedRead -> ST st r)
-> ST st r
packedStep VersionWriter st
writer Limits
limits PackageName
name PackedRead
acc NpmFieldOf VersionValue
field (PrimVar st Int
-> (PackedRead -> RetainedEntries)
-> (Either LimitError PackedRead -> ST st r)
-> Either LimitError PackedRead
-> ST st r
forall st s r.
PrimVar st Int
-> (s -> RetainedEntries)
-> (Either LimitError s -> ST st r)
-> Either LimitError s
-> ST st r
countingEntries PrimVar st Int
owed PackedRead -> RetainedEntries
retainedEntries Either LimitError PackedRead -> ST st r
next)
{-# INLINE npmFullWalk #-}
finishNpmFull :: Limits -> PackageName -> Text -> StreamResult NpmFullRead -> Either MetadataError (PackageInfo, CachedDoc)
finishNpmFull :: Limits
-> PackageName
-> Text
-> StreamResult NpmFullRead
-> Either MetadataError (PackageInfo, CachedDoc)
finishNpmFull Limits
limits PackageName
name Text
base StreamResult NpmFullRead
streamed =
(PackageInfo -> PackageInfo)
-> (PackedPackument -> CachedDoc)
-> (PackageInfo, PackedPackument)
-> (PackageInfo, CachedDoc)
forall a b c d. (a -> b) -> (c -> d) -> (a, c) -> (b, d)
forall (p :: * -> * -> *) a b c d.
Bifunctor p =>
(a -> b) -> (c -> d) -> p a c -> p b d
bimap (AllowedHostPorts -> Text -> PackageInfo -> PackageInfo
enforceArtifactLocations AllowedHostPorts
npmArtifactAuthorities Text
base) ((PackedPackument -> CachedDoc, CachedDoc -> Maybe PackedPackument)
-> PackedPackument -> CachedDoc
forall a b. (a, b) -> a
fst (PackedPackument -> CachedDoc, CachedDoc -> Maybe PackedPackument)
npmPacked) ((PackageInfo, PackedPackument) -> (PackageInfo, CachedDoc))
-> Either MetadataError (PackageInfo, PackedPackument)
-> Either MetadataError (PackageInfo, CachedDoc)
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> Limits
-> PackageName
-> Text
-> StreamResult NpmFullRead
-> Either MetadataError (PackageInfo, PackedPackument)
projectNpmPacked Limits
limits PackageName
name Text
base StreamResult NpmFullRead
streamed
projectNpmPacked :: Limits -> PackageName -> Text -> StreamResult NpmFullRead -> Either MetadataError (PackageInfo, PackedPackument)
projectNpmPacked :: Limits
-> PackageName
-> Text
-> StreamResult NpmFullRead
-> Either MetadataError (PackageInfo, PackedPackument)
projectNpmPacked Limits
limits PackageName
name Text
base StreamResult NpmFullRead
streamed = do
Walked table acc <- (ParseError -> MetadataError)
-> Either ParseError NpmFullRead
-> Either MetadataError NpmFullRead
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first (Limits -> ParseError -> MetadataError
streamError Limits
limits) (StreamResult NpmFullRead -> Either ParseError NpmFullRead
forall a. StreamResult a -> Either ParseError a
streamValue StreamResult NpmFullRead
streamed)
finishPacked name (authorPointer base name) (docTable (tableTexts table)) acc
finishNpmVersion :: Limits -> PackageName -> Text -> Version -> StreamResult (Walked TreeRead) -> Either MetadataError VersionRead
finishNpmVersion :: Limits
-> PackageName
-> Text
-> Version
-> StreamResult (Walked TreeRead)
-> Either MetadataError VersionRead
finishNpmVersion Limits
limits PackageName
name Text
base Version
version StreamResult (Walked TreeRead)
streamed = do
projected <- Limits
-> PackageName
-> Text
-> StreamResult (Walked TreeRead)
-> Either MetadataError (PackageInfo, Value)
projectNpmStream Limits
limits PackageName
name Text
base StreamResult (Walked TreeRead)
streamed
let selected = Version -> Int -> (PackageInfo, Value) -> VersionRead
selectNpmRead Version
version (StreamResult (Walked TreeRead) -> Int
forall a. StreamResult a -> Int
streamBytes StreamResult (Walked TreeRead)
streamed) (PackageInfo, Value)
projected
pure selected{vrVersion = vrVersion selected >>= locationCheckedDoc base}
projectNpmStream :: Limits -> PackageName -> Text -> StreamResult (Walked TreeRead) -> Either MetadataError (PackageInfo, Value)
projectNpmStream :: Limits
-> PackageName
-> Text
-> StreamResult (Walked TreeRead)
-> Either MetadataError (PackageInfo, Value)
projectNpmStream Limits
limits PackageName
name Text
base StreamResult (Walked TreeRead)
streamed = do
Walked _ acc <- (ParseError -> MetadataError)
-> Either ParseError (Walked TreeRead)
-> Either MetadataError (Walked TreeRead)
forall a b c. (a -> b) -> Either a c -> Either b c
forall (p :: * -> * -> *) a b c.
Bifunctor p =>
(a -> b) -> p a c -> p b c
first (Limits -> ParseError -> MetadataError
streamError Limits
limits) (StreamResult (Walked TreeRead)
-> Either ParseError (Walked TreeRead)
forall a. StreamResult a -> Either ParseError a
streamValue StreamResult (Walked TreeRead)
streamed)
finishTree name (authorPointer base name) acc
selectNpmRead :: Version -> Int -> (PackageInfo, Value) -> VersionRead
selectNpmRead :: Version -> Int -> (PackageInfo, Value) -> VersionRead
selectNpmRead Version
version Int
bodyBytes (PackageInfo
info, Value
raw) =
VersionRead
{ vrVersion :: Maybe VersionDoc
vrVersion = do
details <- Text -> Map Text PackageDetails -> Maybe PackageDetails
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup (Version -> Text
renderVersion Version
version) (PackageInfo -> Map Text PackageDetails
infoVersions PackageInfo
info)
selected <- selectNpmVersionDoc version (fst npmCached raw)
pure VersionDoc{vdDetails = details, vdRaw = Just selected}
, vrBodyBytes :: Int
vrBodyBytes = Int
bodyBytes
, vrUpstreamLatest :: Maybe Version
vrUpstreamLatest = Text -> Map Text Version -> Maybe Version
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup Text
"latest" (PackageInfo -> Map Text Version
infoDistTags PackageInfo
info)
}
authorPointer :: Text -> PackageName -> Text
authorPointer :: Text -> PackageName -> Text
authorPointer Text
base PackageName
name = Text
"See " Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text -> Either UrlFormationError Text -> Text
forall b a. b -> Either a b -> b
fromRight (Text
base Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> Text
"/" Text -> Text -> Text
forall a. Semigroup a => a -> a -> a
<> PackageName -> Text
renderPackageName PackageName
name) (Text -> PackageName -> Either UrlFormationError Text
packageUrl Text
base PackageName
name)
locationCheckedDoc :: Text -> VersionDoc -> Maybe VersionDoc
locationCheckedDoc :: Text -> VersionDoc -> Maybe VersionDoc
locationCheckedDoc Text
upstreamBaseUrl VersionDoc
doc =
(\PackageDetails
details -> VersionDoc
doc{vdDetails = details})
(PackageDetails -> VersionDoc)
-> Maybe PackageDetails -> Maybe VersionDoc
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> AllowedHostPorts -> Text -> PackageDetails -> Maybe PackageDetails
enforceArtifactLocationsOf AllowedHostPorts
npmArtifactAuthorities Text
upstreamBaseUrl (VersionDoc -> PackageDetails
vdDetails VersionDoc
doc)
npmArtifactAuthorities :: AllowedHostPorts
npmArtifactAuthorities :: AllowedHostPorts
npmArtifactAuthorities = [Text] -> AllowedHostPorts
ecosystemArtifactAuthorities [Text]
npmArtifactHosts
selectNpmVersionDoc :: Version -> CachedDoc -> Maybe CachedDoc
selectNpmVersionDoc :: Version -> CachedDoc -> Maybe CachedDoc
selectNpmVersionDoc Version
version CachedDoc
doc = do
Object packument <- (Value -> CachedDoc, CachedDoc -> Maybe Value)
-> CachedDoc -> Maybe Value
forall a b. (a, b) -> b
snd (Value -> CachedDoc, CachedDoc -> Maybe Value)
npmCached CachedDoc
doc
versions <- objectField "versions" packument
fst npmCached <$> KeyMap.lookup (Key.fromText (renderVersion version)) versions