module Ecluse.Core.Server.Pipeline.Packument.Representation (
Contribution (..),
fingerprintPiece,
codedETag,
packumentETag,
outputBasisBytes,
assembleServedBody,
) where
import Control.Monad.ST (ST)
import Crypto.Hash (SHA256 (SHA256), hashWith, hashlazy)
import Data.ByteString.Builder (toLazyByteString)
import Data.ByteString.Internal (c2w)
import Data.Map.Strict qualified as Map
import Data.Text.Unsafe (lengthWord8)
import Ecluse.Core.Package (Artifact (artEntryKey), PackageDetails (pkgArtifacts), PackageInfo (infoVersions), PackageName, renderPackageName)
import Ecluse.Core.Package.Entry (EntryKey (..))
import Ecluse.Core.Package.Merge (MergePlan (mpSurvivors), Provenance (GatedSource, TrustedSource), SourceId)
import Ecluse.Core.Registry.Adapter.Capability (AdapterMetadata (metadataAssemble))
import Ecluse.Core.Registry.CachedDocument (CachedDoc)
import Ecluse.Core.Registry.Json.Scratch (Scratch, decimalLength, putByte, putDecimal, putRawBytes, putRawText, scratchBytes)
import Ecluse.Core.Registry.Metadata (ContentDigest, digestBytes)
import Ecluse.Core.Server.Conditional (ETag, mkStrongETag, mkWeakETag, renderETag)
import Ecluse.Core.Server.Encoding (ContentCoding (GzipCoding, IdentityCoding))
import Ecluse.Core.Server.Framing (frameBytes, putFrame)
import Ecluse.Core.Snapshot (Snapshot (..))
data Contribution = Contribution
{ Contribution -> Provenance
srcProvenance :: Provenance
, Contribution -> PackageInfo
srcInfo :: PackageInfo
, Contribution -> CachedDoc
srcValue :: CachedDoc
, Contribution -> ContentDigest
srcDigest :: ContentDigest
, Contribution -> Int
srcBodyBytes :: Int
}
fingerprintPiece :: Contribution -> (Provenance, ContentDigest, [(Text, [EntryKey])])
fingerprintPiece :: Contribution -> (Provenance, ContentDigest, [(Text, [EntryKey])])
fingerprintPiece Contribution
s =
( Contribution -> Provenance
srcProvenance Contribution
s
, Contribution -> ContentDigest
srcDigest Contribution
s
, [(Text
version, (Artifact -> EntryKey) -> [Artifact] -> [EntryKey]
forall a b. (a -> b) -> [a] -> [b]
map Artifact -> EntryKey
artEntryKey (NonEmpty Artifact -> [Artifact]
forall a. NonEmpty a -> [a]
forall (t :: * -> *) a. Foldable t => t a -> [a]
toList (PackageDetails -> NonEmpty Artifact
pkgArtifacts PackageDetails
details))) | (Text
version, PackageDetails
details) <- Map Text PackageDetails -> [(Text, PackageDetails)]
forall k a. Map k a -> [(k, a)]
Map.toList (PackageInfo -> Map Text PackageDetails
infoVersions (Contribution -> PackageInfo
srcInfo Contribution
s))]
)
codedETag :: ContentCoding -> ETag -> ETag
codedETag :: ContentCoding -> ETag -> ETag
codedETag ContentCoding
coding ETag
identityTag = case ContentCoding
coding of
ContentCoding
IdentityCoding -> ETag
identityTag
ContentCoding
GzipCoding -> Digest SHA256 -> ETag
mkWeakETag (ByteString -> Digest SHA256
forall a. HashAlgorithm a => ByteString -> Digest a
hashlazy (Builder -> ByteString
toLazyByteString Builder
discriminated))
where
discriminated :: Builder
discriminated = Builder
"ecluse:packument-etag:gzip:v1\0" Builder -> Builder -> Builder
forall a. Semigroup a => a -> a -> a
<> ByteString -> Builder
frameBytes (Text -> ByteString
forall a b. ConvertUtf8 a b => a -> b
encodeUtf8 (ETag -> Text
renderETag ETag
identityTag))
packumentETag :: Text -> [Text] -> PackageName -> [(Provenance, ContentDigest, [(Text, [EntryKey])])] -> ETag
packumentETag :: Text
-> [Text]
-> PackageName
-> [(Provenance, ContentDigest, [(Text, [EntryKey])])]
-> ETag
packumentETag Text
mountBaseUrl [Text]
originBaseUrls PackageName
name [(Provenance, ContentDigest, [(Text, [EntryKey])])]
sources =
Digest SHA256 -> ETag
mkStrongETag (SHA256 -> ByteString -> Digest SHA256
forall ba alg.
(ByteArrayAccess ba, HashAlgorithm alg) =>
alg -> ba -> Digest alg
hashWith SHA256
SHA256 (Int -> (forall st. Scratch st -> ST st ()) -> ByteString
scratchBytes Int
4096 Scratch st -> ST st ()
forall st. Scratch st -> ST st ()
fingerprint))
where
fingerprint :: Scratch st -> ST st ()
fingerprint :: forall st. Scratch st -> ST st ()
fingerprint Scratch st
scratch = do
Scratch st -> ByteString -> ST st ()
forall st. Scratch st -> ByteString -> ST st ()
putRawBytes Scratch st
scratch ByteString
"ecluse:packument-etag:v4\0"
(Text -> ST st ()) -> [Text] -> ST st ()
forall (t :: * -> *) (f :: * -> *) a b.
(Foldable t, Applicative f) =>
(a -> f b) -> t a -> f ()
traverse_ (Scratch st -> Text -> ST st ()
forall st. Scratch st -> Text -> ST st ()
putFramedText Scratch st
scratch) [Text]
originBaseUrls
Scratch st -> Word8 -> ST st ()
forall st. Scratch st -> Word8 -> ST st ()
putByte Scratch st
scratch Word8
0
Scratch st -> Text -> ST st ()
forall st. Scratch st -> Text -> ST st ()
putRawText Scratch st
scratch Text
mountBaseUrl
Scratch st -> Word8 -> ST st ()
forall st. Scratch st -> Word8 -> ST st ()
putByte Scratch st
scratch Word8
0
Scratch st -> Text -> ST st ()
forall st. Scratch st -> Text -> ST st ()
putRawText Scratch st
scratch (PackageName -> Text
renderPackageName PackageName
name)
Scratch st -> Word8 -> ST st ()
forall st. Scratch st -> Word8 -> ST st ()
putByte Scratch st
scratch Word8
0
((Provenance, ContentDigest, [(Text, [EntryKey])]) -> ST st ())
-> [(Provenance, ContentDigest, [(Text, [EntryKey])])] -> ST st ()
forall (t :: * -> *) (f :: * -> *) a b.
(Foldable t, Applicative f) =>
(a -> f b) -> t a -> f ()
traverse_ (Scratch st
-> (Provenance, ContentDigest, [(Text, [EntryKey])]) -> ST st ()
forall st.
Scratch st
-> (Provenance, ContentDigest, [(Text, [EntryKey])]) -> ST st ()
putSourcePiece Scratch st
scratch) [(Provenance, ContentDigest, [(Text, [EntryKey])])]
sources
putSourcePiece :: Scratch st -> (Provenance, ContentDigest, [(Text, [EntryKey])]) -> ST st ()
putSourcePiece :: forall st.
Scratch st
-> (Provenance, ContentDigest, [(Text, [EntryKey])]) -> ST st ()
putSourcePiece Scratch st
scratch (Provenance
provenance, ContentDigest
digest, [(Text, [EntryKey])]
survivors) = do
Scratch st -> Word8 -> ST st ()
forall st. Scratch st -> Word8 -> ST st ()
putByte Scratch st
scratch (Provenance -> Word8
provenanceTag Provenance
provenance)
Scratch st -> Word8 -> ST st ()
forall st. Scratch st -> Word8 -> ST st ()
putByte Scratch st
scratch Word8
0
Scratch st -> ByteString -> ST st ()
forall st. Scratch st -> ByteString -> ST st ()
putRawBytes Scratch st
scratch (ContentDigest -> ByteString
digestBytes ContentDigest
digest)
((Text, [EntryKey]) -> ST st ())
-> [(Text, [EntryKey])] -> ST st ()
forall (t :: * -> *) (f :: * -> *) a b.
(Foldable t, Applicative f) =>
(a -> f b) -> t a -> f ()
traverse_ (Scratch st -> (Text, [EntryKey]) -> ST st ()
forall st. Scratch st -> (Text, [EntryKey]) -> ST st ()
putVersionPiece Scratch st
scratch) [(Text, [EntryKey])]
survivors
Scratch st -> Word8 -> ST st ()
forall st. Scratch st -> Word8 -> ST st ()
putByte Scratch st
scratch Word8
1
putVersionPiece :: Scratch st -> (Text, [EntryKey]) -> ST st ()
putVersionPiece :: forall st. Scratch st -> (Text, [EntryKey]) -> ST st ()
putVersionPiece Scratch st
scratch (Text
version, [EntryKey]
entries) = do
Scratch st -> Text -> ST st ()
forall st. Scratch st -> Text -> ST st ()
putFramedText Scratch st
scratch Text
version
(EntryKey -> ST st ()) -> [EntryKey] -> ST st ()
forall (t :: * -> *) (f :: * -> *) a b.
(Foldable t, Applicative f) =>
(a -> f b) -> t a -> f ()
traverse_ (Scratch st -> EntryKey -> ST st ()
forall st. Scratch st -> EntryKey -> ST st ()
putEntryPiece Scratch st
scratch) [EntryKey]
entries
Scratch st -> Word8 -> ST st ()
forall st. Scratch st -> Word8 -> ST st ()
putByte Scratch st
scratch Word8
2
putEntryPiece :: Scratch st -> EntryKey -> ST st ()
putEntryPiece :: forall st. Scratch st -> EntryKey -> ST st ()
putEntryPiece Scratch st
scratch = \case
ArrayEntry Int
index -> Scratch st -> Word8 -> ST st ()
forall st. Scratch st -> Word8 -> ST st ()
putByte Scratch st
scratch (Char -> Word8
c2w Char
'a') ST st () -> ST st () -> ST st ()
forall a b. ST st a -> ST st b -> ST st b
forall (m :: * -> *) a b. Monad m => m a -> m b -> m b
>> Scratch st -> Int -> ST st () -> ST st ()
forall st. Scratch st -> Int -> ST st () -> ST st ()
putFrame Scratch st
scratch (Int -> Int
decimalLength Int
index) (Scratch st -> Int -> ST st ()
forall st. Scratch st -> Int -> ST st ()
putDecimal Scratch st
scratch Int
index)
RecordEntry Int
index -> Scratch st -> Word8 -> ST st ()
forall st. Scratch st -> Word8 -> ST st ()
putByte Scratch st
scratch (Char -> Word8
c2w Char
'r') ST st () -> ST st () -> ST st ()
forall a b. ST st a -> ST st b -> ST st b
forall (m :: * -> *) a b. Monad m => m a -> m b -> m b
>> Scratch st -> Int -> ST st () -> ST st ()
forall st. Scratch st -> Int -> ST st () -> ST st ()
putFrame Scratch st
scratch (Int -> Int
decimalLength Int
index) (Scratch st -> Int -> ST st ()
forall st. Scratch st -> Int -> ST st ()
putDecimal Scratch st
scratch Int
index)
ObjectEntry Text
key -> Scratch st -> Word8 -> ST st ()
forall st. Scratch st -> Word8 -> ST st ()
putByte Scratch st
scratch (Char -> Word8
c2w Char
'o') ST st () -> ST st () -> ST st ()
forall a b. ST st a -> ST st b -> ST st b
forall (m :: * -> *) a b. Monad m => m a -> m b -> m b
>> Scratch st -> Text -> ST st ()
forall st. Scratch st -> Text -> ST st ()
putFramedText Scratch st
scratch Text
key
EntryKey
SingletonEntry -> Scratch st -> Word8 -> ST st ()
forall st. Scratch st -> Word8 -> ST st ()
putByte Scratch st
scratch (Char -> Word8
c2w Char
's')
putFramedText :: Scratch st -> Text -> ST st ()
putFramedText :: forall st. Scratch st -> Text -> ST st ()
putFramedText Scratch st
scratch Text
text = Scratch st -> Int -> ST st () -> ST st ()
forall st. Scratch st -> Int -> ST st () -> ST st ()
putFrame Scratch st
scratch (Text -> Int
lengthWord8 Text
text) (Scratch st -> Text -> ST st ()
forall st. Scratch st -> Text -> ST st ()
putRawText Scratch st
scratch Text
text)
provenanceTag :: Provenance -> Word8
provenanceTag :: Provenance -> Word8
provenanceTag = \case
Provenance
TrustedSource -> Char -> Word8
c2w Char
't'
Provenance
GatedSource -> Char -> Word8
c2w Char
'g'
outputBasisBytes :: MergePlan -> [Contribution] -> Int
outputBasisBytes :: MergePlan -> [Contribution] -> Int
outputBasisBytes MergePlan
plan [Contribution]
sources = (Int -> Int -> Int) -> Int -> [Int] -> Int
forall b a. (b -> a -> b) -> b -> [a] -> b
forall (t :: * -> *) b a.
Foldable t =>
(b -> a -> b) -> b -> t a -> b
foldl' Int -> Int -> Int
forall a. Ord a => a -> a -> a
max Int
0 (((Int, Contribution) -> Int) -> [(Int, Contribution)] -> [Int]
forall a b. (a -> b) -> [a] -> [b]
map (Int, Contribution) -> Int
anchoredOn [(Int, Contribution)]
anchors)
where
indexed :: [(Int, Contribution)]
indexed = [Int] -> [Contribution] -> [(Int, Contribution)]
forall a b. [a] -> [b] -> [(a, b)]
zip [Int
0 :: Int ..] [Contribution]
sources
anchors :: [(Int, Contribution)]
anchors = Maybe (Int, Contribution) -> [(Int, Contribution)]
forall a. Maybe a -> [a]
maybeToList ([(Int, Contribution)] -> Maybe (Int, Contribution)
forall a. [a] -> Maybe a
listToMaybe (((Int, Contribution) -> Down Int)
-> [(Int, Contribution)] -> [(Int, Contribution)]
forall b a. Ord b => (a -> b) -> [a] -> [a]
sortOn (Int -> Down Int
forall a. a -> Down a
Down (Int -> Down Int)
-> ((Int, Contribution) -> Int) -> (Int, Contribution) -> Down Int
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Contribution -> Int
srcBodyBytes (Contribution -> Int)
-> ((Int, Contribution) -> Contribution)
-> (Int, Contribution)
-> Int
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Int, Contribution) -> Contribution
forall a b. (a, b) -> b
snd) [(Int, Contribution)]
indexed)) [(Int, Contribution)]
-> [(Int, Contribution)] -> [(Int, Contribution)]
forall a. Semigroup a => a -> a -> a
<> Maybe (Int, Contribution) -> [(Int, Contribution)]
forall a. Maybe a -> [a]
maybeToList ([(Int, Contribution)] -> Maybe (Int, Contribution)
baseSource [(Int, Contribution)]
indexed)
anchoredOn :: (Int, Contribution) -> Int
anchoredOn (Int
position, Contribution
anchor) =
Contribution -> Int
srcBodyBytes Contribution
anchor Int -> Int -> Int
forall a. Num a => a -> a -> a
+ [Int] -> Int
forall a (f :: * -> *). (Foldable f, Num a) => f a -> a
sum [Int -> Contribution -> Int
addedShare (Map Text Int -> Int
forall k a. Map k a -> Int
Map.size (MergePlan -> Map Text Int
mpSurvivors MergePlan
plan) Int -> Int -> Int
forall a. Num a => a -> a -> a
- Contribution -> Int
versionCount Contribution
anchor) Contribution
other | (Int
at, Contribution
other) <- [(Int, Contribution)]
indexed, Int
at Int -> Int -> Bool
forall a. Eq a => a -> a -> Bool
/= Int
position]
addedShare :: Int -> Contribution -> Int
addedShare :: Int -> Contribution -> Int
addedShare Int
outside Contribution
source
| Int
count Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
<= Int
0 = Contribution -> Int
srcBodyBytes Contribution
source
| Bool
otherwise = (Contribution -> Int
srcBodyBytes Contribution
source Int -> Int -> Int
forall a. Num a => a -> a -> a
* Int -> Int -> Int
forall a. Ord a => a -> a -> a
min Int
count (Int -> Int -> Int
forall a. Ord a => a -> a -> a
max Int
0 Int
outside) Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
count Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1) Int -> Int -> Int
forall a. Integral a => a -> a -> a
`div` Int
count
where
count :: Int
count = Contribution -> Int
versionCount Contribution
source
versionCount :: Contribution -> Int
versionCount :: Contribution -> Int
versionCount = Map Text PackageDetails -> Int
forall k a. Map k a -> Int
Map.size (Map Text PackageDetails -> Int)
-> (Contribution -> Map Text PackageDetails) -> Contribution -> Int
forall b c a. (b -> c) -> (a -> b) -> a -> c
. PackageInfo -> Map Text PackageDetails
infoVersions (PackageInfo -> Map Text PackageDetails)
-> (Contribution -> PackageInfo)
-> Contribution
-> Map Text PackageDetails
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Contribution -> PackageInfo
srcInfo
assembleServedBody :: AdapterMetadata -> Text -> [Contribution] -> MergePlan -> CachedDoc
assembleServedBody :: AdapterMetadata -> Text -> [Contribution] -> MergePlan -> CachedDoc
assembleServedBody AdapterMetadata
metadata Text
mountBase [Contribution]
sources MergePlan
plan =
AdapterMetadata
-> Text
-> Map Int (Snapshot CachedDoc)
-> MergePlan
-> Maybe CachedDoc
-> CachedDoc
metadataAssemble AdapterMetadata
metadata Text
mountBase Map Int (Snapshot CachedDoc)
bySource MergePlan
plan ([Contribution] -> Maybe CachedDoc
baseDocument [Contribution]
sources)
where
bySource :: Map SourceId (Snapshot CachedDoc)
bySource :: Map Int (Snapshot CachedDoc)
bySource = [(Int, Snapshot CachedDoc)] -> Map Int (Snapshot CachedDoc)
forall k a. Ord k => [(k, a)] -> Map k a
Map.fromList ([Int] -> [Snapshot CachedDoc] -> [(Int, Snapshot CachedDoc)]
forall a b. [a] -> [b] -> [(a, b)]
zip [Int
0 ..] [ContentDigest -> CachedDoc -> Snapshot CachedDoc
forall a. ContentDigest -> a -> Snapshot a
Snapshot (Contribution -> ContentDigest
srcDigest Contribution
source) (Contribution -> CachedDoc
srcValue Contribution
source) | Contribution
source <- [Contribution]
sources])
baseDocument :: [Contribution] -> Maybe CachedDoc
baseDocument :: [Contribution] -> Maybe CachedDoc
baseDocument = ((Int, Contribution) -> CachedDoc)
-> Maybe (Int, Contribution) -> Maybe CachedDoc
forall a b. (a -> b) -> Maybe a -> Maybe b
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
fmap (Contribution -> CachedDoc
srcValue (Contribution -> CachedDoc)
-> ((Int, Contribution) -> Contribution)
-> (Int, Contribution)
-> CachedDoc
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Int, Contribution) -> Contribution
forall a b. (a, b) -> b
snd) (Maybe (Int, Contribution) -> Maybe CachedDoc)
-> ([Contribution] -> Maybe (Int, Contribution))
-> [Contribution]
-> Maybe CachedDoc
forall b c a. (b -> c) -> (a -> b) -> a -> c
. [(Int, Contribution)] -> Maybe (Int, Contribution)
baseSource ([(Int, Contribution)] -> Maybe (Int, Contribution))
-> ([Contribution] -> [(Int, Contribution)])
-> [Contribution]
-> Maybe (Int, Contribution)
forall b c a. (b -> c) -> (a -> b) -> a -> c
. [Int] -> [Contribution] -> [(Int, Contribution)]
forall a b. [a] -> [b] -> [(a, b)]
zip [Int
0 :: Int ..]
baseSource :: [(Int, Contribution)] -> Maybe (Int, Contribution)
baseSource :: [(Int, Contribution)] -> Maybe (Int, Contribution)
baseSource [(Int, Contribution)]
indexed = ((Int, Contribution) -> Bool)
-> [(Int, Contribution)] -> Maybe (Int, Contribution)
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Maybe a
find ((Provenance -> Provenance -> Bool
forall a. Eq a => a -> a -> Bool
== Provenance
TrustedSource) (Provenance -> Bool)
-> ((Int, Contribution) -> Provenance)
-> (Int, Contribution)
-> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Contribution -> Provenance
srcProvenance (Contribution -> Provenance)
-> ((Int, Contribution) -> Contribution)
-> (Int, Contribution)
-> Provenance
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Int, Contribution) -> Contribution
forall a b. (a, b) -> b
snd) [(Int, Contribution)]
indexed Maybe (Int, Contribution)
-> Maybe (Int, Contribution) -> Maybe (Int, Contribution)
forall a. Maybe a -> Maybe a -> Maybe a
forall (f :: * -> *) a. Alternative f => f a -> f a -> f a
<|> [(Int, Contribution)] -> Maybe (Int, Contribution)
forall a. [a] -> Maybe a
listToMaybe [(Int, Contribution)]
indexed