mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(users/Profpatsch/whatcd-resolver): query all pages in search

Browse source 
If more than one page is returned by the search, query all of them.

Also add an ON CONFLICT clause in case the torrent group already
exists, to update it.

This function is getting a bit unwieldy (plus it suffers from an n+1
problem), but ok.

Change-Id: Ib505a2be8286d658ae44a3fe124a4fb42d0fc0c6
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9334
Autosubmit: Profpatsch <mail@profpatsch.de>
Tested-by: BuildkiteCI
Reviewed-by: Profpatsch <mail@profpatsch.de>
This commit is contained in:
Profpatsch 2023-09-15 14:28:11 +02:00 • committed by clbot
parent 99f2ba1f56
commit 8cfe6bc99b
1 changed files with 78 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

132
users/Profpatsch/whatcd-resolver/src/WhatcdResolver.hs
View file

@ -636,7 +636,9 @@ bla = do
] ]
pure (t1 >> t2 >> t3) pure (t1 >> t2 >> t3)
-- | Do the search, return a transaction that inserts all results from all pages of the search.
redactedSearchAndInsert :: redactedSearchAndInsert ::
forall m.
( MonadLogger m, ( MonadLogger m,
MonadIO m, MonadIO m,
MonadPostgres m, MonadPostgres m,
@ -644,80 +646,102 @@ redactedSearchAndInsert ::
) => ) =>
[(ByteString, ByteString)] -> [(ByteString, ByteString)] ->
m (Transaction m ()) m (Transaction m ())
redactedSearchAndInsert x = redactedSearchAndInsert extraArguments = do
redactedSearch -- The first search returns the amount of pages, so we use that to query all results piece by piece.
x firstPage <- go Nothing
( do let otherPagesNum = [(2 :: Natural) .. (firstPage.pages - 1)]
status <- Json.key "status" Json.asText otherPages <- traverse go (Just <$> otherPagesNum)
when (status /= "success") $ do pure $ (firstPage : otherPages) & traverse_ (.transaction)
Json.throwCustomError [fmt|Status was not "success", but {status}|] where
Json.key "response" $ do go :: Maybe Natural -> m (T2 "pages" Natural "transaction" (Transaction m ()))
Json.key "results" $ go mpage =
sequence_ redactedSearch
<$> ( Json.eachInArray $ do ( extraArguments
groupId <- Json.key "groupId" (Json.asIntegral @_ @Int) -- pass the page (for every search but the first one)
groupName <- Json.key "groupName" Json.asText <> ifExists (mpage <&> (\page -> [("page", (page :: Natural) & showToText & textToBytesUtf8)]))
fullJsonResult <- )
Json.asObject ( do
-- remove torrents cause they are inserted separately below status <- Json.key "status" Json.asText
<&> KeyMap.filterWithKey (\k _ -> k /= "torrents") when (status /= "success") $ do
<&> Json.Object Json.throwCustomError [fmt|Status was not "success", but {status}|]
let insertTourGroup = do Json.key "response" $ do
_ <- pages <- Json.key "pages" (Field.jsonParser (Field.mapError singleError $ Field.jsonNumber >>> Field.boundedScientificIntegral @Int "not an Integer" >>> Field.integralToNatural))
execute Json.key "results" $ do
[fmt| transaction <-
sequence_
<$> ( Json.eachInArray $ do
groupId <- Json.key "groupId" (Json.asIntegral @_ @Int)
groupName <- Json.key "groupName" Json.asText
fullJsonResult <-
Json.asObject
-- remove torrents cause they are inserted separately below
<&> KeyMap.filterWithKey (\k _ -> k /= "torrents")
<&> Json.Object
let insertTourGroup = do
_ <-
execute
[fmt|
DELETE FROM redacted.torrent_groups DELETE FROM redacted.torrent_groups
WHERE group_id = ?::integer WHERE group_id = ?::integer
|] |]
(Only groupId) (Only groupId)
executeManyReturningWith executeManyReturningWith
[fmt| [fmt|
INSERT INTO redacted.torrent_groups ( INSERT INTO redacted.torrent_groups (
group_id, group_name, full_json_result group_id, group_name, full_json_result
) VALUES ) VALUES
( ?, ? , ? ) ( ?, ? , ? )
ON CONFLICT (group_id) DO UPDATE SET
group_id = excluded.group_id,
group_name = excluded.group_name,
full_json_result = excluded.full_json_result
RETURNING (id) RETURNING (id)
|] |]
[ ( groupId, [ ( groupId,
groupName, groupName,
fullJsonResult fullJsonResult
) )
] ]
(label @"tourGroupIdPg" <$> Dec.fromField @Int) (label @"tourGroupIdPg" <$> Dec.fromField @Int)
>>= ensureSingleRow >>= ensureSingleRow
insertTorrents <- Json.key "torrents" $ do insertTorrents <- Json.key "torrents" $ do
torrents <- Json.eachInArray $ do torrents <- Json.eachInArray $ do
torrentId <- Json.keyLabel @"torrentId" "torrentId" (Json.asIntegral @_ @Int) torrentId <- Json.keyLabel @"torrentId" "torrentId" (Json.asIntegral @_ @Int)
fullJsonResultT <- label @"fullJsonResult" <$> Json.asValue fullJsonResultT <- label @"fullJsonResult" <$> Json.asValue
pure $ T2 torrentId fullJsonResultT pure $ T2 torrentId fullJsonResultT
pure $ \dat -> do pure $ \dat -> do
_ <- _ <-
execute execute
[sql| [sql|
DELETE FROM redacted.torrents_json DELETE FROM redacted.torrents_json
WHERE torrent_id = ANY (?::integer[]) WHERE torrent_id = ANY (?::integer[])
|] |]
(Only $ torrents & unzipT2 & (.torrentId) & PGArray) (Only $ torrents & unzipT2 & (.torrentId) & PGArray)
execute execute
[sql| [sql|
INSERT INTO redacted.torrents_json INSERT INTO redacted.torrents_json
(torrent_id, torrent_group, full_json_result) (torrent_id, torrent_group, full_json_result)
SELECT inputs.torrent_id, static.torrent_group, inputs.full_json_result FROM SELECT inputs.torrent_id, static.torrent_group, inputs.full_json_result FROM
(SELECT * FROM UNNEST(?::integer[], ?::jsonb[])) AS inputs(torrent_id, full_json_result) (SELECT * FROM UNNEST(?::integer[], ?::jsonb[])) AS inputs(torrent_id, full_json_result)
CROSS JOIN (VALUES(?::integer)) as static(torrent_group) CROSS JOIN (VALUES(?::integer)) as static(torrent_group)
|] |]
( torrents ( torrents
& unzipT2 & unzipT2
& \t -> & \t ->
( t.torrentId & PGArray, ( t.torrentId & PGArray,
t.fullJsonResult & PGArray, t.fullJsonResult & PGArray,
dat.tourGroupIdPg dat.tourGroupIdPg
)
) )
) pure ()
pure () pure (insertTourGroup >>= insertTorrents)
pure (insertTourGroup >>= insertTorrents) )
pure
( T2
(label @"pages" pages)
(label @"transaction" transaction)
) )
) )
redactedGetTorrentFileAndInsert :: redactedGetTorrentFileAndInsert ::
( HasField "torrentId" r Int, ( HasField "torrentId" r Int,