feat(users/Profpatsch/whatcd-resolver): query all pages in search
If more than one page is returned by the search, query all of them. Also add an ON CONFLICT clause in case the torrent group already exists, to update it. This function is getting a bit unwieldy (plus it suffers from an n+1 problem), but ok. Change-Id: Ib505a2be8286d658ae44a3fe124a4fb42d0fc0c6 Reviewed-on: https://cl.tvl.fyi/c/depot/+/9334 Autosubmit: Profpatsch <mail@profpatsch.de> Tested-by: BuildkiteCI Reviewed-by: Profpatsch <mail@profpatsch.de>
This commit is contained in:
parent
99f2ba1f56
commit
8cfe6bc99b
1 changed files with 78 additions and 54 deletions
|
@ -636,7 +636,9 @@ bla = do
|
|||
]
|
||||
pure (t1 >> t2 >> t3)
|
||||
|
||||
-- | Do the search, return a transaction that inserts all results from all pages of the search.
|
||||
redactedSearchAndInsert ::
|
||||
forall m.
|
||||
( MonadLogger m,
|
||||
MonadIO m,
|
||||
MonadPostgres m,
|
||||
|
@ -644,15 +646,28 @@ redactedSearchAndInsert ::
|
|||
) =>
|
||||
[(ByteString, ByteString)] ->
|
||||
m (Transaction m ())
|
||||
redactedSearchAndInsert x =
|
||||
redactedSearchAndInsert extraArguments = do
|
||||
-- The first search returns the amount of pages, so we use that to query all results piece by piece.
|
||||
firstPage <- go Nothing
|
||||
let otherPagesNum = [(2 :: Natural) .. (firstPage.pages - 1)]
|
||||
otherPages <- traverse go (Just <$> otherPagesNum)
|
||||
pure $ (firstPage : otherPages) & traverse_ (.transaction)
|
||||
where
|
||||
go :: Maybe Natural -> m (T2 "pages" Natural "transaction" (Transaction m ()))
|
||||
go mpage =
|
||||
redactedSearch
|
||||
x
|
||||
( extraArguments
|
||||
-- pass the page (for every search but the first one)
|
||||
<> ifExists (mpage <&> (\page -> [("page", (page :: Natural) & showToText & textToBytesUtf8)]))
|
||||
)
|
||||
( do
|
||||
status <- Json.key "status" Json.asText
|
||||
when (status /= "success") $ do
|
||||
Json.throwCustomError [fmt|Status was not "success", but {status}|]
|
||||
Json.key "response" $ do
|
||||
Json.key "results" $
|
||||
pages <- Json.key "pages" (Field.jsonParser (Field.mapError singleError $ Field.jsonNumber >>> Field.boundedScientificIntegral @Int "not an Integer" >>> Field.integralToNatural))
|
||||
Json.key "results" $ do
|
||||
transaction <-
|
||||
sequence_
|
||||
<$> ( Json.eachInArray $ do
|
||||
groupId <- Json.key "groupId" (Json.asIntegral @_ @Int)
|
||||
|
@ -676,6 +691,10 @@ redactedSearchAndInsert x =
|
|||
group_id, group_name, full_json_result
|
||||
) VALUES
|
||||
( ?, ? , ? )
|
||||
ON CONFLICT (group_id) DO UPDATE SET
|
||||
group_id = excluded.group_id,
|
||||
group_name = excluded.group_name,
|
||||
full_json_result = excluded.full_json_result
|
||||
RETURNING (id)
|
||||
|]
|
||||
[ ( groupId,
|
||||
|
@ -717,6 +736,11 @@ redactedSearchAndInsert x =
|
|||
pure ()
|
||||
pure (insertTourGroup >>= insertTorrents)
|
||||
)
|
||||
pure
|
||||
( T2
|
||||
(label @"pages" pages)
|
||||
(label @"transaction" transaction)
|
||||
)
|
||||
)
|
||||
|
||||
redactedGetTorrentFileAndInsert ::
|
||||
|
|
Loading…
Reference in a new issue