From 3e1fb2dec781c816e07b1a91ab096e69a12647e4 Mon Sep 17 00:00:00 2001 From: Chaitanya Tata Date: Mon, 6 Nov 2023 22:49:15 +0530 Subject: [PATCH] dragonfly: Fix legendre symbol calculation failure handling In case of low-memory conditions, the computation for legendre symbol can fail and return -2 as per documentation, but the check for that was missed here. And this can can cause an infinite loop searching for qr and qnr if the error repeats for each attempt. Break the loop if calculation fails, we can leave retry to the callers or user. This is similar to the way allocation and generation of a new random number was handled in this loop. Signed-off-by: Chaitanya Tata --- src/common/dragonfly.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/common/dragonfly.c b/src/common/dragonfly.c index 1e8427166..d039e5f9e 100644 --- a/src/common/dragonfly.c +++ b/src/common/dragonfly.c @@ -67,12 +67,15 @@ int dragonfly_get_random_qr_qnr(const struct crypto_bignum *prime, } res = crypto_bignum_legendre(tmp, prime); - if (res == 1 && !(*qr)) + if (res == 1 && !(*qr)) { *qr = tmp; - else if (res == -1 && !(*qnr)) + } else if (res == -1 && !(*qnr)) { *qnr = tmp; - else + } else { crypto_bignum_deinit(tmp, 0); + if (res == -2) + break; + } } if (*qr && *qnr)