Fix possible infinite loop in BN_mod_sqrt()
The calculation in some cases does not finish for non-prime p.
This fixes CVE-2022-0778.
Based on patch by David Benjamin
@@ -14,7 +14,8 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
/*
* Returns 'ret' such that ret^2 == a (mod p), using the Tonelli/Shanks
* algorithm (cf. Henri Cohen, "A Course in Algebraic Computational Number
- * Theory", algorithm 1.5.1). 'p' must be prime!
+ * Theory", algorithm 1.5.1). 'p' must be prime, otherwise an error or
+ * an incorrect "result" will be returned.
*/
{
BIGNUM *ret = in;
@@ -303,17 +304,22 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
goto vrfy;
}
- /* find smallest i such that b^(2^i) = 1 */
- i = 1;
+ /* Find the smallest i, 0 < i < e, such that b^(2^i) = 1. */
+ for (i = 1; i < e; i++) {
+ if (i == 1) {
if (!BN_mod_sqr(t, b, p, ctx))
goto end;
- while (!BN_is_one(t)) {
- i++;
- if (i == e) {
- ERR_raise(ERR_LIB_BN, BN_R_NOT_A_SQUARE);
+
+ } else {
+ if (!BN_mod_mul(t, t, t, p, ctx))
goto end;
}
- if (!BN_mod_mul(t, t, t, p, ctx))
+ if (BN_is_one(t))
+ break;
+ }
+ /* If not found, a is not a square or p is not prime. */
+ if (i >= e) {
+ ERR_raise(ERR_LIB_BN, BN_R_NOT_A_SQUARE);
goto end;
}
