libtommath: Fix possible integer overflow CVE-2023-36328
[heimdal.git] / lib / hcrypto / libtommath / bn_mp_radix_size.c
blobb96f4874c4927bba331dea58444024d76ec88cff
1 #include "tommath_private.h"
2 #ifdef BN_MP_RADIX_SIZE_C
3 /* LibTomMath, multiple-precision integer library -- Tom St Denis */
4 /* SPDX-License-Identifier: Unlicense */
6 /* returns size of ASCII representation */
7 mp_err mp_radix_size(const mp_int *a, int radix, int *size)
9 mp_err err;
10 int digs;
11 mp_int t;
12 mp_digit d;
14 *size = 0;
16 /* make sure the radix is in range */
17 if ((radix < 2) || (radix > 64)) {
18 return MP_VAL;
21 if (MP_IS_ZERO(a)) {
22 *size = 2;
23 return MP_OKAY;
26 /* special case for binary */
27 if (radix == 2) {
28 *size = (mp_count_bits(a) + ((a->sign == MP_NEG) ? 1 : 0) + 1);
29 return MP_OKAY;
32 /* digs is the digit count */
33 digs = 0;
35 /* if it's negative add one for the sign */
36 if (a->sign == MP_NEG) {
37 ++digs;
40 /* init a copy of the input */
41 if ((err = mp_init_copy(&t, a)) != MP_OKAY) {
42 return err;
45 /* force temp to positive */
46 t.sign = MP_ZPOS;
48 /* fetch out all of the digits */
49 while (!MP_IS_ZERO(&t)) {
50 if ((err = mp_div_d(&t, (mp_digit)radix, &t, &d)) != MP_OKAY) {
51 goto LBL_ERR;
53 ++digs;
56 /* return digs + 1, the 1 is for the NULL byte that would be required. */
57 *size = digs + 1;
58 err = MP_OKAY;
60 LBL_ERR:
61 mp_clear(&t);
62 return err;
65 #endif