Init{RGB,YUV}Rescaler: fix a few more int overflows
promote out_width to size_t before multiplying
src/dec/io_dec.c:301:30: runtime error: signed integer overflow: 2 *
cannot be represented in type 'int'
0x55fd9e8de2bd in InitYUVRescaler src/dec/io_dec.c:301:30
0x55fd9e8de2bd in CustomSetup src/dec/io_dec.c:571:54
@@ -298,7 +298,8 @@ static int InitYUVRescaler(const VP8Io* const io, WebPDecParams* const p) {
const int uv_out_height = (out_height + 1) >> 1;
const int uv_in_width = (io->mb_w + 1) >> 1;
const int uv_in_height = (io->mb_h + 1) >> 1;
- const size_t work_size = 2 * out_width; // scratch memory for luma rescaler
+ // scratch memory for luma rescaler
+ const size_t work_size = 2 * (size_t)out_width;
const size_t uv_work_size = 2 * uv_out_width; // and for each u/v ones
uint64_t total_size;
size_t rescaler_size;
@@ -486,7 +487,8 @@ static int InitRGBRescaler(const VP8Io* const io, WebPDecParams* const p) {
const int out_height = io->scaled_height;
const int uv_in_width = (io->mb_w + 1) >> 1;
const int uv_in_height = (io->mb_h + 1) >> 1;
- const size_t work_size = 2 * out_width; // scratch memory for one rescaler
+ // scratch memory for one rescaler
+ const size_t work_size = 2 * (size_t)out_width;
rescaler_t* work; // rescalers work area
uint8_t* tmp; // tmp storage for scaled YUV444 samples before RGB conversion
uint64_t tmp_size1, tmp_size2, total_size;
