check integer overflow in 'map[pix * width + x] |= (1 << i);'
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <limits.h>
#if defined(HAVE_INTTYPES_H)
# include <inttypes.h>
@@ -502,6 +503,7 @@ sixel_encode_body(
int mx;
int len;
int pix;
+ int check_integer_overflow;
unsigned char *map = NULL;
sixel_node_t *np, *tp, top;
int fillable;
@@ -557,8 +559,30 @@ sixel_encode_body(
fillable = 1;
}
for (x = 0; x < width; x++) {
- pix = pixels[y * width + x]; /* color index */
+ if (y > INT_MAX / width) {
+ /* integer overflow */
+ status = SIXEL_BAD_INTEGER_OVERFLOW;
+ goto end;
+ }
+ check_integer_overflow = y * width;
+ if (check_integer_overflow > INT_MAX - x) {
+ /* integer overflow */
+ status = SIXEL_BAD_INTEGER_OVERFLOW;
+ goto end;
+ }
+ pix = pixels[check_integer_overflow + x]; /* color index */
if (pix >= 0 && pix < ncolors && pix != keycolor) {
+ if (pix > INT_MAX / width) {
+ /* integer overflow */
+ status = SIXEL_BAD_INTEGER_OVERFLOW;
+ goto end;
+ }
+ check_integer_overflow = pix * width;
+ if (check_integer_overflow > INT_MAX - x) {
+ /* integer overflow */
+ status = SIXEL_BAD_INTEGER_OVERFLOW;
+ goto end;
+ }
map[pix * width + x] |= (1 << i);
}
else if (!palstate) {
