Fixes out of array access
Fixes: da14e86d8462be6493eab16bc2d40f88/asan_heap-oob_204cfd2_528_cov_340150052_COMPRESS.BMP
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit f7e1367f58263593e6cee3c282f7277d7ee9d553)
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
... | ... |
@@ -37,16 +37,14 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic, |
37 | 37 |
unsigned char extra_byte, odd_pixel; |
38 | 38 |
unsigned char stream_byte; |
39 | 39 |
unsigned int pixel_ptr = 0; |
40 |
- int row_dec = pic->linesize[0]; |
|
41 |
- int row_ptr = (avctx->height - 1) * row_dec; |
|
42 |
- int frame_size = row_dec * avctx->height; |
|
40 |
+ int line = avctx->height - 1; |
|
43 | 41 |
int i; |
44 | 42 |
|
45 |
- while (row_ptr >= 0) { |
|
43 |
+ while (line >= 0 && pixel_ptr <= avctx->width) { |
|
46 | 44 |
if (bytestream2_get_bytes_left(gb) <= 0) { |
47 | 45 |
av_log(avctx, AV_LOG_ERROR, |
48 |
- "MS RLE: bytestream overrun, %d rows left\n", |
|
49 |
- row_ptr); |
|
46 |
+ "MS RLE: bytestream overrun, %dx%d left\n", |
|
47 |
+ avctx->width - pixel_ptr, line); |
|
50 | 48 |
return AVERROR_INVALIDDATA; |
51 | 49 |
} |
52 | 50 |
rle_code = stream_byte = bytestream2_get_byteu(gb); |
... | ... |
@@ -55,7 +53,7 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic, |
55 | 55 |
stream_byte = bytestream2_get_byte(gb); |
56 | 56 |
if (stream_byte == 0) { |
57 | 57 |
/* line is done, goto the next one */ |
58 |
- row_ptr -= row_dec; |
|
58 |
+ line--; |
|
59 | 59 |
pixel_ptr = 0; |
60 | 60 |
} else if (stream_byte == 1) { |
61 | 61 |
/* decode is done */ |
... | ... |
@@ -65,13 +63,12 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic, |
65 | 65 |
stream_byte = bytestream2_get_byte(gb); |
66 | 66 |
pixel_ptr += stream_byte; |
67 | 67 |
stream_byte = bytestream2_get_byte(gb); |
68 |
- row_ptr -= stream_byte * row_dec; |
|
69 | 68 |
} else { |
70 | 69 |
// copy pixels from encoded stream |
71 | 70 |
odd_pixel = stream_byte & 1; |
72 | 71 |
rle_code = (stream_byte + 1) / 2; |
73 | 72 |
extra_byte = rle_code & 0x01; |
74 |
- if (row_ptr + pixel_ptr + stream_byte > frame_size || |
|
73 |
+ if (pixel_ptr + 2*rle_code - odd_pixel > avctx->width || |
|
75 | 74 |
bytestream2_get_bytes_left(gb) < rle_code) { |
76 | 75 |
av_log(avctx, AV_LOG_ERROR, |
77 | 76 |
"MS RLE: frame/stream ptr just went out of bounds (copy)\n"); |
... | ... |
@@ -82,13 +79,13 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic, |
82 | 82 |
if (pixel_ptr >= avctx->width) |
83 | 83 |
break; |
84 | 84 |
stream_byte = bytestream2_get_byteu(gb); |
85 |
- pic->data[0][row_ptr + pixel_ptr] = stream_byte >> 4; |
|
85 |
+ pic->data[0][line * pic->linesize[0] + pixel_ptr] = stream_byte >> 4; |
|
86 | 86 |
pixel_ptr++; |
87 | 87 |
if (i + 1 == rle_code && odd_pixel) |
88 | 88 |
break; |
89 | 89 |
if (pixel_ptr >= avctx->width) |
90 | 90 |
break; |
91 |
- pic->data[0][row_ptr + pixel_ptr] = stream_byte & 0x0F; |
|
91 |
+ pic->data[0][line * pic->linesize[0] + pixel_ptr] = stream_byte & 0x0F; |
|
92 | 92 |
pixel_ptr++; |
93 | 93 |
} |
94 | 94 |
|
... | ... |
@@ -98,7 +95,7 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic, |
98 | 98 |
} |
99 | 99 |
} else { |
100 | 100 |
// decode a run of data |
101 |
- if (row_ptr + pixel_ptr + stream_byte > frame_size) { |
|
101 |
+ if (pixel_ptr + rle_code > avctx->width + 1) { |
|
102 | 102 |
av_log(avctx, AV_LOG_ERROR, |
103 | 103 |
"MS RLE: frame ptr just went out of bounds (run)\n"); |
104 | 104 |
return AVERROR_INVALIDDATA; |
... | ... |
@@ -108,9 +105,9 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic, |
108 | 108 |
if (pixel_ptr >= avctx->width) |
109 | 109 |
break; |
110 | 110 |
if ((i & 1) == 0) |
111 |
- pic->data[0][row_ptr + pixel_ptr] = stream_byte >> 4; |
|
111 |
+ pic->data[0][line * pic->linesize[0] + pixel_ptr] = stream_byte >> 4; |
|
112 | 112 |
else |
113 |
- pic->data[0][row_ptr + pixel_ptr] = stream_byte & 0x0F; |
|
113 |
+ pic->data[0][line * pic->linesize[0] + pixel_ptr] = stream_byte & 0x0F; |
|
114 | 114 |
pixel_ptr++; |
115 | 115 |
} |
116 | 116 |
} |