Prevents out of array writes
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit f67a0d115254461649470452058fa3c28c0df294)
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 95ab8d33e1a680f30a5a9605175112008ab81afc)
Conflicts:
libavcodec/huffyuv.c
(cherry picked from commit 277def59fce10d91e3113e5c0f63e22bc4abfa88)
Conflicts:
libavcodec/huffyuv.c
(cherry picked from commit adf022f458d75e2c8041262e1906a249366ad518)
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
... | ... |
@@ -285,6 +285,7 @@ static void generate_joint_tables(HYuvContext *s){ |
285 | 285 |
int len1 = s->len[p][u]; |
286 | 286 |
if(len1 > limit) |
287 | 287 |
continue; |
288 |
+ assert(i < (1 << VLC_BITS)); |
|
288 | 289 |
len[i] = len0 + len1; |
289 | 290 |
bits[i] = (s->bits[0][y] << len1) + s->bits[p][u]; |
290 | 291 |
symbols[i] = (y<<8) + u; |
... | ... |
@@ -318,6 +319,7 @@ static void generate_joint_tables(HYuvContext *s){ |
318 | 318 |
int len2 = s->len[2][r&255]; |
319 | 319 |
if(len2 > limit1) |
320 | 320 |
continue; |
321 |
+ assert(i < (1 << VLC_BITS)); |
|
321 | 322 |
len[i] = len0 + len1 + len2; |
322 | 323 |
bits[i] = (code << len2) + s->bits[2][r&255]; |
323 | 324 |
if(s->decorrelate){ |
... | ... |
@@ -341,6 +343,7 @@ static void generate_joint_tables(HYuvContext *s){ |
341 | 341 |
static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length){ |
342 | 342 |
GetBitContext gb; |
343 | 343 |
int i; |
344 |
+ int ret; |
|
344 | 345 |
|
345 | 346 |
init_get_bits(&gb, src, length*8); |
346 | 347 |
|
... | ... |
@@ -356,7 +359,8 @@ printf("%6X, %2d, %3d\n", s->bits[i][j], s->len[i][j], j); |
356 | 356 |
} |
357 | 357 |
#endif |
358 | 358 |
free_vlc(&s->vlc[i]); |
359 |
- init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, s->bits[i], 4, 4, 0); |
|
359 |
+ if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, s->bits[i], 4, 4, 0)) < 0) |
|
360 |
+ return ret; |
|
360 | 361 |
} |
361 | 362 |
|
362 | 363 |
generate_joint_tables(s); |
... | ... |
@@ -368,6 +372,7 @@ static int read_old_huffman_tables(HYuvContext *s){ |
368 | 368 |
#if 1 |
369 | 369 |
GetBitContext gb; |
370 | 370 |
int i; |
371 |
+ int ret; |
|
371 | 372 |
|
372 | 373 |
init_get_bits(&gb, classic_shift_luma, sizeof(classic_shift_luma)*8); |
373 | 374 |
if(read_len_table(s->len[0], &gb)<0) |
... | ... |
@@ -388,7 +393,8 @@ static int read_old_huffman_tables(HYuvContext *s){ |
388 | 388 |
|
389 | 389 |
for(i=0; i<3; i++){ |
390 | 390 |
free_vlc(&s->vlc[i]); |
391 |
- init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, s->bits[i], 4, 4, 0); |
|
391 |
+ if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, s->bits[i], 4, 4, 0)) < 0) |
|
392 |
+ return ret; |
|
392 | 393 |
} |
393 | 394 |
|
394 | 395 |
generate_joint_tables(s); |