Browse code

huffyuvdec: Check init_vlc() return codes.

Prevents out of array writes

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit f67a0d115254461649470452058fa3c28c0df294)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 95ab8d33e1a680f30a5a9605175112008ab81afc)

Conflicts:

libavcodec/huffyuv.c
(cherry picked from commit 277def59fce10d91e3113e5c0f63e22bc4abfa88)

Conflicts:

libavcodec/huffyuv.c
(cherry picked from commit adf022f458d75e2c8041262e1906a249366ad518)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

Michael Niedermayer authored on 2013/01/30 02:29:41
Showing 1 changed files
... ...
@@ -28,6 +28,7 @@
28 28
  * huffyuv codec for libavcodec.
29 29
  */
30 30
 
31
+#include "libavutil/avassert.h"
31 32
 #include "avcodec.h"
32 33
 #include "get_bits.h"
33 34
 #include "put_bits.h"
... ...
@@ -289,6 +290,7 @@ static void generate_joint_tables(HYuvContext *s){
289 289
                     int len1 = s->len[p][u];
290 290
                     if(len1 > limit)
291 291
                         continue;
292
+                    av_assert0(i < (1 << VLC_BITS));
292 293
                     len[i] = len0 + len1;
293 294
                     bits[i] = (s->bits[0][y] << len1) + s->bits[p][u];
294 295
                     symbols[i] = (y<<8) + u;
... ...
@@ -322,6 +324,7 @@ static void generate_joint_tables(HYuvContext *s){
322 322
                     int len2 = s->len[2][r&255];
323 323
                     if(len2 > limit1)
324 324
                         continue;
325
+                    av_assert0(i < (1 << VLC_BITS));
325 326
                     len[i] = len0 + len1 + len2;
326 327
                     bits[i] = (code << len2) + s->bits[2][r&255];
327 328
                     if(s->decorrelate){
... ...
@@ -345,6 +348,7 @@ static void generate_joint_tables(HYuvContext *s){
345 345
 static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length){
346 346
     GetBitContext gb;
347 347
     int i;
348
+    int ret;
348 349
 
349 350
     init_get_bits(&gb, src, length*8);
350 351
 
... ...
@@ -355,7 +359,8 @@ static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length){
355 355
             return -1;
356 356
         }
357 357
         free_vlc(&s->vlc[i]);
358
-        init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, s->bits[i], 4, 4, 0);
358
+        if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, s->bits[i], 4, 4, 0)) < 0)
359
+            return ret;
359 360
     }
360 361
 
361 362
     generate_joint_tables(s);
... ...
@@ -367,6 +372,7 @@ static int read_old_huffman_tables(HYuvContext *s){
367 367
 #if 1
368 368
     GetBitContext gb;
369 369
     int i;
370
+    int ret;
370 371
 
371 372
     init_get_bits(&gb, classic_shift_luma, classic_shift_luma_table_size*8);
372 373
     if(read_len_table(s->len[0], &gb)<0)
... ...
@@ -387,7 +393,8 @@ static int read_old_huffman_tables(HYuvContext *s){
387 387
 
388 388
     for(i=0; i<3; i++){
389 389
         free_vlc(&s->vlc[i]);
390
-        init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, s->bits[i], 4, 4, 0);
390
+        if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, s->bits[i], 4, 4, 0)) < 0)
391
+            return ret;
391 392
     }
392 393
 
393 394
     generate_joint_tables(s);