Browse code

avcodec/cabac: Check initial cabac decoder state

Fixes integer overflows
Fixes: 1430e9c43fae47a24c179c7c54f94918/signal_sigsegv_421427_2340_591e9810c7b09efe501ad84638c9e9f8.264

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Found-by: xiedingbao (Ticket4727)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 8000d484b83aafa752d84fbdbfb352ffe0dc64f8)

Conflicts:

libavcodec/cabac.h

Michael Niedermayer authored on 2015/11/27 21:37:50
Showing 5 changed files
... ...
@@ -51,7 +51,7 @@ void ff_init_cabac_encoder(CABACContext *c, uint8_t *buf, int buf_size){
51 51
  *
52 52
  * @param buf_size size of buf in bits
53 53
  */
54
-void ff_init_cabac_decoder(CABACContext *c, const uint8_t *buf, int buf_size){
54
+int ff_init_cabac_decoder(CABACContext *c, const uint8_t *buf, int buf_size){
55 55
     c->bytestream_start=
56 56
     c->bytestream= buf;
57 57
     c->bytestream_end= buf + buf_size;
... ...
@@ -64,6 +64,9 @@ void ff_init_cabac_decoder(CABACContext *c, const uint8_t *buf, int buf_size){
64 64
 #endif
65 65
     c->low+= ((*c->bytestream++)<<2) + 2;
66 66
     c->range= 0x1FE;
67
+    if ((c->range<<(CABAC_BITS+1)) < c->low)
68
+        return AVERROR_INVALIDDATA;
69
+    return 0;
67 70
 }
68 71
 
69 72
 void ff_init_cabac_states(void)
... ...
@@ -56,7 +56,7 @@ typedef struct CABACContext{
56 56
 }CABACContext;
57 57
 
58 58
 void ff_init_cabac_encoder(CABACContext *c, uint8_t *buf, int buf_size);
59
-void ff_init_cabac_decoder(CABACContext *c, const uint8_t *buf, int buf_size);
59
+int ff_init_cabac_decoder(CABACContext *c, const uint8_t *buf, int buf_size);
60 60
 void ff_init_cabac_states(void);
61 61
 
62 62
 #endif /* AVCODEC_CABAC_H */
... ...
@@ -191,7 +191,8 @@ static av_unused const uint8_t* skip_bytes(CABACContext *c, int n) {
191 191
 #endif
192 192
     if ((int) (c->bytestream_end - ptr) < n)
193 193
         return NULL;
194
-    ff_init_cabac_decoder(c, ptr + n, c->bytestream_end - ptr - n);
194
+    if (ff_init_cabac_decoder(c, ptr + n, c->bytestream_end - ptr - n) < 0)
195
+        return NULL;
195 196
 
196 197
     return ptr;
197 198
 }
... ...
@@ -2026,6 +2026,7 @@ decode_intra_mb:
2026 2026
         const int mb_size = ff_h264_mb_sizes[h->sps.chroma_format_idc] *
2027 2027
                             h->sps.bit_depth_luma >> 3;
2028 2028
         const uint8_t *ptr;
2029
+        int ret;
2029 2030
 
2030 2031
         // We assume these blocks are very rare so we do not optimize it.
2031 2032
         // FIXME The two following lines get the bitstream position in the cabac
... ...
@@ -2042,7 +2043,9 @@ decode_intra_mb:
2042 2042
         sl->intra_pcm_ptr = ptr;
2043 2043
         ptr += mb_size;
2044 2044
 
2045
-        ff_init_cabac_decoder(&sl->cabac, ptr, sl->cabac.bytestream_end - ptr);
2045
+        ret = ff_init_cabac_decoder(&sl->cabac, ptr, sl->cabac.bytestream_end - ptr);
2046
+        if (ret < 0)
2047
+            return ret;
2046 2048
 
2047 2049
         // All blocks are present
2048 2050
         h->cbp_table[mb_xy] = 0xf7ef;
... ...
@@ -2372,9 +2372,11 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg)
2372 2372
         align_get_bits(&sl->gb);
2373 2373
 
2374 2374
         /* init cabac */
2375
-        ff_init_cabac_decoder(&sl->cabac,
2375
+        ret = ff_init_cabac_decoder(&sl->cabac,
2376 2376
                               sl->gb.buffer + get_bits_count(&sl->gb) / 8,
2377 2377
                               (get_bits_left(&sl->gb) + 7) / 8);
2378
+        if (ret < 0)
2379
+            return ret;
2378 2380
 
2379 2381
         ff_h264_init_cabac_states(h, sl);
2380 2382