Browse code

golomb: use unsigned arithmetics in svq3_get_ue_golomb()

This prevents undefined behaviour of signed left shift if the coded
value is larger than 2^31. Large values are most likely invalid and
caused errors or by feeding random.

Validate every use of svq3_get_ue_golomb() and changed the place there
the return value was compared with negative numbers. dirac.c was clean,
fixed rv30 and svq3.

Janne Grunau authored on 2012/11/30 23:00:47
Showing 3 changed files
... ...
@@ -107,7 +107,8 @@ static inline int get_ue_golomb_31(GetBitContext *gb){
107 107
     return ff_ue_golomb_vlc_code[buf];
108 108
 }
109 109
 
110
-static inline int svq3_get_ue_golomb(GetBitContext *gb){
110
+static inline unsigned svq3_get_ue_golomb(GetBitContext *gb)
111
+{
111 112
     uint32_t buf;
112 113
 
113 114
     OPEN_READER(re, gb);
... ...
@@ -121,7 +122,7 @@ static inline int svq3_get_ue_golomb(GetBitContext *gb){
121 121
 
122 122
         return ff_interleaved_ue_golomb_vlc_code[buf];
123 123
     }else{
124
-        int ret = 1;
124
+        unsigned ret = 1;
125 125
 
126 126
         do {
127 127
             buf >>= 32 - 8;
... ...
@@ -73,7 +73,7 @@ static int rv30_decode_intra_types(RV34DecContext *r, GetBitContext *gb, int8_t
73 73
 
74 74
     for(i = 0; i < 4; i++, dst += r->intra_types_stride - 4){
75 75
         for(j = 0; j < 4; j+= 2){
76
-            int code = svq3_get_ue_golomb(gb) << 1;
76
+            unsigned code = svq3_get_ue_golomb(gb) << 1;
77 77
             if(code >= 81*2){
78 78
                 av_log(r->s.avctx, AV_LOG_ERROR, "Incorrect intra prediction code\n");
79 79
                 return -1;
... ...
@@ -101,9 +101,9 @@ static int rv30_decode_mb_info(RV34DecContext *r)
101 101
     static const int rv30_b_types[6] = { RV34_MB_SKIP, RV34_MB_B_DIRECT, RV34_MB_B_FORWARD, RV34_MB_B_BACKWARD, RV34_MB_TYPE_INTRA, RV34_MB_TYPE_INTRA16x16 };
102 102
     MpegEncContext *s = &r->s;
103 103
     GetBitContext *gb = &s->gb;
104
-    int code = svq3_get_ue_golomb(gb);
104
+    unsigned code     = svq3_get_ue_golomb(gb);
105 105
 
106
-    if (code < 0 || code > 11) {
106
+    if (code > 11) {
107 107
         av_log(s->avctx, AV_LOG_ERROR, "Incorrect MB type code\n");
108 108
         return -1;
109 109
     }
... ...
@@ -216,17 +216,15 @@ static inline int svq3_decode_block(GetBitContext *gb, DCTELEM *block,
216 216
     static const uint8_t *const scan_patterns[4] =
217 217
     { luma_dc_zigzag_scan, zigzag_scan, svq3_scan, chroma_dc_scan };
218 218
 
219
-    int run, level, sign, vlc, limit;
219
+    int run, level, limit;
220
+    unsigned vlc;
220 221
     const int intra           = 3 * type >> 2;
221 222
     const uint8_t *const scan = scan_patterns[type];
222 223
 
223 224
     for (limit = (16 >> intra); index < 16; index = limit, limit += 8) {
224 225
         for (; (vlc = svq3_get_ue_golomb(gb)) != 0; index++) {
225
-            if (vlc == INVALID_VLC)
226
-                return -1;
227
-
228
-            sign = (vlc & 0x1) - 1;
229
-            vlc  = vlc + 1 >> 1;
226
+            int sign = (vlc & 1) ? 0 : -1;
227
+            vlc      = vlc + 1 >> 1;
230 228
 
231 229
             if (type == 3) {
232 230
                 if (vlc < 3) {
... ...
@@ -786,7 +784,7 @@ static int svq3_decode_slice_header(AVCodecContext *avctx)
786 786
         skip_bits_long(&s->gb, 0);
787 787
     }
788 788
 
789
-    if ((i = svq3_get_ue_golomb(&s->gb)) == INVALID_VLC || i >= 3) {
789
+    if ((i = svq3_get_ue_golomb(&s->gb)) >= 3) {
790 790
         av_log(h->s.avctx, AV_LOG_ERROR, "illegal slice type %d \n", i);
791 791
         return -1;
792 792
     }
... ...
@@ -1010,7 +1008,7 @@ static int svq3_decode_frame(AVCodecContext *avctx, void *data,
1010 1010
     H264Context *h     = &svq3->h;
1011 1011
     MpegEncContext *s  = &h->s;
1012 1012
     int buf_size       = avpkt->size;
1013
-    int m, mb_type;
1013
+    int m;
1014 1014
 
1015 1015
     /* special case for last picture */
1016 1016
     if (buf_size == 0) {
... ...
@@ -1093,6 +1091,7 @@ static int svq3_decode_frame(AVCodecContext *avctx, void *data,
1093 1093
 
1094 1094
     for (s->mb_y = 0; s->mb_y < s->mb_height; s->mb_y++) {
1095 1095
         for (s->mb_x = 0; s->mb_x < s->mb_width; s->mb_x++) {
1096
+            unsigned mb_type;
1096 1097
             h->mb_xy = s->mb_x + s->mb_y * s->mb_stride;
1097 1098
 
1098 1099
             if ((get_bits_count(&s->gb) + 7) >= s->gb.size_in_bits &&
... ...
@@ -1113,7 +1112,7 @@ static int svq3_decode_frame(AVCodecContext *avctx, void *data,
1113 1113
                 mb_type += 8;
1114 1114
             else if (s->pict_type == AV_PICTURE_TYPE_B && mb_type >= 4)
1115 1115
                 mb_type += 4;
1116
-            if ((unsigned)mb_type > 33 || svq3_decode_mb(svq3, mb_type)) {
1116
+            if (mb_type > 33 || svq3_decode_mb(svq3, mb_type)) {
1117 1117
                 av_log(h->s.avctx, AV_LOG_ERROR,
1118 1118
                        "error while decoding MB %d %d\n", s->mb_x, s->mb_y);
1119 1119
                 return -1;