Browse code

avcodec/motion_est: Fix mv_penalty table size

Fixes out of array read

Found-by: Tyson Smith <twsmith@mozilla.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

Michael Niedermayer authored on 2016/01/05 22:41:04
Showing 6 changed files
... ...
@@ -45,7 +45,7 @@
45 45
 /**
46 46
  * Table of number of bits a motion vector component needs.
47 47
  */
48
-static uint8_t mv_penalty[MAX_FCODE+1][MAX_MV*2+1];
48
+static uint8_t mv_penalty[MAX_FCODE+1][MAX_DMV*2+1];
49 49
 
50 50
 /**
51 51
  * Minimal fcode that a motion vector component would need.
... ...
@@ -678,7 +678,7 @@ static av_cold void init_mv_penalty_and_fcode(MpegEncContext *s)
678 678
     int mv;
679 679
 
680 680
     for(f_code=1; f_code<=MAX_FCODE; f_code++){
681
-        for(mv=-MAX_MV; mv<=MAX_MV; mv++){
681
+        for(mv=-MAX_DMV; mv<=MAX_DMV; mv++){
682 682
             int len;
683 683
 
684 684
             if(mv==0) len= ff_mvtab[0][1];
... ...
@@ -699,7 +699,7 @@ static av_cold void init_mv_penalty_and_fcode(MpegEncContext *s)
699 699
                 }
700 700
             }
701 701
 
702
-            mv_penalty[f_code][mv+MAX_MV]= len;
702
+            mv_penalty[f_code][mv+MAX_DMV]= len;
703 703
         }
704 704
     }
705 705
 
... ...
@@ -923,7 +923,7 @@ void ff_estimate_p_frame_motion(MpegEncContext * s,
923 923
     c->penalty_factor    = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
924 924
     c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp);
925 925
     c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp);
926
-    c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV;
926
+    c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV;
927 927
 
928 928
     get_limits(s, 16*mb_x, 16*mb_y);
929 929
     c->skip=0;
... ...
@@ -1090,7 +1090,7 @@ int ff_pre_estimate_p_frame_motion(MpegEncContext * s,
1090 1090
     av_assert0(s->quarter_sample==0 || s->quarter_sample==1);
1091 1091
 
1092 1092
     c->pre_penalty_factor    = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_pre_cmp);
1093
-    c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV;
1093
+    c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV;
1094 1094
 
1095 1095
     get_limits(s, 16*mb_x, 16*mb_y);
1096 1096
     c->skip=0;
... ...
@@ -1139,7 +1139,7 @@ static int estimate_motion_b(MpegEncContext *s, int mb_x, int mb_y,
1139 1139
     const int shift= 1+s->quarter_sample;
1140 1140
     const int mot_stride = s->mb_stride;
1141 1141
     const int mot_xy = mb_y*mot_stride + mb_x;
1142
-    uint8_t * const mv_penalty= c->mv_penalty[f_code] + MAX_MV;
1142
+    uint8_t * const mv_penalty= c->mv_penalty[f_code] + MAX_DMV;
1143 1143
     int mv_scale;
1144 1144
 
1145 1145
     c->penalty_factor    = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
... ...
@@ -1203,8 +1203,8 @@ static inline int check_bidir_mv(MpegEncContext * s,
1203 1203
     //FIXME better f_code prediction (max mv & distance)
1204 1204
     //FIXME pointers
1205 1205
     MotionEstContext * const c= &s->me;
1206
-    uint8_t * const mv_penalty_f= c->mv_penalty[s->f_code] + MAX_MV; // f_code of the prev frame
1207
-    uint8_t * const mv_penalty_b= c->mv_penalty[s->b_code] + MAX_MV; // f_code of the prev frame
1206
+    uint8_t * const mv_penalty_f= c->mv_penalty[s->f_code] + MAX_DMV; // f_code of the prev frame
1207
+    uint8_t * const mv_penalty_b= c->mv_penalty[s->b_code] + MAX_DMV; // f_code of the prev frame
1208 1208
     int stride= c->stride;
1209 1209
     uint8_t *dest_y = c->scratchpad;
1210 1210
     uint8_t *ptr;
... ...
@@ -1417,7 +1417,7 @@ static inline int direct_search(MpegEncContext * s, int mb_x, int mb_y)
1417 1417
     int mx, my, xmin, xmax, ymin, ymax;
1418 1418
     int16_t (*mv_table)[2]= s->b_direct_mv_table;
1419 1419
 
1420
-    c->current_mv_penalty= c->mv_penalty[1] + MAX_MV;
1420
+    c->current_mv_penalty= c->mv_penalty[1] + MAX_DMV;
1421 1421
     ymin= xmin=(-32)>>shift;
1422 1422
     ymax= xmax=   31>>shift;
1423 1423
 
... ...
@@ -1553,11 +1553,11 @@ void ff_estimate_b_frame_motion(MpegEncContext * s,
1553 1553
     if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
1554 1554
 //FIXME mb type penalty
1555 1555
         c->skip=0;
1556
-        c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV;
1556
+        c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV;
1557 1557
         fimin= interlaced_search(s, 0,
1558 1558
                                  s->b_field_mv_table[0], s->b_field_select_table[0],
1559 1559
                                  s->b_forw_mv_table[xy][0], s->b_forw_mv_table[xy][1], 0);
1560
-        c->current_mv_penalty= c->mv_penalty[s->b_code] + MAX_MV;
1560
+        c->current_mv_penalty= c->mv_penalty[s->b_code] + MAX_DMV;
1561 1561
         bimin= interlaced_search(s, 2,
1562 1562
                                  s->b_field_mv_table[1], s->b_field_select_table[1],
1563 1563
                                  s->b_back_mv_table[xy][0], s->b_back_mv_table[xy][1], 0);
... ...
@@ -30,6 +30,7 @@
30 30
 struct MpegEncContext;
31 31
 
32 32
 #define MAX_MV 4096
33
+#define MAX_DMV (2*MAX_MV)
33 34
 #define ME_MAP_SIZE 64
34 35
 
35 36
 #define FF_ME_ZERO 0
... ...
@@ -85,7 +86,7 @@ typedef struct MotionEstContext {
85 85
     op_pixels_func(*hpel_avg)[4];
86 86
     qpel_mc_func(*qpel_put)[16];
87 87
     qpel_mc_func(*qpel_avg)[16];
88
-    uint8_t (*mv_penalty)[MAX_MV * 2 + 1]; ///< bit amount needed to encode a MV
88
+    uint8_t (*mv_penalty)[MAX_DMV * 2 + 1]; ///< bit amount needed to encode a MV
89 89
     uint8_t *current_mv_penalty;
90 90
     int (*sub_motion_search)(struct MpegEncContext *s,
91 91
                              int *mx_ptr, int *my_ptr, int dmin,
... ...
@@ -47,7 +47,7 @@ static const uint8_t svcd_scan_offset_placeholder[] = {
47 47
     0x81, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
48 48
 };
49 49
 
50
-static uint8_t mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
50
+static uint8_t mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1];
51 51
 static uint8_t fcode_tab[MAX_MV * 2 + 1];
52 52
 
53 53
 static uint8_t uni_mpeg1_ac_vlc_len[64 * 64 * 2];
... ...
@@ -1041,7 +1041,7 @@ av_cold void ff_mpeg1_encode_init(MpegEncContext *s)
1041 1041
         }
1042 1042
 
1043 1043
         for (f_code = 1; f_code <= MAX_FCODE; f_code++)
1044
-            for (mv = -MAX_MV; mv <= MAX_MV; mv++) {
1044
+            for (mv = -MAX_DMV; mv <= MAX_DMV; mv++) {
1045 1045
                 int len;
1046 1046
 
1047 1047
                 if (mv == 0) {
... ...
@@ -1064,7 +1064,7 @@ av_cold void ff_mpeg1_encode_init(MpegEncContext *s)
1064 1064
                               2 + bit_size;
1065 1065
                 }
1066 1066
 
1067
-                mv_penalty[f_code][mv + MAX_MV] = len;
1067
+                mv_penalty[f_code][mv + MAX_DMV] = len;
1068 1068
             }
1069 1069
 
1070 1070
 
... ...
@@ -78,7 +78,7 @@ static int sse_mb(MpegEncContext *s);
78 78
 static void denoise_dct_c(MpegEncContext *s, int16_t *block);
79 79
 static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
80 80
 
81
-static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
81
+static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1];
82 82
 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
83 83
 
84 84
 const AVOption ff_mpv_generic_options[] = {
... ...
@@ -292,7 +292,7 @@ static int encode_q_branch(SnowContext *s, int level, int x, int y){
292 292
     c->penalty_factor    = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
293 293
     c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp);
294 294
     c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp);
295
-    c->current_mv_penalty= c->mv_penalty[s->m.f_code=1] + MAX_MV;
295
+    c->current_mv_penalty= c->mv_penalty[s->m.f_code=1] + MAX_DMV;
296 296
 
297 297
     c->xmin = - x*block_w - 16+3;
298 298
     c->ymin = - y*block_w - 16+3;