Fixes out of array read
Found-by: Tyson Smith <twsmith@mozilla.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
... | ... |
@@ -45,7 +45,7 @@ |
45 | 45 |
/** |
46 | 46 |
* Table of number of bits a motion vector component needs. |
47 | 47 |
*/ |
48 |
-static uint8_t mv_penalty[MAX_FCODE+1][MAX_MV*2+1]; |
|
48 |
+static uint8_t mv_penalty[MAX_FCODE+1][MAX_DMV*2+1]; |
|
49 | 49 |
|
50 | 50 |
/** |
51 | 51 |
* Minimal fcode that a motion vector component would need. |
... | ... |
@@ -678,7 +678,7 @@ static av_cold void init_mv_penalty_and_fcode(MpegEncContext *s) |
678 | 678 |
int mv; |
679 | 679 |
|
680 | 680 |
for(f_code=1; f_code<=MAX_FCODE; f_code++){ |
681 |
- for(mv=-MAX_MV; mv<=MAX_MV; mv++){ |
|
681 |
+ for(mv=-MAX_DMV; mv<=MAX_DMV; mv++){ |
|
682 | 682 |
int len; |
683 | 683 |
|
684 | 684 |
if(mv==0) len= ff_mvtab[0][1]; |
... | ... |
@@ -699,7 +699,7 @@ static av_cold void init_mv_penalty_and_fcode(MpegEncContext *s) |
699 | 699 |
} |
700 | 700 |
} |
701 | 701 |
|
702 |
- mv_penalty[f_code][mv+MAX_MV]= len; |
|
702 |
+ mv_penalty[f_code][mv+MAX_DMV]= len; |
|
703 | 703 |
} |
704 | 704 |
} |
705 | 705 |
|
... | ... |
@@ -923,7 +923,7 @@ void ff_estimate_p_frame_motion(MpegEncContext * s, |
923 | 923 |
c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp); |
924 | 924 |
c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp); |
925 | 925 |
c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp); |
926 |
- c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV; |
|
926 |
+ c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV; |
|
927 | 927 |
|
928 | 928 |
get_limits(s, 16*mb_x, 16*mb_y); |
929 | 929 |
c->skip=0; |
... | ... |
@@ -1090,7 +1090,7 @@ int ff_pre_estimate_p_frame_motion(MpegEncContext * s, |
1090 | 1090 |
av_assert0(s->quarter_sample==0 || s->quarter_sample==1); |
1091 | 1091 |
|
1092 | 1092 |
c->pre_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_pre_cmp); |
1093 |
- c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV; |
|
1093 |
+ c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV; |
|
1094 | 1094 |
|
1095 | 1095 |
get_limits(s, 16*mb_x, 16*mb_y); |
1096 | 1096 |
c->skip=0; |
... | ... |
@@ -1139,7 +1139,7 @@ static int estimate_motion_b(MpegEncContext *s, int mb_x, int mb_y, |
1139 | 1139 |
const int shift= 1+s->quarter_sample; |
1140 | 1140 |
const int mot_stride = s->mb_stride; |
1141 | 1141 |
const int mot_xy = mb_y*mot_stride + mb_x; |
1142 |
- uint8_t * const mv_penalty= c->mv_penalty[f_code] + MAX_MV; |
|
1142 |
+ uint8_t * const mv_penalty= c->mv_penalty[f_code] + MAX_DMV; |
|
1143 | 1143 |
int mv_scale; |
1144 | 1144 |
|
1145 | 1145 |
c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp); |
... | ... |
@@ -1203,8 +1203,8 @@ static inline int check_bidir_mv(MpegEncContext * s, |
1203 | 1203 |
//FIXME better f_code prediction (max mv & distance) |
1204 | 1204 |
//FIXME pointers |
1205 | 1205 |
MotionEstContext * const c= &s->me; |
1206 |
- uint8_t * const mv_penalty_f= c->mv_penalty[s->f_code] + MAX_MV; // f_code of the prev frame |
|
1207 |
- uint8_t * const mv_penalty_b= c->mv_penalty[s->b_code] + MAX_MV; // f_code of the prev frame |
|
1206 |
+ uint8_t * const mv_penalty_f= c->mv_penalty[s->f_code] + MAX_DMV; // f_code of the prev frame |
|
1207 |
+ uint8_t * const mv_penalty_b= c->mv_penalty[s->b_code] + MAX_DMV; // f_code of the prev frame |
|
1208 | 1208 |
int stride= c->stride; |
1209 | 1209 |
uint8_t *dest_y = c->scratchpad; |
1210 | 1210 |
uint8_t *ptr; |
... | ... |
@@ -1417,7 +1417,7 @@ static inline int direct_search(MpegEncContext * s, int mb_x, int mb_y) |
1417 | 1417 |
int mx, my, xmin, xmax, ymin, ymax; |
1418 | 1418 |
int16_t (*mv_table)[2]= s->b_direct_mv_table; |
1419 | 1419 |
|
1420 |
- c->current_mv_penalty= c->mv_penalty[1] + MAX_MV; |
|
1420 |
+ c->current_mv_penalty= c->mv_penalty[1] + MAX_DMV; |
|
1421 | 1421 |
ymin= xmin=(-32)>>shift; |
1422 | 1422 |
ymax= xmax= 31>>shift; |
1423 | 1423 |
|
... | ... |
@@ -1553,11 +1553,11 @@ void ff_estimate_b_frame_motion(MpegEncContext * s, |
1553 | 1553 |
if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) { |
1554 | 1554 |
//FIXME mb type penalty |
1555 | 1555 |
c->skip=0; |
1556 |
- c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV; |
|
1556 |
+ c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV; |
|
1557 | 1557 |
fimin= interlaced_search(s, 0, |
1558 | 1558 |
s->b_field_mv_table[0], s->b_field_select_table[0], |
1559 | 1559 |
s->b_forw_mv_table[xy][0], s->b_forw_mv_table[xy][1], 0); |
1560 |
- c->current_mv_penalty= c->mv_penalty[s->b_code] + MAX_MV; |
|
1560 |
+ c->current_mv_penalty= c->mv_penalty[s->b_code] + MAX_DMV; |
|
1561 | 1561 |
bimin= interlaced_search(s, 2, |
1562 | 1562 |
s->b_field_mv_table[1], s->b_field_select_table[1], |
1563 | 1563 |
s->b_back_mv_table[xy][0], s->b_back_mv_table[xy][1], 0); |
... | ... |
@@ -30,6 +30,7 @@ |
30 | 30 |
struct MpegEncContext; |
31 | 31 |
|
32 | 32 |
#define MAX_MV 4096 |
33 |
+#define MAX_DMV (2*MAX_MV) |
|
33 | 34 |
#define ME_MAP_SIZE 64 |
34 | 35 |
|
35 | 36 |
#define FF_ME_ZERO 0 |
... | ... |
@@ -85,7 +86,7 @@ typedef struct MotionEstContext { |
85 | 85 |
op_pixels_func(*hpel_avg)[4]; |
86 | 86 |
qpel_mc_func(*qpel_put)[16]; |
87 | 87 |
qpel_mc_func(*qpel_avg)[16]; |
88 |
- uint8_t (*mv_penalty)[MAX_MV * 2 + 1]; ///< bit amount needed to encode a MV |
|
88 |
+ uint8_t (*mv_penalty)[MAX_DMV * 2 + 1]; ///< bit amount needed to encode a MV |
|
89 | 89 |
uint8_t *current_mv_penalty; |
90 | 90 |
int (*sub_motion_search)(struct MpegEncContext *s, |
91 | 91 |
int *mx_ptr, int *my_ptr, int dmin, |
... | ... |
@@ -47,7 +47,7 @@ static const uint8_t svcd_scan_offset_placeholder[] = { |
47 | 47 |
0x81, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, |
48 | 48 |
}; |
49 | 49 |
|
50 |
-static uint8_t mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1]; |
|
50 |
+static uint8_t mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1]; |
|
51 | 51 |
static uint8_t fcode_tab[MAX_MV * 2 + 1]; |
52 | 52 |
|
53 | 53 |
static uint8_t uni_mpeg1_ac_vlc_len[64 * 64 * 2]; |
... | ... |
@@ -1041,7 +1041,7 @@ av_cold void ff_mpeg1_encode_init(MpegEncContext *s) |
1041 | 1041 |
} |
1042 | 1042 |
|
1043 | 1043 |
for (f_code = 1; f_code <= MAX_FCODE; f_code++) |
1044 |
- for (mv = -MAX_MV; mv <= MAX_MV; mv++) { |
|
1044 |
+ for (mv = -MAX_DMV; mv <= MAX_DMV; mv++) { |
|
1045 | 1045 |
int len; |
1046 | 1046 |
|
1047 | 1047 |
if (mv == 0) { |
... | ... |
@@ -1064,7 +1064,7 @@ av_cold void ff_mpeg1_encode_init(MpegEncContext *s) |
1064 | 1064 |
2 + bit_size; |
1065 | 1065 |
} |
1066 | 1066 |
|
1067 |
- mv_penalty[f_code][mv + MAX_MV] = len; |
|
1067 |
+ mv_penalty[f_code][mv + MAX_DMV] = len; |
|
1068 | 1068 |
} |
1069 | 1069 |
|
1070 | 1070 |
|
... | ... |
@@ -78,7 +78,7 @@ static int sse_mb(MpegEncContext *s); |
78 | 78 |
static void denoise_dct_c(MpegEncContext *s, int16_t *block); |
79 | 79 |
static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow); |
80 | 80 |
|
81 |
-static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1]; |
|
81 |
+static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1]; |
|
82 | 82 |
static uint8_t default_fcode_tab[MAX_MV * 2 + 1]; |
83 | 83 |
|
84 | 84 |
const AVOption ff_mpv_generic_options[] = { |
... | ... |
@@ -292,7 +292,7 @@ static int encode_q_branch(SnowContext *s, int level, int x, int y){ |
292 | 292 |
c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp); |
293 | 293 |
c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp); |
294 | 294 |
c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp); |
295 |
- c->current_mv_penalty= c->mv_penalty[s->m.f_code=1] + MAX_MV; |
|
295 |
+ c->current_mv_penalty= c->mv_penalty[s->m.f_code=1] + MAX_DMV; |
|
296 | 296 |
|
297 | 297 |
c->xmin = - x*block_w - 16+3; |
298 | 298 |
c->ymin = - y*block_w - 16+3; |