Originally committed as revision 7287 to svn://svn.ffmpeg.org/ffmpeg/trunk
Michael Niedermayer authored on 2006/12/12 22:31:12... | ... |
@@ -823,20 +823,27 @@ static int sab_diamond_search(MpegEncContext * s, int *best, int dmin, |
823 | 823 |
cmpf= s->dsp.me_cmp[size]; |
824 | 824 |
chroma_cmpf= s->dsp.me_cmp[size+1]; |
825 | 825 |
|
826 |
- for(j=i=0; i<ME_MAP_SIZE; i++){ |
|
826 |
+ /*Note j<MAX_SAB_SIZE is needed if MAX_SAB_SIZE < ME_MAP_SIZE as j can |
|
827 |
+ become larger due to MVs overflowing their ME_MAP_MV_BITS bits space in map |
|
828 |
+ */ |
|
829 |
+ for(j=i=0; i<ME_MAP_SIZE && j<MAX_SAB_SIZE; i++){ |
|
827 | 830 |
uint32_t key= map[i]; |
828 | 831 |
|
829 | 832 |
key += (1<<(ME_MAP_MV_BITS-1)) + (1<<(2*ME_MAP_MV_BITS-1)); |
830 | 833 |
|
831 | 834 |
if((key&((-1)<<(2*ME_MAP_MV_BITS))) != map_generation) continue; |
832 | 835 |
|
833 |
- assert(j<MAX_SAB_SIZE); //max j = number of predictors |
|
834 |
- |
|
835 | 836 |
minima[j].height= score_map[i]; |
836 | 837 |
minima[j].x= key & ((1<<ME_MAP_MV_BITS)-1); key>>=ME_MAP_MV_BITS; |
837 | 838 |
minima[j].y= key & ((1<<ME_MAP_MV_BITS)-1); |
838 | 839 |
minima[j].x-= (1<<(ME_MAP_MV_BITS-1)); |
839 | 840 |
minima[j].y-= (1<<(ME_MAP_MV_BITS-1)); |
841 |
+ |
|
842 |
+ // all entries in map should be in range except if the mv overflows their ME_MAP_MV_BITS bits space |
|
843 |
+ if( minima[j].x > xmax || minima[j].x < xmin |
|
844 |
+ || minima[j].y > ymax || minima[j].y < ymin) |
|
845 |
+ continue; |
|
846 |
+ |
|
840 | 847 |
minima[j].checked=0; |
841 | 848 |
if(minima[j].x || minima[j].y) |
842 | 849 |
minima[j].height+= (mv_penalty[((minima[j].x)<<shift)-pred_x] + mv_penalty[((minima[j].y)<<shift)-pred_y])*penalty_factor; |