Browse code

mv overflow in map fix (untested except regression tests)

Originally committed as revision 7287 to svn://svn.ffmpeg.org/ffmpeg/trunk

Michael Niedermayer authored on 2006/12/12 22:31:12
Showing 1 changed files
... ...
@@ -823,20 +823,27 @@ static int sab_diamond_search(MpegEncContext * s, int *best, int dmin,
823 823
     cmpf= s->dsp.me_cmp[size];
824 824
     chroma_cmpf= s->dsp.me_cmp[size+1];
825 825
 
826
-    for(j=i=0; i<ME_MAP_SIZE; i++){
826
+    /*Note j<MAX_SAB_SIZE is needed if MAX_SAB_SIZE < ME_MAP_SIZE as j can
827
+      become larger due to MVs overflowing their ME_MAP_MV_BITS bits space in map
828
+     */
829
+    for(j=i=0; i<ME_MAP_SIZE && j<MAX_SAB_SIZE; i++){
827 830
         uint32_t key= map[i];
828 831
 
829 832
         key += (1<<(ME_MAP_MV_BITS-1)) + (1<<(2*ME_MAP_MV_BITS-1));
830 833
 
831 834
         if((key&((-1)<<(2*ME_MAP_MV_BITS))) != map_generation) continue;
832 835
 
833
-        assert(j<MAX_SAB_SIZE); //max j = number of predictors
834
-
835 836
         minima[j].height= score_map[i];
836 837
         minima[j].x= key & ((1<<ME_MAP_MV_BITS)-1); key>>=ME_MAP_MV_BITS;
837 838
         minima[j].y= key & ((1<<ME_MAP_MV_BITS)-1);
838 839
         minima[j].x-= (1<<(ME_MAP_MV_BITS-1));
839 840
         minima[j].y-= (1<<(ME_MAP_MV_BITS-1));
841
+
842
+        // all entries in map should be in range except if the mv overflows their ME_MAP_MV_BITS bits space
843
+        if(   minima[j].x > xmax || minima[j].x < xmin
844
+           || minima[j].y > ymax || minima[j].y < ymin)
845
+            continue;
846
+
840 847
         minima[j].checked=0;
841 848
         if(minima[j].x || minima[j].y)
842 849
             minima[j].height+= (mv_penalty[((minima[j].x)<<shift)-pred_x] + mv_penalty[((minima[j].y)<<shift)-pred_y])*penalty_factor;