Browse code

swscale: fix overflow in gray16 vertical scaling.

This fixes the same overflow as in the RGB48/16-bit YUV scaling;
some filters can overflow both negatively and positively (e.g.
spline/lanczos), so we bias a signed integer so it's "half signed"
and "half unsigned", and can cover overflows in both directions
while maintaining full 31-bit depth.

Signed-off-by: Mans Rullgard <mans@mansr.com>

Ronald S. Bultje authored on 2011/12/18 04:56:40
Showing 1 changed files
... ...
@@ -386,8 +386,8 @@ yuv2gray16_X_c_template(SwsContext *c, const int16_t *lumFilter,
386 386
 
387 387
     for (i = 0; i < (dstW >> 1); i++) {
388 388
         int j;
389
-        int Y1 = 1 << 14;
390
-        int Y2 = 1 << 14;
389
+        int Y1 = (1 << 14) - 0x40000000;
390
+        int Y2 = (1 << 14) - 0x40000000;
391 391
 
392 392
         for (j = 0; j < lumFilterSize; j++) {
393 393
             Y1 += lumSrc[j][i * 2]     * lumFilter[j];
... ...
@@ -395,12 +395,10 @@ yuv2gray16_X_c_template(SwsContext *c, const int16_t *lumFilter,
395 395
         }
396 396
         Y1 >>= 15;
397 397
         Y2 >>= 15;
398
-        if ((Y1 | Y2) & 0x10000) {
399
-            Y1 = av_clip_uint16(Y1);
400
-            Y2 = av_clip_uint16(Y2);
401
-        }
402
-        output_pixel(&dest[i * 2 + 0], Y1);
403
-        output_pixel(&dest[i * 2 + 1], Y2);
398
+        Y1 = av_clip_int16(Y1);
399
+        Y2 = av_clip_int16(Y2);
400
+        output_pixel(&dest[i * 2 + 0], 0x8000 + Y1);
401
+        output_pixel(&dest[i * 2 + 1], 0x8000 + Y2);
404 402
     }
405 403
 }
406 404