This fixes the same overflow as in the RGB48/16-bit YUV scaling;
some filters can overflow both negatively and positively (e.g.
spline/lanczos), so we bias a signed integer so it's "half signed"
and "half unsigned", and can cover overflows in both directions
while maintaining full 31-bit depth.
Signed-off-by: Mans Rullgard <mans@mansr.com>
| ... | ... |
@@ -386,8 +386,8 @@ yuv2gray16_X_c_template(SwsContext *c, const int16_t *lumFilter, |
| 386 | 386 |
|
| 387 | 387 |
for (i = 0; i < (dstW >> 1); i++) {
|
| 388 | 388 |
int j; |
| 389 |
- int Y1 = 1 << 14; |
|
| 390 |
- int Y2 = 1 << 14; |
|
| 389 |
+ int Y1 = (1 << 14) - 0x40000000; |
|
| 390 |
+ int Y2 = (1 << 14) - 0x40000000; |
|
| 391 | 391 |
|
| 392 | 392 |
for (j = 0; j < lumFilterSize; j++) {
|
| 393 | 393 |
Y1 += lumSrc[j][i * 2] * lumFilter[j]; |
| ... | ... |
@@ -395,12 +395,10 @@ yuv2gray16_X_c_template(SwsContext *c, const int16_t *lumFilter, |
| 395 | 395 |
} |
| 396 | 396 |
Y1 >>= 15; |
| 397 | 397 |
Y2 >>= 15; |
| 398 |
- if ((Y1 | Y2) & 0x10000) {
|
|
| 399 |
- Y1 = av_clip_uint16(Y1); |
|
| 400 |
- Y2 = av_clip_uint16(Y2); |
|
| 401 |
- } |
|
| 402 |
- output_pixel(&dest[i * 2 + 0], Y1); |
|
| 403 |
- output_pixel(&dest[i * 2 + 1], Y2); |
|
| 398 |
+ Y1 = av_clip_int16(Y1); |
|
| 399 |
+ Y2 = av_clip_int16(Y2); |
|
| 400 |
+ output_pixel(&dest[i * 2 + 0], 0x8000 + Y1); |
|
| 401 |
+ output_pixel(&dest[i * 2 + 1], 0x8000 + Y2); |
|
| 404 | 402 |
} |
| 405 | 403 |
} |
| 406 | 404 |
|