Browse code

swscale/input: Fix several invalid shifts related to rgb2yuv constants

Fixes: Invalid shifts
Fixes: #8140
Fixes: #8146

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

Michael Niedermayer authored on 2020/01/22 06:52:19
Showing 1 changed files
... ...
@@ -286,8 +286,8 @@ static av_always_inline void rgb16_32ToUV_c_template(int16_t *dstU,
286 286
                                                      int gsh, int bsh, int S,
287 287
                                                      int32_t *rgb2yuv)
288 288
 {
289
-    const int ru       = rgb2yuv[RU_IDX] << rsh, gu = rgb2yuv[GU_IDX] << gsh, bu = rgb2yuv[BU_IDX] << bsh,
290
-              rv       = rgb2yuv[RV_IDX] << rsh, gv = rgb2yuv[GV_IDX] << gsh, bv = rgb2yuv[BV_IDX] << bsh;
289
+    const int ru       = rgb2yuv[RU_IDX] * (1 << rsh), gu = rgb2yuv[GU_IDX] * (1 << gsh), bu = rgb2yuv[BU_IDX] * (1 << bsh),
290
+              rv       = rgb2yuv[RV_IDX] * (1 << rsh), gv = rgb2yuv[GV_IDX] * (1 << gsh), bv = rgb2yuv[BV_IDX] * (1 << bsh);
291 291
     const unsigned rnd = (256u<<((S)-1)) + (1<<(S-7));
292 292
     int i;
293 293
 
... ...
@@ -314,8 +314,8 @@ static av_always_inline void rgb16_32ToUV_half_c_template(int16_t *dstU,
314 314
                                                           int gsh, int bsh, int S,
315 315
                                                           int32_t *rgb2yuv)
316 316
 {
317
-    const int ru       = rgb2yuv[RU_IDX] << rsh, gu = rgb2yuv[GU_IDX] << gsh, bu = rgb2yuv[BU_IDX] << bsh,
318
-              rv       = rgb2yuv[RV_IDX] << rsh, gv = rgb2yuv[GV_IDX] << gsh, bv = rgb2yuv[BV_IDX] << bsh,
317
+    const int ru       = rgb2yuv[RU_IDX] * (1 << rsh), gu = rgb2yuv[GU_IDX] * (1 << gsh), bu = rgb2yuv[BU_IDX] * (1 << bsh),
318
+              rv       = rgb2yuv[RV_IDX] * (1 << rsh), gv = rgb2yuv[GV_IDX] * (1 << gsh), bv = rgb2yuv[BV_IDX] * (1 << bsh),
319 319
               maskgx   = ~(maskr | maskb);
320 320
     const unsigned rnd = (256U<<(S)) + (1<<(S-6));
321 321
     int i;