Browse code

hqdn3d: Fix out of array read in LOWPASS

CC:libav-stable@libav.org
Signed-off-by: Anton Khirnov <anton@khirnov.net>
(cherry picked from commit 5b3c1aecb253828d09fa9825c5a4aed97badf086)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>

Loren Merritt authored on 2012/09/22 08:43:16
Showing 2 changed files
... ...
@@ -50,10 +50,10 @@ void ff_hqdn3d_row_10_x86(uint8_t *src, uint8_t *dst, uint16_t *line_ant, uint16
50 50
 void ff_hqdn3d_row_16_x86(uint8_t *src, uint8_t *dst, uint16_t *line_ant, uint16_t *frame_ant, ptrdiff_t w, int16_t *spatial, int16_t *temporal);
51 51
 
52 52
 #define LUT_BITS (depth==16 ? 8 : 4)
53
-#define RIGHTSHIFT(a,b) (((a)+(((1<<(b))-1)>>1))>>(b))
54
-#define LOAD(x) ((depth==8 ? src[x] : AV_RN16A(src+(x)*2)) << (16-depth))
55
-#define STORE(x,val) (depth==8 ? dst[x] = RIGHTSHIFT(val, 16-depth)\
56
-                    : AV_WN16A(dst+(x)*2, RIGHTSHIFT(val, 16-depth)))
53
+#define LOAD(x) (((depth == 8 ? src[x] : AV_RN16A(src + (x) * 2)) << (16 - depth))\
54
+                 + (((1 << (16 - depth)) - 1) >> 1))
55
+#define STORE(x,val) (depth == 8 ? dst[x] = (val) >> (16 - depth) : \
56
+                                   AV_WN16A(dst + (x) * 2, (val) >> (16 - depth)))
57 57
 
58 58
 av_always_inline
59 59
 static uint32_t lowpass(int prev, int cur, int16_t *coef, int depth)
... ...
@@ -39,6 +39,7 @@ SECTION .text
39 39
 %endif
40 40
 %if %3 != 16
41 41
     shl    %1, 16-%3
42
+    add    %1, (1<<(15-%3))-1
42 43
 %endif
43 44
 %endmacro
44 45
 
... ...
@@ -86,7 +87,6 @@ ALIGN 16
86 86
     mov       [frameantq+xq*2], t0w
87 87
     movifnidn dstq, dstmp
88 88
 %if %1 != 16
89
-    add    t0d, (1<<(15-%1))-1
90 89
     shr    t0d, 16-%1 ; could eliminate this by storing from t0h, but only with some contraints on register allocation
91 90
 %endif
92 91
 %if %1 == 8