Browse code

dnxhdenc: fix invalid reads in dnxhd_mb_var_thread().

Do not assume that frame dimensions are mod16 (or that height is mod32
for interlaced).

CC:libav-stable@libav.org

Anton Khirnov authored on 2013/01/22 19:55:54
Showing 3 changed files
... ...
@@ -615,14 +615,35 @@ static void dnxhd_setup_threads_slices(DNXHDEncContext *ctx)
615 615
 static int dnxhd_mb_var_thread(AVCodecContext *avctx, void *arg, int jobnr, int threadnr)
616 616
 {
617 617
     DNXHDEncContext *ctx = avctx->priv_data;
618
-    int mb_y = jobnr, mb_x;
618
+    int mb_y = jobnr, mb_x, x, y;
619
+    int partial_last_row = (mb_y == ctx->m.mb_height - 1) &&
620
+                           ((avctx->height >> ctx->interlaced) & 0xF);
621
+
619 622
     ctx = ctx->thread[threadnr];
620 623
     if (ctx->cid_table->bit_depth == 8) {
621 624
         uint8_t *pix = ctx->thread[0]->src[0] + ((mb_y<<4) * ctx->m.linesize);
622 625
         for (mb_x = 0; mb_x < ctx->m.mb_width; ++mb_x, pix += 16) {
623 626
             unsigned mb  = mb_y * ctx->m.mb_width + mb_x;
624
-            int sum = ctx->m.dsp.pix_sum(pix, ctx->m.linesize);
625
-            int varc = (ctx->m.dsp.pix_norm1(pix, ctx->m.linesize) - (((unsigned)sum*sum)>>8)+128)>>8;
627
+            int sum;
628
+            int varc;
629
+
630
+            if (!partial_last_row && mb_x * 16 <= avctx->width - 16) {
631
+                sum  = ctx->m.dsp.pix_sum(pix, ctx->m.linesize);
632
+                varc = ctx->m.dsp.pix_norm1(pix, ctx->m.linesize);
633
+            } else {
634
+                int bw = FFMIN(avctx->width - 16 * mb_x, 16);
635
+                int bh = FFMIN((avctx->height >> ctx->interlaced) - 16 * mb_y, 16);
636
+                sum = varc = 0;
637
+                for (y = 0; y < bh; y++) {
638
+                    for (x = 0; x < bw; x++) {
639
+                        uint8_t val = pix[x + y * ctx->m.linesize];
640
+                        sum  += val;
641
+                        varc += val * val;
642
+                    }
643
+                }
644
+            }
645
+            varc = (varc - (((unsigned)sum * sum) >> 8) + 128) >> 8;
646
+
626 647
             ctx->mb_cmp[mb].value = varc;
627 648
             ctx->mb_cmp[mb].mb = mb;
628 649
         }
... ...
@@ -1,4 +1,4 @@
1
-3cfbe36a7dd5b48859b8a569d626ef77 *tests/data/fate/vsynth1-dnxhd-1080i.mov
1
+2412f206f5efcbbcc3f2bba0c86b73d4 *tests/data/fate/vsynth1-dnxhd-1080i.mov
2 2
 3031875 tests/data/fate/vsynth1-dnxhd-1080i.mov
3
-0c651e840f860592f0d5b66030d9fa32 *tests/data/fate/vsynth1-dnxhd-1080i.out.rawvideo
3
+34076f61254997c8157eafed1c916472 *tests/data/fate/vsynth1-dnxhd-1080i.out.rawvideo
4 4
 stddev:    6.29 PSNR: 32.15 MAXDIFF:   64 bytes:  7603200/   760320
... ...
@@ -1,4 +1,4 @@
1
-19a91b7da35cecf41e5e3cb322485627 *tests/data/fate/vsynth2-dnxhd-1080i.mov
1
+65ca6385b565b6ea9a2e28150eef1d46 *tests/data/fate/vsynth2-dnxhd-1080i.mov
2 2
 3031875 tests/data/fate/vsynth2-dnxhd-1080i.mov
3
-3c559af629ae0a8fb1a9a0e4b4da7733 *tests/data/fate/vsynth2-dnxhd-1080i.out.rawvideo
3
+42262a2325441b38b3b3c8a42d888e7d *tests/data/fate/vsynth2-dnxhd-1080i.out.rawvideo
4 4
 stddev:    1.31 PSNR: 45.77 MAXDIFF:   23 bytes:  7603200/   760320