Browse code

avcodec/msrledec: restructure msrle_decode_pal4() based on the line number instead of the pixel pointer

Fixes out of array access
Fixes: da14e86d8462be6493eab16bc2d40f88/asan_heap-oob_204cfd2_528_cov_340150052_COMPRESS.BMP

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit f7e1367f58263593e6cee3c282f7277d7ee9d553)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

Michael Niedermayer authored on 2015/03/24 23:50:12
Showing 1 changed files
... ...
@@ -37,16 +37,14 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic,
37 37
     unsigned char extra_byte, odd_pixel;
38 38
     unsigned char stream_byte;
39 39
     unsigned int pixel_ptr = 0;
40
-    int row_dec = pic->linesize[0];
41
-    int row_ptr = (avctx->height - 1) * row_dec;
42
-    int frame_size = row_dec * avctx->height;
40
+    int line = avctx->height - 1;
43 41
     int i;
44 42
 
45
-    while (row_ptr >= 0) {
43
+    while (line >= 0 && pixel_ptr <= avctx->width) {
46 44
         if (bytestream2_get_bytes_left(gb) <= 0) {
47 45
             av_log(avctx, AV_LOG_ERROR,
48
-                   "MS RLE: bytestream overrun, %d rows left\n",
49
-                   row_ptr);
46
+                   "MS RLE: bytestream overrun, %dx%d left\n",
47
+                   avctx->width - pixel_ptr, line);
50 48
             return AVERROR_INVALIDDATA;
51 49
         }
52 50
         rle_code = stream_byte = bytestream2_get_byteu(gb);
... ...
@@ -55,7 +53,7 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic,
55 55
             stream_byte = bytestream2_get_byte(gb);
56 56
             if (stream_byte == 0) {
57 57
                 /* line is done, goto the next one */
58
-                row_ptr -= row_dec;
58
+                line--;
59 59
                 pixel_ptr = 0;
60 60
             } else if (stream_byte == 1) {
61 61
                 /* decode is done */
... ...
@@ -65,13 +63,12 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic,
65 65
                 stream_byte = bytestream2_get_byte(gb);
66 66
                 pixel_ptr += stream_byte;
67 67
                 stream_byte = bytestream2_get_byte(gb);
68
-                row_ptr -= stream_byte * row_dec;
69 68
             } else {
70 69
                 // copy pixels from encoded stream
71 70
                 odd_pixel =  stream_byte & 1;
72 71
                 rle_code = (stream_byte + 1) / 2;
73 72
                 extra_byte = rle_code & 0x01;
74
-                if (row_ptr + pixel_ptr + stream_byte > frame_size ||
73
+                if (pixel_ptr + 2*rle_code - odd_pixel > avctx->width ||
75 74
                     bytestream2_get_bytes_left(gb) < rle_code) {
76 75
                     av_log(avctx, AV_LOG_ERROR,
77 76
                            "MS RLE: frame/stream ptr just went out of bounds (copy)\n");
... ...
@@ -82,13 +79,13 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic,
82 82
                     if (pixel_ptr >= avctx->width)
83 83
                         break;
84 84
                     stream_byte = bytestream2_get_byteu(gb);
85
-                    pic->data[0][row_ptr + pixel_ptr] = stream_byte >> 4;
85
+                    pic->data[0][line * pic->linesize[0] + pixel_ptr] = stream_byte >> 4;
86 86
                     pixel_ptr++;
87 87
                     if (i + 1 == rle_code && odd_pixel)
88 88
                         break;
89 89
                     if (pixel_ptr >= avctx->width)
90 90
                         break;
91
-                    pic->data[0][row_ptr + pixel_ptr] = stream_byte & 0x0F;
91
+                    pic->data[0][line * pic->linesize[0] + pixel_ptr] = stream_byte & 0x0F;
92 92
                     pixel_ptr++;
93 93
                 }
94 94
 
... ...
@@ -98,7 +95,7 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic,
98 98
             }
99 99
         } else {
100 100
             // decode a run of data
101
-            if (row_ptr + pixel_ptr + stream_byte > frame_size) {
101
+            if (pixel_ptr + rle_code > avctx->width + 1) {
102 102
                 av_log(avctx, AV_LOG_ERROR,
103 103
                        "MS RLE: frame ptr just went out of bounds (run)\n");
104 104
                 return AVERROR_INVALIDDATA;
... ...
@@ -108,9 +105,9 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic,
108 108
                 if (pixel_ptr >= avctx->width)
109 109
                     break;
110 110
                 if ((i & 1) == 0)
111
-                    pic->data[0][row_ptr + pixel_ptr] = stream_byte >> 4;
111
+                    pic->data[0][line * pic->linesize[0] + pixel_ptr] = stream_byte >> 4;
112 112
                 else
113
-                    pic->data[0][row_ptr + pixel_ptr] = stream_byte & 0x0F;
113
+                    pic->data[0][line * pic->linesize[0] + pixel_ptr] = stream_byte & 0x0F;
114 114
                 pixel_ptr++;
115 115
             }
116 116
         }