Browse code

unrar - adding proposed changes to fix RAR VMSF_DELTA Filter Signedness error

Mickey Sola authored on 2017/06/30 03:02:03
Showing 1 changed files
... ...
@@ -213,9 +213,9 @@ void rarvm_addbits(rarvm_input_t *rarvm_input, int bits)
213 213
 
214 214
 unsigned int rarvm_getbits(rarvm_input_t *rarvm_input)
215 215
 {
216
-	unsigned int bit_field = 0;
216
+        unsigned int bit_field = 0;
217 217
 
218
-	if (rarvm_input->in_addr < rarvm_input->buf_size) {
218
+        if (rarvm_input->in_addr < rarvm_input->buf_size) {
219 219
             bit_field = (unsigned int) rarvm_input->in_buf[rarvm_input->in_addr] << 16;
220 220
             if (rarvm_input->in_addr+1 < rarvm_input->buf_size) {
221 221
                 bit_field |= (unsigned int) rarvm_input->in_buf[rarvm_input->in_addr+1] << 8;
... ...
@@ -314,10 +314,10 @@ static unsigned int *rarvm_get_operand(rarvm_data_t *rarvm_data,
314 314
 	}
315 315
 }
316 316
 
317
-static unsigned int filter_itanium_getbits(unsigned char *data, int bit_pos, int bit_count)
317
+static unsigned int filter_itanium_getbits(unsigned char *data, unsigned int bit_pos, unsigned int bit_count)
318 318
 {
319
-	int in_addr=bit_pos/8;
320
-	int in_bit=bit_pos&7;
319
+	unsigned int in_addr=bit_pos/8;
320
+	unsigned int in_bit=bit_pos&7;
321 321
 	unsigned int bit_field=(unsigned int)data[in_addr++];
322 322
 	bit_field|=(unsigned int)data[in_addr++] << 8;
323 323
 	bit_field|=(unsigned int)data[in_addr++] << 16;
... ...
@@ -326,10 +326,10 @@ static unsigned int filter_itanium_getbits(unsigned char *data, int bit_pos, int
326 326
 	return(bit_field & (0xffffffff>>(32-bit_count)));
327 327
 }
328 328
 
329
-static void filter_itanium_setbits(unsigned char *data, unsigned int bit_field, int bit_pos, int bit_count)
329
+static void filter_itanium_setbits(unsigned char *data, unsigned int bit_field, unsigned int bit_pos, unsigned int bit_count)
330 330
 {
331
-	int i, in_addr=bit_pos/8;
332
-	int in_bit=bit_pos&7;
331
+	unsigned int i, in_addr=bit_pos/8;
332
+	unsigned int in_bit=bit_pos&7;
333 333
 	unsigned int and_mask=0xffffffff>>(32-bit_count);
334 334
 	and_mask=~(and_mask<<in_bit);
335 335
 
... ...
@@ -346,11 +346,12 @@ static void filter_itanium_setbits(unsigned char *data, unsigned int bit_field,
346 346
 static void execute_standard_filter(rarvm_data_t *rarvm_data, rarvm_standard_filters_t filter_type)
347 347
 {
348 348
 	unsigned char *data, cmp_byte2, cur_byte, *src_data, *dest_data;
349
-	int i, j, data_size, channels, src_pos, dest_pos, border, width, PosR;
350
-	int op_type, cur_channel, byte_count, start_pos, pa, pb, pc;
349
+	unsigned int i, j, data_size, channels, src_pos, dest_pos, border, width, PosR;
350
+	unsigned int op_type, cur_channel, byte_count, start_pos;
351
+	int pa, pb, pc;
351 352
 	unsigned int file_offset, cur_pos, predicted;
352
-	int32_t offset, addr;
353
-	const int file_size=0x1000000;
353
+	uint32_t offset, addr;
354
+	const unsigned int file_size=0x1000000;
354 355
 
355 356
 	switch(filter_type) {
356 357
 	case VMSF_E8:
... ...
@@ -359,7 +360,7 @@ static void execute_standard_filter(rarvm_data_t *rarvm_data, rarvm_standard_fil
359 359
 		data_size = rarvm_data->R[4];
360 360
 		file_offset = rarvm_data->R[6];
361 361
 
362
-		if (((unsigned int)data_size >= VM_GLOBALMEMADDR) || (data_size < 4)) {
362
+		if ((data_size > VM_GLOBALMEMADDR) || (data_size < 4)) {
363 363
 			break;
364 364
 		}
365 365
 
... ...
@@ -370,12 +371,14 @@ static void execute_standard_filter(rarvm_data_t *rarvm_data, rarvm_standard_fil
370 370
 			if (cur_byte==0xe8 || cur_byte==cmp_byte2) {
371 371
 				offset = cur_pos+file_offset;
372 372
 				addr = GET_VALUE(FALSE, data);
373
-				if (addr < 0) {
374
-					if (addr+offset >=0 ) {
373
+				// We check 0x80000000 bit instead of '< 0' comparison
374
+				// not assuming int32 presence or uint size and endianness.
375
+				if ((addr & 0x80000000)!=0) {              // addr<0
376
+					if (((addr+offset) & 0x80000000)==0) {   // addr+offset>=0
375 377
 						SET_VALUE(FALSE, data, addr+file_size);
376 378
 					}
377 379
 				} else {
378
-					if (addr<file_size) {
380
+					if (((addr-file_size) & 0x80000000)!=0) { // addr<file_size
379 381
 						SET_VALUE(FALSE, data, addr-offset);
380 382
 					}
381 383
 				}
... ...
@@ -389,7 +392,7 @@ static void execute_standard_filter(rarvm_data_t *rarvm_data, rarvm_standard_fil
389 389
 		data_size = rarvm_data->R[4];
390 390
 		file_offset = rarvm_data->R[6];
391 391
 		
392
-		if (((unsigned int)data_size >= VM_GLOBALMEMADDR) || (data_size < 21)) {
392
+		if ((data_size > VM_GLOBALMEMADDR) || (data_size < 21)) {
393 393
 			break;
394 394
 		}
395 395
 		
... ...
@@ -432,7 +435,7 @@ static void execute_standard_filter(rarvm_data_t *rarvm_data, rarvm_standard_fil
432 432
 		border = data_size*2;
433 433
 		
434 434
 		SET_VALUE(FALSE, &rarvm_data->mem[VM_GLOBALMEMADDR+0x20], data_size);
435
-		if ((unsigned int)data_size >= VM_GLOBALMEMADDR/2) {
435
+		if (data_size > VM_GLOBALMEMADDR/2 || channels > 1024 || channels == 0) {
436 436
 			break;
437 437
 		}
438 438
 		for (cur_channel=0 ; cur_channel < channels ; cur_channel++) {
... ...
@@ -443,7 +446,7 @@ static void execute_standard_filter(rarvm_data_t *rarvm_data, rarvm_standard_fil
443 443
 		}
444 444
 		break;
445 445
 	case VMSF_RGB: {
446
-		const int channels=3;
446
+		const unsigned int channels=3;
447 447
 		data_size = rarvm_data->R[4];
448 448
 		width = rarvm_data->R[0] - 3;
449 449
 		PosR = rarvm_data->R[1];
... ...
@@ -451,15 +454,14 @@ static void execute_standard_filter(rarvm_data_t *rarvm_data, rarvm_standard_fil
451 451
 		dest_data = src_data + data_size;
452 452
 		
453 453
 		SET_VALUE(FALSE, &rarvm_data->mem[VM_GLOBALMEMADDR+0x20], data_size);
454
-		if ((unsigned int)data_size >= VM_GLOBALMEMADDR/2) {
454
+		if (data_size > VM_GLOBALMEMADDR/2 || data_size < 3 || width > data_size || PosR > 2) {
455 455
 			break;
456 456
 		}
457 457
 		for (cur_channel=0 ; cur_channel < channels; cur_channel++) {
458 458
 			unsigned int prev_byte = 0;
459 459
 			for (i=cur_channel ; i<data_size ; i+=channels) {
460
-				int upper_pos=i-width;
461
-				if (upper_pos >= 3) {
462
-					unsigned char *upper_data = dest_data+upper_pos;
460
+				if (i >= width+3) {
461
+					unsigned char *upper_data = dest_data+i-width;
463 462
 					unsigned int upper_byte = *upper_data;
464 463
 					unsigned int upper_left_byte = *(upper_data-3);
465 464
 					predicted = prev_byte+upper_byte-upper_left_byte;
... ...
@@ -489,13 +491,14 @@ static void execute_standard_filter(rarvm_data_t *rarvm_data, rarvm_standard_fil
489 489
 		break;
490 490
 	}
491 491
 	case VMSF_AUDIO: {
492
-		int channels=rarvm_data->R[0];
492
+		unsigned int channels=rarvm_data->R[0];
493 493
 		data_size = rarvm_data->R[4];
494 494
 		src_data = rarvm_data->mem;
495 495
 		dest_data = src_data + data_size;
496 496
 		
497 497
 		SET_VALUE(FALSE, &rarvm_data->mem[VM_GLOBALMEMADDR+0x20], data_size);
498
-		if ((unsigned int)data_size >= VM_GLOBALMEMADDR/2) {
498
+		// In fact, audio channels never exceed 4.
499
+		if (data_size > VM_GLOBALMEMADDR/2 || channels > 128 || channels == 0) {
499 500
 			break;
500 501
 		}
501 502
 		for (cur_channel=0 ; cur_channel < channels ; cur_channel++) {
... ...
@@ -556,7 +559,7 @@ static void execute_standard_filter(rarvm_data_t *rarvm_data, rarvm_standard_fil
556 556
 		data_size = rarvm_data->R[4];
557 557
 		src_pos = 0;
558 558
 		dest_pos = data_size;
559
-		if ((unsigned int)data_size >= VM_GLOBALMEMADDR/2) {
559
+		if (data_size > VM_GLOBALMEMADDR/2) {
560 560
 			break;
561 561
 		}
562 562
 		while (src_pos < data_size) {