@@ -1,3 +1,9 @@

+Sat Dec 15 20:50:02 CET 2007 (tk)

+---------------------------------

+  * libclamav: - use B-M to handle .hdb and .fp databases

+	       - whitelisting now works for MD5 sigs

+	       - other minor cleanups

+

 Sat Dec 15 15:22:54 EET 2007 (edwin)

 -----------------------------------

   * libclamav/phishcheck.c: fix leaks introduced by r3417.

@@ -110,11 +110,14 @@ struct cl_engine {

     /* Roots table */

     void **root;

-    /* MD5 */

-    void **md5_hlist;

+    /* B-M matcher for standard MD5 sigs */

+    void *md5_hdb;

     /* B-M matcher for MD5 sigs for PE sections */

-    void *md5_sect;

+    void *md5_mdb;

+

+    /* B-M matcher for whitelist db */

+    void *md5_fp;

     /* Zip metadata */

     void *zip_mlist;

@@ -93,24 +93,6 @@ int cli_scanbuff(const unsigned char *buffer, uint32_t length, const char **virn

     return ret;

 }

-struct cli_md5_node *cli_vermd5(const unsigned char *md5, const struct cl_engine *engine)

-{

-	struct cli_md5_node *pt;

-

-

-    if(!(pt = engine->md5_hlist[md5[0] & 0xff]))

-	return NULL;

-

-    while(pt) {

-	if(!memcmp(pt->md5, md5, 16))

-	    return pt;

-

-	pt = pt->next;

-    }

-

-    return NULL;

-}

-

 off_t cli_caloff(const char *offstr, struct cli_target_info *info, int fd, cli_file_t ftype, int *ret, unsigned int *maxshift)

 {

 	int (*einfo)(int, struct cli_exe_info *) = NULL;

@@ -214,32 +196,21 @@ off_t cli_caloff(const char *offstr, struct cli_target_info *info, int fd, cli_f

 static int cli_checkfp(int fd, const struct cl_engine *engine)

 {

-	struct cli_md5_node *md5_node;

 	unsigned char *digest;

+	const char *virname;

-    if(engine->md5_hlist) {

-

+    if(engine->md5_fp) {

 	if(!(digest = cli_md5digest(fd))) {

 	    cli_errmsg("cli_checkfp(): Can't generate MD5 checksum\n");

 	    return 0;

 	}

-	if((md5_node = cli_vermd5(digest, engine)) && md5_node->fp) {

-		struct stat sb;

-

-	    if(fstat(fd, &sb))

-		return CL_EIO;

-

-	    if((unsigned int) sb.st_size != md5_node->size) {

-		cli_warnmsg("Detected false positive MD5 match. Please report.\n");

-	    } else {

-		cli_dbgmsg("Eliminated false positive match (fp sig: %s)\n", md5_node->virname);

-		free(digest);

-		return 1;

-	    }

+	if(cli_bm_scanbuff(digest, 16, &virname, engine->md5_fp, 0, 0, -1) == CL_VIRUS) {

+	    cli_dbgmsg("Eliminated false positive match (fp sig: %s)\n", virname);

+	    free(digest);

+	    return 1;

 	}

-

 	free(digest);

     }

@@ -283,7 +254,6 @@ int cli_scandesc(int desc, cli_ctx *ctx, uint8_t otfrec, cli_file_t ftype, uint8

 	struct cli_ac_data gdata, tdata;

 	cli_md5_ctx md5ctx;

 	unsigned char digest[16];

-	struct cli_md5_node *md5_node;

 	struct cli_matcher *groot = NULL, *troot = NULL;

@@ -331,7 +301,7 @@ int cli_scandesc(int desc, cli_ctx *ctx, uint8_t otfrec, cli_file_t ftype, uint8

 	    return ret;

     }

-    if(!ftonly && ctx->engine->md5_hlist)

+    if(!ftonly && ctx->engine->md5_hdb)

 	cli_md5_init(&md5ctx);

     buff = buffer;

@@ -388,7 +358,7 @@ int cli_scandesc(int desc, cli_ctx *ctx, uint8_t otfrec, cli_file_t ftype, uint8

 		    type = ret;

 	    }

-	    if(ctx->engine->md5_hlist)

+	    if(ctx->engine->md5_hdb)

 		cli_md5_update(&md5ctx, buff + shift, bytes);

 	}

@@ -415,24 +385,10 @@ int cli_scandesc(int desc, cli_ctx *ctx, uint8_t otfrec, cli_file_t ftype, uint8

     if(troot)

 	cli_ac_freedata(&tdata);

-    if(!ftonly && ctx->engine->md5_hlist) {

+    if(!ftonly && ctx->engine->md5_hdb) {

 	cli_md5_final(digest, &md5ctx);

-

-	if((md5_node = cli_vermd5(digest, ctx->engine)) && !md5_node->fp) {

-		struct stat sb;

-

-	    if(fstat(desc, &sb))

-		return CL_EIO;

-

-	    if((unsigned int) sb.st_size != md5_node->size) {

-		cli_warnmsg("Detected false positive MD5 match. Please report.\n");

-	    } else {

-		if(ctx->virname)

-		    *ctx->virname = md5_node->virname;

-

-		return CL_VIRUS;

-	    }

-	}

+	if(cli_bm_scanbuff(digest, 16, ctx->virname, ctx->engine->md5_hdb, 0, 0, -1) == CL_VIRUS && (cli_bm_scanbuff(digest, 16, NULL, ctx->engine->md5_fp, 0, 0, -1) != CL_VIRUS))

+	    return CL_VIRUS;

     }

     return otfrec ? type : CL_CLEAN;

@@ -54,14 +54,6 @@ struct cli_matcher {

     uint32_t ac_partsigs, ac_nodes, ac_patterns;

 };

-struct cli_md5_node {

-    char *virname;

-    unsigned char *md5;

-    unsigned int size;

-    unsigned short fp;

-    struct cli_md5_node *next;

-};

-

 struct cli_meta_node {

     int csize, size, method;

     unsigned int crc32, fileno, encrypted, maxdepth;

@@ -83,8 +75,6 @@ int cli_scandesc(int desc, cli_ctx *ctx, uint8_t otfrec, cli_file_t ftype, uint8

 int cli_validatesig(cli_file_t ftype, const char *offstr, off_t fileoff, struct cli_target_info *info, int desc, const char *virname);

-struct cli_md5_node *cli_vermd5(const unsigned char *md5, const struct cl_engine *engine);

-

 off_t cli_caloff(const char *offstr, struct cli_target_info *info, int fd, cli_file_t ftype, int *ret, unsigned int *maxshift);

 #endif

@@ -797,13 +797,13 @@ int cli_scanpe(int desc, cli_ctx *ctx)

 	    if(SCAN_ALGO && (DCONF & PE_CONF_POLIPOS) && !*sname && exe_sections[i].vsz > 40000 && exe_sections[i].vsz < 70000 && exe_sections[i].chr == 0xe0000060) polipos = i;

 	    /* check MD5 section sigs */

-	    md5_sect = ctx->engine->md5_sect;

+	    md5_sect = ctx->engine->md5_mdb;

 	    if((DCONF & PE_CONF_MD5SECT) && md5_sect) {

 		found = 0;

 		for(j = 0; j < md5_sect->soff_len && md5_sect->soff[j] <= exe_sections[i].rsz; j++) {

 		    if(md5_sect->soff[j] == exe_sections[i].rsz) {

 			unsigned char md5_dig[16];

-			if(cli_md5sect(desc, &exe_sections[i], md5_dig) && cli_bm_scanbuff(md5_dig, 16, ctx->virname, ctx->engine->md5_sect, 0, 0, -1) == CL_VIRUS) {

+			if(cli_md5sect(desc, &exe_sections[i], md5_dig) && cli_bm_scanbuff(md5_dig, 16, ctx->virname, ctx->engine->md5_mdb, 0, 0, -1) == CL_VIRUS) {

 				free(section_hdr);

 				free(exe_sections);

 				return CL_VIRUS;

@@ -696,15 +696,48 @@ static int scomp(const void *a, const void *b)

 #define MD5_HDB	    0

 #define MD5_MDB	    1

 #define MD5_FP	    2

-static int cli_loadmd5(FILE *fd, struct cl_engine **engine, unsigned int *signo, uint8_t mode, unsigned int options)

+

+static int cli_md5db_init(struct cl_engine **engine, unsigned int mode)

+{

+	struct cli_matcher *bm = NULL;

+	int ret;

+

+

+    if(mode == MD5_HDB) {

+	bm = (*engine)->md5_hdb = (struct cli_matcher *) cli_calloc(sizeof(struct cli_matcher), 1);

+    } else if(mode == MD5_MDB) {

+	bm = (*engine)->md5_mdb = (struct cli_matcher *) cli_calloc(sizeof(struct cli_matcher), 1);

+    } else {

+	bm = (*engine)->md5_fp = (struct cli_matcher *) cli_calloc(sizeof(struct cli_matcher), 1);

+    }

+

+    if(!bm)

+	return CL_EMEM;

+

+    if((ret = cli_bm_init(bm))) {

+	cli_errmsg("cli_md5db_init: Failed to initialize B-M\n");

+	return ret;

+    }

+

+    return CL_SUCCESS;

+}

+

+#define MD5_DB			    \

+    if(mode == MD5_HDB)		    \

+	db = (*engine)->md5_hdb;    \

+    else if(mode == MD5_MDB)	    \

+	db = (*engine)->md5_mdb;    \

+    else			    \

+	db = (*engine)->md5_fp;

+

+static int cli_loadmd5(FILE *fd, struct cl_engine **engine, unsigned int *signo, unsigned int mode, unsigned int options)

 {

 	char buffer[FILEBUFF], *pt;

 	int ret = CL_SUCCESS;

-	uint8_t size_field = 1, md5_field = 0, found;

-	uint32_t line = 0, i;

-	struct cli_md5_node *new;

-	struct cli_bm_patt *bm_new;

-	struct cli_matcher *md5_sect = NULL;

+	unsigned int size_field = 1, md5_field = 0, found, line = 0, i;

+	uint32_t size;

+	struct cli_bm_patt *new;

+	struct cli_matcher *db = NULL;

     if((ret = cli_initengine(engine, options))) {

@@ -721,22 +754,19 @@ static int cli_loadmd5(FILE *fd, struct cl_engine **engine, unsigned int *signo,

 	line++;

 	cli_chomp(buffer);

-	new = (struct cli_md5_node *) cli_calloc(1, sizeof(struct cli_md5_node));

+	new = (struct cli_bm_patt *) cli_calloc(1, sizeof(struct cli_bm_patt));

 	if(!new) {

 	    ret = CL_EMEM;

 	    break;

 	}

-	if(mode == MD5_FP) /* fp */

-	    new->fp = 1;

-

 	if(!(pt = cli_strtok(buffer, md5_field, ":"))) {

 	    free(new);

 	    ret = CL_EMALFDB;

 	    break;

 	}

-	if(!(new->md5 = (unsigned char *) cli_hex2str(pt))) {

+	if(strlen(pt) != 32 || !(new->pattern = (unsigned char *) cli_hex2str(pt))) {

 	    cli_errmsg("cli_loadmd5: Malformed MD5 string at line %u\n", line);

 	    free(pt);

 	    free(new);

@@ -744,105 +774,60 @@ static int cli_loadmd5(FILE *fd, struct cl_engine **engine, unsigned int *signo,

 	    break;

 	}

 	free(pt);

+	new->length = 16;

 	if(!(pt = cli_strtok(buffer, size_field, ":"))) {

-	    free(new->md5);

+	    free(new->pattern);

 	    free(new);

 	    ret = CL_EMALFDB;

 	    break;

 	}

-	new->size = atoi(pt);

+	size = atoi(pt);

 	free(pt);

 	if(!(new->virname = cli_strtok(buffer, 2, ":"))) {

-	    free(new->md5);

+	    free(new->pattern);

 	    free(new);

 	    ret = CL_EMALFDB;

 	    break;

 	}

-	if(mode == MD5_MDB) { /* section MD5 */

-	    if(!(*engine)->md5_sect) {

-		(*engine)->md5_sect = (struct cli_matcher *) cli_calloc(sizeof(struct cli_matcher), 1);

-		if(!(*engine)->md5_sect) {

-		    free(new->virname);

-		    free(new->md5);

-		    free(new);

-		    ret = CL_EMEM;

-		    break;

-		}

-		if((ret = cli_bm_init((*engine)->md5_sect))) {

-		    cli_errmsg("cli_loadmd5: Can't initialise BM pattern matcher\n");

-		    free(new->virname);

-		    free(new->md5);

-		    free(new);

-		    break;

-		}

-	    }

-	    md5_sect = (*engine)->md5_sect;

-

-	    bm_new = (struct cli_bm_patt *) cli_calloc(1, sizeof(struct cli_bm_patt));

-	    if(!bm_new) {

-		cli_errmsg("cli_loadmd5: Can't allocate memory for bm_new\n");

-		free(new->virname);

-		free(new->md5);

-		free(new);

-		ret = CL_EMEM;

-		break;

-	    }

+	MD5_DB;

+	if(!db && (ret = cli_md5db_init(engine, mode))) {

+	    free(new->pattern);

+	    free(new->virname);

+	    free(new);

+	    break;

+	} else {

+	    MD5_DB;

+	}

-	    bm_new->pattern = new->md5;

-	    bm_new->length = 16;

-	    bm_new->virname = new->virname;

+	if((ret = cli_bm_addpatt(db, new))) {

+	    cli_errmsg("cli_loadmd5: Error adding BM pattern\n");

+	    free(new->pattern);

+	    free(new->virname);

+	    free(new);

+	    break;

+	}

+	if(mode == MD5_MDB) { /* section MD5 */

 	    found = 0;

-	    for(i = 0; i < md5_sect->soff_len; i++) {

-		if(md5_sect->soff[i] == new->size) {

+	    for(i = 0; i < db->soff_len; i++) {

+		if(db->soff[i] == size) {

 		    found = 1;

 		    break;

 		}

 	    }

-

 	    if(!found) {

-		md5_sect->soff_len++;

-		md5_sect->soff = (uint32_t *) cli_realloc2(md5_sect->soff, md5_sect->soff_len * sizeof(uint32_t));

-		if(!md5_sect->soff) {

-		    cli_errmsg("cli_loadmd5: Can't realloc md5_sect->soff\n");

-		    free(bm_new->pattern);

-		    free(bm_new->virname);

-		    free(bm_new);

-		    free(new);

-		    ret = CL_EMEM;

-		    break;

-		}

-		md5_sect->soff[md5_sect->soff_len - 1] = new->size;

-	    }

-

-	    free(new);

-

-	    if((ret = cli_bm_addpatt(md5_sect, bm_new))) {

-		cli_errmsg("cli_loadmd5: Error adding BM pattern\n");

-		free(bm_new->pattern);

-		free(bm_new->virname);

-		free(bm_new);

-		break;

-	    }

-

-	} else {

-	    if(!(*engine)->md5_hlist) {

-		cli_dbgmsg("cli_loadmd5: Initializing MD5 list structure\n");

-		(*engine)->md5_hlist = cli_calloc(256, sizeof(struct cli_md5_node *));

-		if(!(*engine)->md5_hlist) {

-		    free(new->virname);

-		    free(new->md5);

-		    free(new);

+		db->soff_len++;

+		db->soff = (uint32_t *) cli_realloc2(db->soff, db->soff_len * sizeof(uint32_t));

+		if(!db->soff) {

+		    cli_errmsg("cli_loadmd5: Can't realloc db->soff\n");

 		    ret = CL_EMEM;

 		    break;

 		}

+		db->soff[db->soff_len - 1] = size;

 	    }

-

-	    new->next = (*engine)->md5_hlist[new->md5[0] & 0xff];

-	    (*engine)->md5_hlist[new->md5[0] & 0xff] = new;

 	}

     }

@@ -861,8 +846,8 @@ static int cli_loadmd5(FILE *fd, struct cl_engine **engine, unsigned int *signo,

     if(signo)

 	*signo += line;

-    if(md5_sect)

-	qsort(md5_sect->soff, md5_sect->soff_len, sizeof(uint32_t), scomp);

+    if(db && mode == MD5_MDB)

+	qsort(db->soff, db->soff_len, sizeof(uint32_t), scomp);

     return CL_SUCCESS;

 }

@@ -1523,7 +1508,6 @@ int cl_statfree(struct cl_stat *dbstat)

 void cl_free(struct cl_engine *engine)

 {

 	int i;

-	struct cli_md5_node *md5pt, *md5h;

 	struct cli_meta_node *metapt, *metah;

 	struct cli_matcher *root;

@@ -1561,26 +1545,22 @@ void cl_free(struct cl_engine *engine)

 	free(engine->root);

     }

-    if(engine->md5_hlist) {

-	for(i = 0; i < 256; i++) {

-	    md5pt = engine->md5_hlist[i];

-	    while(md5pt) {

-		md5h = md5pt;

-		md5pt = md5pt->next;

-		free(md5h->md5);

-		free(md5h->virname);

-		free(md5h);

-	    }

-	}

-	free(engine->md5_hlist);

+    if((root = engine->md5_hdb)) {

+	cli_bm_free(root);

+	free(root);

     }

-    if((root = engine->md5_sect)) {

+    if((root = engine->md5_mdb)) {

 	cli_bm_free(root);

 	free(root->soff);

 	free(root);

     }

+    if((root = engine->md5_fp)) {

+	cli_bm_free(root);

+	free(root);

+    }

+

     metapt = engine->zip_mlist;

     while(metapt) {

 	metah = metapt;

@@ -174,14 +174,6 @@ static int cli_unrar_scanmetadata(int desc, unrar_metadata_t *metadata, cli_ctx

 	}

     }

-/*

-    TROG - TODO: multi-volume files

-    if((rarlist->item.Flags & 0x03) != 0) {

-	cli_dbgmsg("RAR: Skipping %s (split)\n", rarlist->item.Name);

-	rarlist = rarlist->next;

-	continue;

-    }

-*/

     return ret;

 }

@@ -231,6 +223,10 @@ static int cli_scanrar(int desc, cli_ctx *ctx, off_t sfx_offset, uint32_t *sfx_c

     cli_dbgmsg("in scanrar()\n");

+    if(sfx_offset)

+	if(lseek(desc, sfx_offset, SEEK_SET) == -1)

+	    return CL_EIO;

+

     /* generate the temporary directory */

     dir = cli_gentemp(NULL);

     if(mkdir(dir, 0700)) {

@@ -239,9 +235,6 @@ static int cli_scanrar(int desc, cli_ctx *ctx, off_t sfx_offset, uint32_t *sfx_c

 	return CL_ETMPDIR;

     }

-    if(sfx_offset)

-	lseek(desc, sfx_offset, SEEK_SET);

-

     if((ret = unrar_open(desc, dir, &rar_state)) != UNRAR_OK) {

 	if(!cli_leavetemps_flag)

 	    cli_rmdirs(dir);

@@ -893,10 +886,7 @@ static int cli_vba_scandir(const char *dirname, cli_ctx *ctx)

 		}

 		free(fullname);

 		cli_dbgmsg("VBADir: Decompress WM project '%s' macro:%d key:%d length:%d\n", vba_project->name[i], i, vba_project->key[i], vba_project->length[i]);

-		if(vba_project->length[i])

-		    data = (unsigned char *) wm_decrypt_macro(fd, vba_project->offset[i], vba_project->length[i], vba_project->key[i]);

-		else

-		    data = NULL;

+		data = (unsigned char *) wm_decrypt_macro(fd, vba_project->offset[i], vba_project->length[i], vba_project->key[i]);

 		close(fd);

 		if(!data) {

@@ -1199,6 +1199,9 @@ wm_decrypt_macro(int fd, off_t offset, uint32_t len, unsigned char key)

 {

 	unsigned char *buff;

+	if(!len)

+		return NULL;

+

 	buff = (unsigned char *)cli_malloc(len);

 	if(buff == NULL)

 		return NULL;