@@ -1,3 +1,7 @@

+Sun Aug  3 16:12:17 CEST 2008 (acab)

+------------------------------------

+  * libclamav: use md5 based lookup for ole2/vba instead of hashtab (bb#1071)

+

 Fri Aug  1 21:29:33 CEST 2008 (tk)

 ----------------------------------

   * clamconf: handle multiple select options

@@ -198,7 +198,9 @@ libclamav_la_SOURCES = \

 	jsparse/js-norm.c \

 	jsparse/js-norm.h \

 	jsparse/lexglobal.h \

-	jsparse/textbuf.h

+	jsparse/textbuf.h \

+	uniq.c \

+	uniq.h

 libclamav_internal_utils_la_SOURCES=str.c \

 				    str.h \

@@ -90,7 +90,7 @@ am_libclamav_la_OBJECTS = matcher-ac.lo matcher-bm.lo matcher.lo \

 	uuencode.lo phishcheck.lo phish_domaincheck_db.lo \

 	phish_whitelist.lo regex_list.lo regex_suffix.lo mspack.lo \

 	cab.lo entconv.lo hashtab.lo dconf.lo lzma_iface.lo explode.lo \

-	textnorm.lo dlp.lo js-norm.lo

+	textnorm.lo dlp.lo js-norm.lo uniq.lo

 libclamav_la_OBJECTS = $(am_libclamav_la_OBJECTS)

 libclamav_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) \

 	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \

@@ -440,7 +440,9 @@ libclamav_la_SOURCES = \

 	jsparse/js-norm.c \

 	jsparse/js-norm.h \

 	jsparse/lexglobal.h \

-	jsparse/textbuf.h

+	jsparse/textbuf.h \

+	uniq.c \

+	uniq.h

 libclamav_internal_utils_la_SOURCES = str.c \

 				    str.h \

@@ -606,6 +608,7 @@ distclean-compile:

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/textnorm.Plo@am__quote@

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tnef.Plo@am__quote@

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unarj.Plo@am__quote@

+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/uniq.Plo@am__quote@

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unsp.Plo@am__quote@

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/untar.Plo@am__quote@

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unzip.Plo@am__quote@

@@ -567,31 +567,3 @@ ssize_t hashset_toarray(const struct hashset* hs, uint32_t** array)

 	}

 	return j;

 }

-

-struct uniq *uniq_init(uint32_t count) {

-	struct uniq *U;

-	if(!count) return NULL;

-	count = count <= 256 ? 256 : count + (count*20/100);

-	U = cli_malloc(sizeof(*U));

-	if(!U) return NULL;

-	U->uniques = cli_calloc(count, sizeof(uint32_t));

-	if(!U->uniques) {

-		free(U);

-		return NULL;

-	}

-	U->count = count;

-	return U;

-}

-

-uint32_t uniq_add(struct uniq *U, const char *key, uint32_t key_len, uint32_t *rhash) {

-	uint32_t h = hash((const unsigned char *)key, key_len, U->count);

-	if(rhash) *rhash = h;

-	return U->uniques[h]++;

-}

-

-uint32_t uniq_get(struct uniq *U, const char *key, uint32_t key_len, uint32_t *rhash) {

-	uint32_t h = hash((const unsigned char *)key, key_len, U->count);

-	if(rhash) *rhash = h;

-	return U->uniques[h];

-}

-

@@ -104,18 +104,5 @@ int hashset_contains(const struct hashset* hs, const uint32_t key);

 int hashset_clear(struct hashset* hs);

 void hashset_destroy(struct hashset* hs);

 ssize_t hashset_toarray(const struct hashset* hs, uint32_t** array);

-

-

-/* A basic storage for unique IDs */

-struct uniq {

-	uint32_t count;

-        uint32_t *uniques;

-};

-

-struct uniq *uniq_init(uint32_t);

-#define uniq_free(X) do { if(X) free(X->uniques); free(X); } while (0)

-uint32_t uniq_add(struct uniq *, const char *, uint32_t, uint32_t *);

-uint32_t uniq_get(struct uniq *, const char *, uint32_t, uint32_t *);

-

 #endif

@@ -625,7 +625,8 @@ static int handler_writefile(int fd, ole2_header_t *hdr, property_t *prop, const

 	int32_t current_block, ofd, len, offset;

 	char *name, newname[1024];

 	bitset_t *blk_bitset;

-	uint32_t hash, cnt;

+	char *hash;

+	uint32_t cnt;

 	if (prop->type != 2) {

 		/* Not a file */

@@ -640,7 +641,7 @@ static int handler_writefile(int fd, ole2_header_t *hdr, property_t *prop, const

 	name = get_property_name2(prop->name, prop->name_size);

 	if (name) cnt = uniq_add(hdr->U, name, strlen(name), &hash);

 	else cnt = uniq_add(hdr->U, NULL, 0, &hash);

-	snprintf(newname, sizeof(newname), "%s/%u_%u", dir, hash, cnt);

+	snprintf(newname, sizeof(newname), "%s/%s_%u", dir, hash, cnt);

 	newname[sizeof(newname)-1]='\0';

 	cli_dbgmsg("OLE2 [handler_writefile]: Dumping '%s' to '%s'\n", name ? name : "<empty>", newname);

 	if (name) free(name);

@@ -24,7 +24,7 @@

 #define __OLE2_EXTRACT_H

 #include "others.h"

-#include "hashtab.h"

+#include "uniq.h"

 int cli_ole2_extract(int fd, const char *dirname, cli_ctx *ctx, struct uniq **);

@@ -735,7 +735,7 @@ static int cli_scanmscab(int desc, cli_ctx *ctx, off_t sfx_offset)

 static int cli_vba_scandir(const char *dirname, cli_ctx *ctx, struct uniq *U)

 {

-    int ret = CL_CLEAN, i, j, fd, ofd, data_len;

+    int ret = CL_CLEAN, i, j, fd, data_len;

 	vba_project_t *vba_project;

 	DIR *dd;

 	struct dirent *dent;

@@ -748,7 +748,8 @@ static int cli_vba_scandir(const char *dirname, cli_ctx *ctx, struct uniq *U)

 	struct stat statbuf;

 	char *fullname, vbaname[1024];

 	unsigned char *data;

-	uint32_t hash, hashcnt;

+	char *hash;

+	uint32_t hashcnt;

     cli_dbgmsg("VBADir: %s\n", dirname);

@@ -757,17 +758,17 @@ static int cli_vba_scandir(const char *dirname, cli_ctx *ctx, struct uniq *U)

 	if(!(vba_project = (vba_project_t *)cli_vba_readdir(dirname, U, hashcnt))) continue;

 	for(i = 0; i < vba_project->count; i++) {

-	    for(j = 0; j < vba_project->colls[i]; j++) {

-		snprintf(vbaname, 1024, "%s/%u_%u", vba_project->dir, vba_project->name[i], j);

+	    for(j = 0; (unsigned int)j < vba_project->colls[i]; j++) {

+		snprintf(vbaname, 1024, "%s/%s_%u", vba_project->dir, vba_project->name[i], j);

 		vbaname[sizeof(vbaname)-1] = '\0';

 		fd = open(vbaname, O_RDONLY|O_BINARY);

 		if(fd == -1) continue;

-		cli_dbgmsg("VBADir: Decompress VBA project '%u_%u'\n", vba_project->name[i], j);

+		cli_dbgmsg("VBADir: Decompress VBA project '%s_%u'\n", vba_project->name[i], j);

 		data = (unsigned char *)cli_vba_inflate(fd, vba_project->offset[i], &data_len);

 		close(fd);

 		if(!data) {

-		    cli_dbgmsg("VBADir: WARNING: VBA project '%u_%u' decompressed to NULL\n", vba_project->name[i], j);

+		    cli_dbgmsg("VBADir: WARNING: VBA project '%s_%u' decompressed to NULL\n", vba_project->name[i], j);

 		} else {

 		    /* cli_dbgmsg("Project content:\n%s", data); */

 		    if(ctx->scanned)

@@ -792,7 +793,7 @@ static int cli_vba_scandir(const char *dirname, cli_ctx *ctx, struct uniq *U)

     if(ret == CL_CLEAN && (hashcnt = uniq_get(U, "powerpoint document", 19, &hash))) {

 	while(hashcnt--) {

-	    snprintf(vbaname, 1024, "%s/%u_%u", dirname, hash, hashcnt);

+	    snprintf(vbaname, 1024, "%s/%s_%u", dirname, hash, hashcnt);

 	    vbaname[sizeof(vbaname)-1] = '\0';

 	    fd = open(vbaname, O_RDONLY|O_BINARY);

 	    if (fd == -1) continue;

@@ -810,7 +811,7 @@ static int cli_vba_scandir(const char *dirname, cli_ctx *ctx, struct uniq *U)

     if (ret == CL_CLEAN && (hashcnt = uniq_get(U, "worddocument", 12, &hash))) {

 	while(hashcnt--) {

-	    snprintf(vbaname, sizeof(vbaname), "%s/%u_%u", dirname, hash, hashcnt);

+	    snprintf(vbaname, sizeof(vbaname), "%s/%s_%u", dirname, hash, hashcnt);

 	    vbaname[sizeof(vbaname)-1] = '\0';

 	    fd = open(vbaname, O_RDONLY|O_BINARY);

 	    if (fd == -1) continue;

@@ -857,7 +858,7 @@ static int cli_vba_scandir(const char *dirname, cli_ctx *ctx, struct uniq *U)

     /* Check directory for embedded OLE objects */

     hashcnt = uniq_get(U, "_1_ole10native", 14, &hash);

     while(hashcnt--) {

-	snprintf(vbaname, sizeof(vbaname), "%s/%u_%u", dirname, hash, hashcnt);

+	snprintf(vbaname, sizeof(vbaname), "%s/%s_%u", dirname, hash, hashcnt);

 	vbaname[sizeof(vbaname)-1] = '\0';

 	fd = open(vbaname, O_RDONLY|O_BINARY);

new file mode 100644

@@ -0,0 +1,205 @@

+/*

+ *  md5 based hashtab

+ *

+ *  Copyright (C) 2008 Sourcefire, Inc.

+ *

+ *  Authors: aCaB <acab@clamav.net>

+ *

+ *  This program is free software; you can redistribute it and/or modify

+ *  it under the terms of the GNU General Public License version 2 as

+ *  published by the Free Software Foundation.

+ *

+ *  This program is distributed in the hope that it will be useful,

+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of

+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ *  GNU General Public License for more details.

+ *

+ *  You should have received a copy of the GNU General Public License

+ *  along with this program; if not, write to the Free Software

+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

+ *  MA 02110-1301, USA.

+ */

+

+#if HAVE_CONFIG_H

+#include "clamav-config.h"

+#endif

+

+#include "uniq.h"

+#include "md5.h"

+

+#if 0

+struct uniq *uniq_init(uint32_t count) {

+  struct uniq *U;

+  uint32_t i;

+

+  if(!count) return NULL;

+  U = cli_calloc(1, sizeof(*U));

+  if(!U) return NULL;

+  if(cli_ac_init(&U->matcher, 16, 16)) {

+    uniq_free(U);

+    return NULL;

+  }

+  U->custs = cli_calloc(count, sizeof(U->custs));

+  if(!U->custs) {

+    uniq_free(U);

+    return NULL;

+  }

+  U->patts = cli_calloc(count, sizeof(U->patts));

+  if(!U->patts) {

+    uniq_free(U);

+    return NULL;

+  }

+  U->md5s = cli_malloc(count*sizeof(U->md5s));

+  if(!U->md5s) {

+    uniq_free(U);

+    return NULL;

+  }

+

+  U->entries = count;

+

+  for(i=0; i<count; i++) {

+    U->patts[i].pattern = U->md5s[i].md5;

+    U->patts[i].length = 16;

+    U->patts[i].ch[0] = U->patts[i].ch[1] |= CLI_MATCH_IGNORE;

+    U->patts[i].customdata = &U->custs[i];

+  }

+

+  return U;

+}

+

+void uniq_free(struct uniq *U) {

+  uint32_t i;

+  U->matcher.ac_patterns = 0; /* don't free my arrays! */

+  cli_ac_free(&U->matcher);

+  if(U->custs) free(U->custs);

+  if(U->patts) free(U->patts);

+  if(U->md5s) free(U->md5s);

+  free(U);

+}

+

+

+uint32_t uniq_add(struct uniq *U, const char *key, uint32_t key_len, char **rhash) {

+  uint8_t digest[16];

+  struct UNIQCUST *cust;

+  struct cli_ac_data mdata;

+

+  cli_md5_ctx md5;

+  cli_md5_init(&md5);

+  cli_md5_update(&md5, key, key_len);

+  cli_md5_final(digest, &md5);

+

+  cli_ac_initdata(&mdata, 0, 0, AC_DEFAULT_TRACKLEN); /* This can't fail as we don't have parts or lsigs */

+  if (cli_ac_scanbuff(digest,16, NULL, (void *)&cust, NULL, &U->matcher, &mdata,0,0,-1,NULL,AC_SCAN_VIR,NULL)!=CL_VIRUS) {

+    int i;

+    char HEX[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };

+    struct cli_ac_patt *patt = &U->patts[U->matcher.ac_patterns];

+

+    cust = patt->customdata;

+    for(i = 0; i < 16; i++) {

+      cust->name[i*2] = HEX[digest[i]>>4 & 0xf];

+      cust->name[i*2+1] = HEX[digest[i] & 0xf];

+      patt->pattern[i] = digest[i];

+    }

+    cli_ac_addpatt(&U->matcher,patt); /* FIXME this can fail */

+    cli_ac_buildtrie(&U->matcher);

+  }

+

+  cust->count++;

+  if(rhash) *rhash = cust->name;

+  return cust->count;

+}

+

+uint32_t uniq_get(struct uniq *U, const char *key, uint32_t key_len, char **rhash) {

+  uint8_t digest[16];

+  struct UNIQCUST *cust;

+  struct cli_ac_data mdata;

+

+  cli_md5_ctx md5;

+  cli_md5_init(&md5);

+  cli_md5_update(&md5, key, key_len);

+  cli_md5_final(digest, &md5);

+

+  cli_ac_initdata(&mdata, 0, 0, AC_DEFAULT_TRACKLEN); /* This can't fail as we don't have parts or lsigs */

+  if (cli_ac_scanbuff(digest,16, NULL, (void *)&cust, NULL, &U->matcher, &mdata,0,0,-1,NULL,AC_SCAN_VIR,NULL)!=CL_VIRUS)

+    return 0;

+

+  if(rhash) *rhash = cust->name;

+  return cust->count;

+}

+

+#else

+#include <string.h>

+

+struct uniq *uniq_init(uint32_t count) {

+  struct uniq *U;

+  

+  if(!count) return NULL;

+  U = cli_malloc(sizeof(*U));

+  if(!U) return NULL;

+

+  U->md5s = cli_malloc(count * sizeof(*U->md5s));

+  if(!U->md5s) {

+    uniq_free(U);

+    return NULL;

+  }

+

+  U->items = 0;

+  return U;

+}

+

+void uniq_free(struct uniq *U) {

+  free(U->md5s);

+  free(U);

+}

+

+uint32_t uniq_add(struct uniq *U, const char *key, uint32_t key_len, char **rhash) {

+  unsigned int i;

+  uint8_t digest[16];

+  cli_md5_ctx md5;

+  struct UNIQMD5 *m;

+

+  cli_md5_init(&md5);

+  cli_md5_update(&md5, key, key_len);

+  cli_md5_final(digest, &md5);

+

+  for(i=0; i<U->items; i++) {

+    if(memcmp(digest, U->md5s[i].md5, 16)) continue;

+    m = &U->md5s[i];

+    break;

+  }

+  

+  if(i==U->items) {

+    char HEX[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };

+    m = &U->md5s[i];

+    m->count = 0;

+    for(i = 0; i < 16; i++) {

+      m->name[i*2] = HEX[digest[i]>>4 & 0xf];

+      m->name[i*2+1] = HEX[digest[i] & 0xf];

+      m->md5[i] = digest[i]; 

+    }

+    m->name[32] = '\0';

+  }

+  

+  U->items++;

+  if(rhash) *rhash = m->name;

+  return m->count++;

+}

+

+uint32_t uniq_get(struct uniq *U, const char *key, uint32_t key_len, char **rhash) {

+  unsigned int i;

+  uint8_t digest[16];

+  cli_md5_ctx md5;

+

+  cli_md5_init(&md5);

+  cli_md5_update(&md5, key, key_len);

+  cli_md5_final(digest, &md5);

+

+  for(i=0; i<U->items; i++) {

+    if(memcmp(digest, U->md5s[i].md5, 16)) continue;

+    if(rhash) *rhash = U->md5s[i].name;

+    return U->md5s[i].count;

+  }

+

+  return 0;

+}

+#endif

new file mode 100644

@@ -0,0 +1,68 @@

+/*

+ *  md5 based hashtab

+ *

+ *  Copyright (C) 2008 Sourcefire, Inc.

+ *

+ *  Authors: aCaB <acab@clamav.net>

+ *

+ *  This program is free software; you can redistribute it and/or modify

+ *  it under the terms of the GNU General Public License version 2 as

+ *  published by the Free Software Foundation.

+ *

+ *  This program is distributed in the hope that it will be useful,

+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of

+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ *  GNU General Public License for more details.

+ *

+ *  You should have received a copy of the GNU General Public License

+ *  along with this program; if not, write to the Free Software

+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

+ *  MA 02110-1301, USA.

+ */

+

+#ifndef _UNIQ_H

+#define _UNIQ_H

+

+#include "matcher.h"

+#include "cltypes.h"

+

+#if 0

+struct UNIQCUST {

+  char name[33];

+  uint32_t count;

+};

+

+struct UNIQMD5 {

+  uint16_t md5[16];

+};

+

+/* A basic storage for unique IDs */

+struct uniq {

+  struct cli_matcher matcher;

+  struct cli_ac_patt *patts;

+  struct UNIQMD5 *md5s;

+  struct UNIQCUST *custs;

+  uint32_t entries;

+};

+

+#else

+

+struct UNIQMD5 {

+  uint32_t count;

+  uint8_t md5[16];

+  char name[33];

+};

+

+struct uniq {

+  uint32_t items;

+  struct UNIQMD5 *md5s;

+};

+

+#endif

+

+struct uniq *uniq_init(uint32_t);

+void uniq_free(struct uniq *);

+uint32_t uniq_add(struct uniq *, const char *, uint32_t, char **);

+uint32_t uniq_get(struct uniq *, const char *, uint32_t, char **);

+

+#endif

@@ -251,13 +251,12 @@ cli_vba_readdir(const char *dir, struct uniq *U, uint32_t which)

 	unsigned char *buf;

 	const unsigned char vba56_signature[] = { 0xcc, 0x61 };

 	uint16_t record_count, buflen, ffff, byte_count;

-	uint32_t offset, sig, hash, colls;

+	uint32_t offset;

 	int i, j, fd, big_endian = FALSE;

 	vba_project_t *vba_project;

-	const vba_version_t *v;

 	struct vba56_header v56h;

 	off_t seekback;

-	char fullname[1024];

+	char fullname[1024], *hash;

 	cli_dbgmsg("in cli_vba_readdir()\n");

@@ -270,7 +269,7 @@ cli_vba_readdir(const char *dir, struct uniq *U, uint32_t which)

 	if (!uniq_get(U, "_vba_project", 12, &hash))

 		return NULL;

-	snprintf(fullname, sizeof(fullname), "%s/%u_%u", dir, hash, which);

+	snprintf(fullname, sizeof(fullname), "%s/%s_%u", dir, hash, which);

 	fullname[sizeof(fullname)-1] = '\0';

 	fd = open(fullname, O_RDONLY|O_BINARY);

@@ -388,10 +387,10 @@ cli_vba_readdir(const char *dir, struct uniq *U, uint32_t which)

 		ptr = get_unicode_name((const char *)buf, length, big_endian);

 		if(ptr == NULL) break;

 		if (!(vba_project->colls[i]=uniq_get(U, ptr, strlen(ptr), &hash))) {

-			cli_dbgmsg("vba_readdir: cannot find project %s (%u)\n", ptr, hash);

+			cli_dbgmsg("vba_readdir: cannot find project %s (%s)\n", ptr, hash);

 			break;

 		}

-		cli_dbgmsg("vba_readdir: project name: %s (%u)\n", ptr, hash);

+		cli_dbgmsg("vba_readdir: project name: %s (%s)\n", ptr, hash);

 		free(ptr);

 		vba_project->name[i] = hash;

 		if(!read_uint16(fd, &length, big_endian))

@@ -1039,9 +1038,6 @@ cli_wm_readdir(int fd)

 	macro_info_t macro_info;

 	vba_project_t *vba_project;

 	mso_fib_t fib;

-	uint32_t hash, hashcnt;

-	char fullname[1024];

-

 	if (!word_read_fib(fd, &fib))

 		return NULL;

@@ -1240,7 +1236,7 @@ create_vba_project(int record_count, const char *dir, struct uniq *U)

 	if(ret == NULL)

 		return NULL;

-	ret->name = (uint32_t *)cli_malloc(sizeof(uint32_t) * record_count);

+	ret->name = (char **)cli_malloc(sizeof(char *) * record_count);

 	ret->colls = (uint32_t *)cli_malloc(sizeof(uint32_t) * record_count);

 	ret->dir = cli_strdup(dir);

 	ret->offset = (uint32_t *)cli_malloc (sizeof(uint32_t) * record_count);

@@ -25,10 +25,10 @@

 #include "others.h"

 #include "cltypes.h"

-#include "hashtab.h"

+#include "uniq.h"

 typedef struct vba_project_tag {

-	uint32_t *name;

+	char **name;

 	uint32_t *colls;

 	uint32_t *offset;

 	uint32_t *length;	/* for Word 6 macros */