@@ -1,3 +1,7 @@

+Fri Mar 8 17:48:34 EDT 2013 (dar)

+------------------------------------

+ * libclamav: SHA1/SHA256 handling changes and wildcard-size support

+

 Thu Mar 7 19:38:34 EDT 2013 (dar)

 ------------------------------------

  * sigtool: Add print-certs command to allow dumping certs without a scan

@@ -201,10 +201,10 @@ attachment.exe: OK

     \section{Signature formats}

-    \subsection{MD5}

-    The easiest way to create signatures for ClamAV is to use MD5 checksums,

-    however this method can be only used against static malware. To create

-    a signature for \verb+test.exe+ use the \verb+--md5+ option of sigtool:

+    \subsection{Hash-based signatures}

+    The easiest way to create signatures for ClamAV is to use filehash checksums,

+    however this method can be only used against static malware. To create a

+    MD5 signature for \verb+test.exe+ use the \verb+--md5+ option of sigtool:

     \begin{verbatim}

 zolw@localhost:/tmp/test$ sigtool --md5 test.exe > test.hdb

 zolw@localhost:/tmp/test$ cat test.hdb 

@@ -238,17 +238,50 @@ Time: 0.024 sec (0 m 0 s)

     left in /tmp. Please keep in mind that a hash signature will stop

     matching as soon as a single byte changes in the target file.}

-    \subsection{MD5, PE section based}

-    You can create a MD5 signature for a specific section in a PE file.

+    \subsection{SHA1, SHA256}

+    ClamAV 0.98 has also added support for SHA1 and SHA256 file checksums.

+    The format is the same as for MD5 file checksum. 

+    It can differentiate between them based on the length of the hash string

+    in the signature. For best backwards compatibility, these should be

+    placed inside a \verb+*.hsb+ file. The format is:

+    \begin{verbatim}

+HashString:FileSize:MalwareName

+    \end{verbatim}

+

+    \subsection{PE section based}

+    You can create a hash signature for a specific section in a PE file.

     Such signatures shall be stored inside \verb+.mdb+ files in the

     following format:

     \begin{verbatim}

-PESectionSize:MD5:MalwareName

+PESectionSize:PESectionHash:MalwareName

     \end{verbatim}

     The easiest way to generate MD5 based section signatures is to extract

     target PE sections into separate files and then run sigtool with the

     option \verb+--mdb+

+    ClamAV 0.98 has also added support for SHA1 and SHA256 section based

+    signatures. The format is the same as for MD5 PE section based signatures.

+    It can differentiate between them based on the length of the hash string

+    in the signature. For best backwards compatibility, these should be

+    placed inside a \verb+*.msb+ file.

+

+    \subsection{Unknown size}

+    ClamAV 0.98 has also added support for hash signatures where the size

+    is not known but the hash is. It is much more performance-efficient to

+    use signatures with specific sizes, so be cautious when using this

+    feature. For these cases, the '*' character can be used in the size

+    field. To ensure proper backwards compatibility with older versions of

+    ClamAV, these signatures must have a minimum functional level of 73 or

+    higher. Signatures that use the wildcard size without this level set

+    will be rejected as malformed.

+    \begin{verbatim}

+Sample .hsb signature matching any size

+HashString:*:MalwareName:73

+

+Sample .msb signature matching any size

+*:PESectionHash:MalwareName:73

+    \end{verbatim}

+

     \subsection{Body-based signatures}

     ClamAV stores all body-based signatures in a hexadecimal format. In this

     section by a hex-signature we mean a fragment of malware's body converted

@@ -670,7 +703,10 @@ fileno:max depth

     \subsection{Whitelist databases}

     To whitelist a specific file use the MD5 signature format and place

-    it inside a database file with the extension of \verb+.fp+.\\

+    it inside a database file with the extension of \verb+.fp+.

+    To whitelist a specific file with the SHA1 or SHA256 file hash signature

+    format, place the signature inside a database file with the extension

+    of \verb+.sfp+.\\

     \noindent

     To whitelist a specific signature from the database you just add

@@ -28,7 +28,7 @@

 int hm_addhash_str(struct cli_matcher *root, const char *strhash, uint32_t size, const char *virusname) {

     enum CLI_HASH_TYPE type;

-    char binhash[32];

+    char binhash[CLI_HASHLEN_MAX];

     int hlen;

     if(!root || !strhash) {

@@ -36,7 +36,8 @@ int hm_addhash_str(struct cli_matcher *root, const char *strhash, uint32_t size,

 	return CL_ENULLARG;

     }

-    if(!size || size == (uint32_t)-1) {

+    /* size 0 here is now a wildcard size match */

+    if(size == (uint32_t)-1) {

 	cli_errmsg("hm_addhash_str: null or invalid size (%u)\n", size);

 	return CL_EARG;

     }

@@ -65,9 +66,9 @@ int hm_addhash_str(struct cli_matcher *root, const char *strhash, uint32_t size,

 }

 static const unsigned int hashlen[] = {

-    16, /* CLI_HASH_MD5 */

-    20, /* CLI_HASH_SHA1 */

-    32, /* CLI_HASH_SHA256 */

+    CLI_HASHLEN_MD5,

+    CLI_HASHLEN_SHA1,

+    CLI_HASHLEN_SHA256

 };

 int hm_addhash_bin(struct cli_matcher *root, const void *binhash, enum CLI_HASH_TYPE type, uint32_t size, const char *virusname) {

@@ -77,32 +78,38 @@ int hm_addhash_bin(struct cli_matcher *root, const void *binhash, enum CLI_HASH_

     struct cli_htu32 *ht;

     int i;

-    ht = &root->hm.sizehashes[type];

-    if(!root->hm.sizehashes[type].capacity) {

-	i = cli_htu32_init(ht, 64, root->mempool);

-	if(i) return i;

+    if (size) {

+        /* size non-zero, find sz_hash element in size-driven hashtable  */

+        ht = &root->hm.sizehashes[type];

+        if(!root->hm.sizehashes[type].capacity) {

+            i = cli_htu32_init(ht, 64, root->mempool);

+            if(i) return i;

+        }

+

+        item = cli_htu32_find(ht, size);

+        if(!item) {

+	    struct cli_htu32_element htitem;

+	    szh = mpool_calloc(root->mempool, 1, sizeof(*szh));

+	    if(!szh) {

+	        cli_errmsg("hm_addhash_bin: failed to allocate size hash\n");

+	        return CL_EMEM;

+	    }

+

+	    htitem.key = size;

+	    htitem.data.as_ptr = szh;

+	    i = cli_htu32_insert(ht, &htitem, root->mempool);

+	    if(i) {

+	        cli_errmsg("hm_addhash_bin: failed to add item to hashtab");

+	        mpool_free(root->mempool, szh);

+	        return i;

+	    }

+        } else

+	    szh = (struct cli_sz_hash *)item->data.as_ptr;

+    }

+    else {

+        /* size 0 = wildcard */

+        szh = &root->hwild.hashes[type];

     }

-

-    item = cli_htu32_find(ht, size);

-    if(!item) {

-	struct cli_htu32_element htitem;

-	szh = mpool_calloc(root->mempool, 1, sizeof(*szh));

-	if(!szh) {

-	    cli_errmsg("hm_addhash_bin: failed to allocate size hash\n");

-	    return CL_EMEM;

-	}

-

-	htitem.key = size;

-	htitem.data.as_ptr = szh;

-	i = cli_htu32_insert(ht, &htitem, root->mempool);

-	if(i) {

-	    cli_errmsg("hm_addhash_bin: failed to add item to hashtab");

-	    mpool_free(root->mempool, szh);

-	    return i;

-	}

-    } else

-	szh = (struct cli_sz_hash *)item->data.as_ptr;

-

     szh->items++;

     szh->hash_array = mpool_realloc2(root->mempool, szh->hash_array, hlen * szh->items);

@@ -129,7 +136,6 @@ int hm_addhash_bin(struct cli_matcher *root, const void *binhash, enum CLI_HASH_

     return 0;

 }

-

 static inline int hm_cmp(const uint8_t *itm, const uint8_t *ref, unsigned int keylen) {

 #if WORDS_BIGENDIAN == 0

     uint32_t i = *(uint32_t *)itm, r = *(uint32_t *)ref;

@@ -142,7 +148,7 @@ static inline int hm_cmp(const uint8_t *itm, const uint8_t *ref, unsigned int ke

 }

 static void hm_sort(struct cli_sz_hash *szh, size_t l, size_t r, unsigned int keylen) {

-    uint8_t piv[32], tmph[32];

+    uint8_t piv[CLI_HASHLEN_MAX], tmph[CLI_HASHLEN_MAX];

     size_t l1, r1;

     const char *tmpv;

@@ -181,7 +187,7 @@ static void hm_sort(struct cli_sz_hash *szh, size_t l, size_t r, unsigned int ke

     hm_sort(szh, r1, r, keylen);

 }

-

+/* flush both size-specific and agnostic hash sets */

 void hm_flush(struct cli_matcher *root) {

     enum CLI_HASH_TYPE type;

@@ -191,18 +197,27 @@ void hm_flush(struct cli_matcher *root) {

     for(type = CLI_HASH_MD5; type < CLI_HASH_AVAIL_TYPES; type++) {

 	struct cli_htu32 *ht = &root->hm.sizehashes[type];

 	const struct cli_htu32_element *item = NULL;

+	struct cli_sz_hash *szh = NULL;

 	if(!root->hm.sizehashes[type].capacity)

 	    continue;

 	while((item = cli_htu32_next(ht, item))) {

-	    struct cli_sz_hash *szh = (struct cli_sz_hash *)item->data.as_ptr;

+	    szh = (struct cli_sz_hash *)item->data.as_ptr;

 	    unsigned int keylen = hashlen[type];

 	    if(szh->items > 1)

 		hm_sort(szh, 0, szh->items, keylen);

 	}

     }

+

+    for(type = CLI_HASH_MD5; type < CLI_HASH_AVAIL_TYPES; type++) {

+	struct cli_sz_hash *szh = &root->hwild.hashes[type];

+	unsigned int keylen = hashlen[type];

+

+	if(szh->items > 1)

+	    hm_sort(szh, 0, szh->items, keylen);

+    }

 }

@@ -210,20 +225,18 @@ int cli_hm_have_size(const struct cli_matcher *root, enum CLI_HASH_TYPE type, ui

     return (size && size != 0xffffffff && root && root->hm.sizehashes[type].capacity && cli_htu32_find(&root->hm.sizehashes[type], size));

 }

-int cli_hm_scan(const unsigned char *digest, uint32_t size, const char **virname, const struct cli_matcher *root, enum CLI_HASH_TYPE type) {

-    const struct cli_htu32_element *item;

+int cli_hm_have_wild(const struct cli_matcher *root, enum CLI_HASH_TYPE type) {

+    return (root && root->hwild.hashes[type].items);

+}

+

+/* cli_hm_scan will scan only size-specific hashes, if any */

+static int hm_scan(const unsigned char *digest, const char **virname, const struct cli_sz_hash *szh, enum CLI_HASH_TYPE type) {

     unsigned int keylen;

-    struct cli_sz_hash *szh;

     size_t l, r;

-    if(!digest || !size || size == 0xffffffff || !root || !root->hm.sizehashes[type].capacity)

+    if(!digest || !szh || !szh->items)

 	return CL_CLEAN;

-    item = cli_htu32_find(&root->hm.sizehashes[type], size);

-    if(!item)

-	return CL_CLEAN;

-

-    szh = (struct cli_sz_hash *)item->data.as_ptr;

     keylen = hashlen[type];

     l = 0;

@@ -247,6 +260,32 @@ int cli_hm_scan(const unsigned char *digest, uint32_t size, const char **virname

     return CL_CLEAN;

 }

+/* cli_hm_scan will scan only size-specific hashes, if any */

+int cli_hm_scan(const unsigned char *digest, uint32_t size, const char **virname, const struct cli_matcher *root, enum CLI_HASH_TYPE type) {

+    const struct cli_htu32_element *item;

+    struct cli_sz_hash *szh;

+

+    if(!digest || !size || size == 0xffffffff || !root || !root->hm.sizehashes[type].capacity)

+	return CL_CLEAN;

+

+    item = cli_htu32_find(&root->hm.sizehashes[type], size);

+    if(!item)

+	return CL_CLEAN;

+

+    szh = (struct cli_sz_hash *)item->data.as_ptr;

+

+    return hm_scan(digest, virname, szh, type);

+}

+

+/* cli_hm_scan_wild will scan only size-agnostic hashes, if any */

+int cli_hm_scan_wild(const unsigned char *digest, const char **virname, const struct cli_matcher *root, enum CLI_HASH_TYPE type) {

+    if(!digest || !root || !root->hwild.hashes[type].items)

+	return CL_CLEAN;

+

+    return hm_scan(digest, virname, &root->hwild.hashes[type], type);

+}

+

+/* free both size-specific and agnostic hash sets */

 void hm_free(struct cli_matcher *root) {

     enum CLI_HASH_TYPE type;

@@ -262,7 +301,6 @@ void hm_free(struct cli_matcher *root) {

 	while((item = cli_htu32_next(ht, item))) {

 	    struct cli_sz_hash *szh = (struct cli_sz_hash *)item->data.as_ptr;

-	    unsigned int keylen = hashlen[type];

 	    mpool_free(root->mempool, szh->hash_array);

 	    while(szh->items)

@@ -272,5 +310,17 @@ void hm_free(struct cli_matcher *root) {

 	}

 	cli_htu32_free(ht, root->mempool);

     }

+

+    for(type = CLI_HASH_MD5; type < CLI_HASH_AVAIL_TYPES; type++) {

+	struct cli_sz_hash *szh = &root->hwild.hashes[type];

+

+	if(!szh->items)

+	    continue;

+

+	mpool_free(root->mempool, szh->hash_array);

+	while(szh->items)

+	    mpool_free(root->mempool, (void *)szh->virusnames[--szh->items]);

+	mpool_free(root->mempool, szh->virusnames);

+    }

 }

@@ -29,7 +29,7 @@

 #include "hashtab.h"

 enum CLI_HASH_TYPE {

-    CLI_HASH_MD5,

+    CLI_HASH_MD5 = 0,

     CLI_HASH_SHA1,

     CLI_HASH_SHA256,

@@ -37,6 +37,13 @@ enum CLI_HASH_TYPE {

     CLI_HASH_AVAIL_TYPES

 };

+#define CLI_HASHLEN_MD5 16

+#define CLI_HASHLEN_SHA1 20

+#define CLI_HASHLEN_SHA256 32

+#define CLI_HASHLEN_MAX 32

+

+#define cli_hashlength(t) ((t == CLI_HASH_MD5) ? CLI_HASHLEN_MD5 : ((t == CLI_HASH_SHA1) ? CLI_HASHLEN_SHA1 : CLI_HASHLEN_SHA256))

+

 struct cli_sz_hash {

     uint8_t *hash_array;

     const char **virusnames;

@@ -48,11 +55,17 @@ struct cli_hash_patt {

     struct cli_htu32 sizehashes[CLI_HASH_AVAIL_TYPES];

 };

+struct cli_hash_wild {

+    struct cli_sz_hash hashes[CLI_HASH_AVAIL_TYPES];

+};

+

 int hm_addhash_str(struct cli_matcher *root, const char *strhash, uint32_t size, const char *virusname);

 int hm_addhash_bin(struct cli_matcher *root, const void *binhash, enum CLI_HASH_TYPE type, uint32_t size, const char *virusname);

 void hm_flush(struct cli_matcher *root);

 int cli_hm_scan(const unsigned char *digest, uint32_t size, const char **virname, const struct cli_matcher *root, enum CLI_HASH_TYPE type);

+int cli_hm_scan_wild(const unsigned char *digest, const char **virname, const struct cli_matcher *root, enum CLI_HASH_TYPE type);

 int cli_hm_have_size(const struct cli_matcher *root, enum CLI_HASH_TYPE type, uint32_t size);

+int cli_hm_have_wild(const struct cli_matcher *root, enum CLI_HASH_TYPE type);

 void hm_free(struct cli_matcher *root);

 #endif

@@ -425,6 +425,10 @@ int cli_checkfp(unsigned char *digest, size_t size, cli_ctx *ctx)

 	cli_dbgmsg("cli_checkfp(md5): Found false positive detection (fp sig: %s), size: %d\n", virname, (int)size);

 	return CL_CLEAN;

     }

+    else if(cli_hm_scan_wild(digest, &virname, ctx->engine->hm_fp, CLI_HASH_MD5) == CL_VIRUS) {

+	cli_dbgmsg("cli_checkfp(md5): Found false positive detection (fp sig: %s), size: *\n", virname);

+	return CL_CLEAN;

+    }

     if(cli_debug_flag || ctx->engine->cb_hash) {

 	for(i = 0; i < 16; i++)

@@ -438,8 +442,11 @@ int cli_checkfp(unsigned char *digest, size_t size, cli_ctx *ctx)

 	do_dsig_check = strncmp("W32S.", cli_get_last_virus(ctx), 5);

     map = *ctx->fmap;

-    have_sha1 = cli_hm_have_size(ctx->engine->hm_fp, CLI_HASH_SHA1, size) || (cli_hm_have_size(ctx->engine->hm_fp, CLI_HASH_SHA1, 1) && do_dsig_check);

-    have_sha256 = cli_hm_have_size(ctx->engine->hm_fp, CLI_HASH_SHA256, size);

+    have_sha1 = cli_hm_have_size(ctx->engine->hm_fp, CLI_HASH_SHA1, size)

+	 || cli_hm_have_wild(ctx->engine->hm_fp, CLI_HASH_SHA1)

+	 || (cli_hm_have_size(ctx->engine->hm_fp, CLI_HASH_SHA1, 1) && do_dsig_check);

+    have_sha256 = cli_hm_have_size(ctx->engine->hm_fp, CLI_HASH_SHA256, size)

+	 || cli_hm_have_wild(ctx->engine->hm_fp, CLI_HASH_SHA256);

     if(have_sha1 || have_sha256) {

 	if((ptr = fmap_need_off_once(map, 0, size))) {

 	    if(have_sha1) {

@@ -450,6 +457,10 @@ int cli_checkfp(unsigned char *digest, size_t size, cli_ctx *ctx)

 		    cli_dbgmsg("cli_checkfp(sha1): Found false positive detection (fp sig: %s)\n", virname);

 		    return CL_CLEAN;

 		}

+		if(cli_hm_scan_wild(&shash1[SHA1_HASH_SIZE], &virname, ctx->engine->hm_fp, CLI_HASH_SHA1) == CL_VIRUS) {

+		    cli_dbgmsg("cli_checkfp(sha1): Found false positive detection (fp sig: %s)\n", virname);

+		    return CL_CLEAN;

+		}

 		if(do_dsig_check && cli_hm_scan(&shash1[SHA1_HASH_SIZE], 1, &virname, ctx->engine->hm_fp, CLI_HASH_SHA1) == CL_VIRUS) {

 		    cli_dbgmsg("cli_checkfp(sha1): Found false positive detection via catalog file\n");

 		    return CL_CLEAN;

@@ -463,6 +474,10 @@ int cli_checkfp(unsigned char *digest, size_t size, cli_ctx *ctx)

 		    cli_dbgmsg("cli_checkfp(sha256): Found false positive detection (fp sig: %s)\n", virname);

 		    return CL_CLEAN;

 		}

+		if(cli_hm_scan_wild(&shash256[SHA256_HASH_SIZE], &virname, ctx->engine->hm_fp, CLI_HASH_SHA256) == CL_VIRUS) {

+		    cli_dbgmsg("cli_checkfp(sha256): Found false positive detection (fp sig: %s)\n", virname);

+		    return CL_CLEAN;

+		}

 	    }

 	}

     }

@@ -771,13 +786,15 @@ int cli_fmap_scandesc(cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli

 	    memcpy(digest[CLI_HASH_MD5], refhash, 16);

 	}

-	if(cli_hm_have_size(hdb, CLI_HASH_SHA1, map->len) || cli_hm_have_size(fp, CLI_HASH_SHA1, map->len)) {

+	if(cli_hm_have_size(hdb, CLI_HASH_SHA1, map->len) || cli_hm_have_wild(hdb, CLI_HASH_SHA1)

+		|| cli_hm_have_size(fp, CLI_HASH_SHA1, map->len) || cli_hm_have_wild(fp, CLI_HASH_SHA1) ) {

 	    SHA1Init(&sha1ctx);

 	    compute_hash[CLI_HASH_SHA1] = 1;

 	} else

 	    compute_hash[CLI_HASH_SHA1] = 0;

-	if(cli_hm_have_size(hdb, CLI_HASH_SHA256, map->len) || cli_hm_have_size(fp, CLI_HASH_SHA256, map->len)) {

+	if(cli_hm_have_size(hdb, CLI_HASH_SHA256, map->len) || cli_hm_have_wild(hdb, CLI_HASH_SHA256)

+		|| cli_hm_have_size(fp, CLI_HASH_SHA256, map->len) || cli_hm_have_wild(fp, CLI_HASH_SHA256)) {

 	    sha256_init(&sha256ctx);

 	    compute_hash[CLI_HASH_SHA256] = 1;

 	} else

@@ -871,25 +888,55 @@ int cli_fmap_scandesc(cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli

 	virname = NULL;

 	for(hashtype = CLI_HASH_MD5; hashtype < CLI_HASH_AVAIL_TYPES; hashtype++) {

-	    if(compute_hash[hashtype] &&

-	       (ret = cli_hm_scan(digest[hashtype], map->len, &virname, hdb, hashtype)) == CL_VIRUS) {

-		if(fp) {

-		    for(hashtype2 = CLI_HASH_MD5; hashtype2 < CLI_HASH_AVAIL_TYPES; hashtype2++) {

-			if(compute_hash[hashtype2] &&

-			   cli_hm_scan(digest[hashtype2], map->len, NULL, fp, hashtype2) == CL_VIRUS) {

-			    ret = CL_CLEAN;

-			    break;

-			}

+	    const char * virname_w = NULL;

+	    int found = 0;

+

+	    /* If no hash, skip to next type */

+	    if(!compute_hash[hashtype])

+		continue;

+

+	    /* Do hash scan */

+	    if((ret = cli_hm_scan(digest[hashtype], map->len, &virname, hdb, hashtype)) == CL_VIRUS) {

+		found += 1;

+	    }

+	    if(!found || SCAN_ALL) {

+		if ((ret = cli_hm_scan_wild(digest[hashtype], &virname_w, hdb, hashtype)) == CL_VIRUS)

+		    found += 2;

+	    }

+

+	    /* If found, do immediate hash-only FP check */

+	    if (found && fp) {

+		for(hashtype2 = CLI_HASH_MD5; hashtype2 < CLI_HASH_AVAIL_TYPES; hashtype2++) {

+		    if(!compute_hash[hashtype2])

+			continue;

+		    if(cli_hm_scan(digest[hashtype2], map->len, NULL, fp, hashtype2) == CL_VIRUS) {

+			found = 0;

+			ret = CL_CLEAN;

+			break;

 		    }

-		}

-		if (ret == CL_VIRUS) {

-		    viruses_found++;

-		    cli_append_virus(ctx, virname);

-		    if (!SCAN_ALL)

+		    else if(cli_hm_scan_wild(digest[hashtype2], NULL, fp, hashtype2) == CL_VIRUS) {

+			found = 0;

+			ret = CL_CLEAN;

 			break;

+		    }

 		}

+	    }

+

+	    /* If matched size-based hash ... */

+	    if (found % 2) {

+		viruses_found++;

+		cli_append_virus(ctx, virname);

+		if (!SCAN_ALL)

+		    break;

 		virname = NULL;

 	    }

+	    /* If matched size-agnostic hash ... */

+	    if (found > 1) {

+		viruses_found++;

+		cli_append_virus(ctx, virname_w);

+		if (!SCAN_ALL)

+		    break;

+	    }

 	}

     }

@@ -93,6 +93,7 @@ struct cli_matcher {

     /* HASH */

     struct cli_hash_patt hm;

+    struct cli_hash_wild hwild;

     /* Extended Aho-Corasick */

     uint32_t ac_partsigs, ac_nodes, ac_patterns, ac_lsigs;

@@ -390,27 +390,6 @@ void findres(uint32_t by_type, uint32_t by_name, uint32_t res_rva, fmap_t *map,

     }

 }

-static unsigned int cli_md5sect(fmap_t *map, struct cli_exe_section *s, unsigned char *digest) {

-    const void *hashme;

-    cli_md5_ctx md5;

-

-    if (s->rsz > CLI_MAX_ALLOCATION) {

-	cli_dbgmsg("cli_md5sect: skipping md5 calculation for too big section\n");

-	return 0;

-    }

-

-    if(!s->rsz) return 0;

-    if(!(hashme=fmap_need_off_once(map, s->raw, s->rsz))) {

-	cli_dbgmsg("cli_md5sect: unable to read section data\n");

-	return 0;

-    }

-

-    cli_md5_init(&md5);

-    cli_md5_update(&md5, hashme, s->rsz);

-    cli_md5_final(digest, &md5);

-    return 1;

-}

-

 static void cli_parseres_special(uint32_t base, uint32_t rva, fmap_t *map, struct cli_exe_section *exe_sections, uint16_t nsections, size_t fsize, uint32_t hdr_size, unsigned int level, uint32_t type, unsigned int *maxres, struct swizz_stats *stats) {

     unsigned int err = 0, i;

     const uint8_t *resdir;

@@ -502,6 +481,100 @@ static void cli_parseres_special(uint32_t base, uint32_t rva, fmap_t *map, struc

     fmap_unneed_ptr(map, oentry, entries*8);

 }

+static unsigned int cli_hashsect(fmap_t *map, struct cli_exe_section *s, unsigned char **digest, int * foundhash, int * foundwild)

+{

+    const void *hashme;

+    cli_md5_ctx md5;

+    SHA1Context sha1ctx;

+    SHA256_CTX sha256ctx;

+

+    if (s->rsz > CLI_MAX_ALLOCATION) {

+        cli_dbgmsg("cli_hashsect: skipping hash calculation for too big section\n");

+        return 0;

+    }

+

+    if(!s->rsz) return 0;

+    if(!(hashme=fmap_need_off_once(map, s->raw, s->rsz))) {

+        cli_dbgmsg("cli_hashsect: unable to read section data\n");

+        return 0;

+    }

+

+    if(foundhash[CLI_HASH_MD5] || foundwild[CLI_HASH_MD5]) {

+        cli_md5_init(&md5);

+        cli_md5_update(&md5, hashme, s->rsz);

+        cli_md5_final(digest[CLI_HASH_MD5], &md5);

+    }

+    if(foundhash[CLI_HASH_SHA1] || foundwild[CLI_HASH_SHA1]) {

+        SHA1Init(&sha1ctx);

+        SHA1Update(&sha1ctx, hashme, s->rsz);

+        SHA1Final(&sha1ctx, digest[CLI_HASH_SHA1]);

+    }

+    if(foundhash[CLI_HASH_SHA256] || foundwild[CLI_HASH_SHA256]) {

+        sha256_init(&sha256ctx);

+        sha256_update(&sha256ctx, hashme, s->rsz);

+        sha256_final(&sha256ctx, digest[CLI_HASH_SHA256]);

+    }

+

+    return 1;

+}

+

+/* check hash section sigs */

+static int scan_pe_mdb (cli_ctx * ctx, struct cli_exe_section *exe_section)

+{

+    struct cli_matcher * mdb_sect = ctx->engine->hm_mdb;

+    unsigned char * hashset[CLI_HASH_AVAIL_TYPES];

+    const char * virname = NULL;

+    int foundsize[CLI_HASH_AVAIL_TYPES];

+    int foundwild[CLI_HASH_AVAIL_TYPES];

+    enum CLI_HASH_TYPE type;

+    int ret = CL_CLEAN;

+ 

+    /* pick hashtypes to generate */

+    for(type = CLI_HASH_MD5; type < CLI_HASH_AVAIL_TYPES; type++) {

+        foundsize[type] = cli_hm_have_size(mdb_sect, type, exe_section->rsz);

+        foundwild[type] = cli_hm_have_wild(mdb_sect, type);

+        if(foundsize[type] || foundwild[type]) {

+            hashset[type] = cli_malloc(cli_hashlength(type));

+            if(!hashset[type]) {

+                cli_errmsg("scan_pe: cli_malloc failed!\n");

+                for(; type > 0;)

+                    free(hashset[--type]);

+                return CL_EMEM;

+            }

+        }

+        else {

+            hashset[type] = NULL;

+        }

+    }

+

+    /* Generate hashes */

+    cli_hashsect(*ctx->fmap, exe_section, hashset, foundsize, foundwild);

+

+    /* Do scans */

+    for(type = CLI_HASH_MD5; type < CLI_HASH_AVAIL_TYPES; type++) {

+       if(foundsize[type] && cli_hm_scan(hashset[type], exe_section->rsz, &virname, mdb_sect, type) == CL_VIRUS) {

+            cli_append_virus(ctx, virname);

+            if (!SCAN_ALL) {

+                for(type = CLI_HASH_AVAIL_TYPES; type > 0;)

+                    free(hashset[--type]);

+                return CL_VIRUS;

+            }

+            ret = CL_VIRUS;

+       }

+       if(foundwild[type] && cli_hm_scan_wild(hashset[type], &virname, mdb_sect, type) == CL_VIRUS) {

+            cli_append_virus(ctx, virname);

+            if (!SCAN_ALL) {

+                for(type = CLI_HASH_AVAIL_TYPES; type > 0;)

+                    free(hashset[--type]);

+                return CL_VIRUS;

+            }

+            ret = CL_VIRUS;

+       }

+    }

+

+    return ret;

+}

+

 int cli_scanpe(cli_ctx *ctx)

 {

 	uint16_t e_magic; /* DOS signature ("MZ") */

@@ -528,7 +601,7 @@ int cli_scanpe(cli_ctx *ctx)

 	size_t fsize;

 	uint32_t valign, falign, hdr_size, j;

 	struct cli_exe_section *exe_sections;

-	struct cli_matcher *md5_sect;

+	struct cli_matcher *mdb_sect;

 	char timestr[32];

 	struct pe_image_data_dir *dirs;

 	struct cli_bc_ctx *bc_ctx;

@@ -977,25 +1050,16 @@ int cli_scanpe(cli_ctx *ctx)

 	    if(SCAN_ALGO && (DCONF & PE_CONF_POLIPOS) && !*sname && exe_sections[i].vsz > 40000 && exe_sections[i].vsz < 70000 && exe_sections[i].chr == 0xe0000060) polipos = i;

-	    /* check MD5 section sigs */

-	    md5_sect = ctx->engine->hm_mdb;

-	    if((DCONF & PE_CONF_MD5SECT) && md5_sect) {

-		unsigned char md5_dig[16];

-		if(cli_hm_have_size(md5_sect, CLI_HASH_MD5, exe_sections[i].rsz) && 

-		   cli_md5sect(map, &exe_sections[i], md5_dig) &&

-		   cli_hm_scan(md5_dig, exe_sections[i].rsz, &virname, md5_sect, CLI_HASH_MD5) == CL_VIRUS) {

-		    cli_append_virus(ctx, virname);

-		    if(cli_hm_scan(md5_dig, fsize, NULL, ctx->engine->hm_fp, CLI_HASH_MD5) != CL_VIRUS) {

-			if (!SCAN_ALL) {

-			    free(section_hdr);

-			    free(exe_sections);

-			    return CL_VIRUS;

-			}

-		    }

-		    viruses_found++;

-		}

+	    /* check hash section sigs */

+	    if((DCONF & PE_CONF_MD5SECT) && ctx->engine->hm_mdb) {

+	        ret = scan_pe_mdb(ctx, &exe_sections[i]);

+	        if (ret != CL_CLEAN) {

+	            cli_errmsg("scan_pe: scan_pe_mdb failed: %d!\n", ret);

+	            free(section_hdr);

+	            free(exe_sections);

+	            return ret;

+	        }

 	    }

-	    

 	}

 	if (exe_sections[i].urva>>31 || exe_sections[i].uvsz>>31 || (exe_sections[i].rsz && exe_sections[i].uraw>>31) || exe_sections[i].ursz>>31) {

@@ -1871,13 +1935,17 @@ int cli_scanpe(cli_ctx *ctx)

 	    if(epbuff[1] != '\xbe' || skew <= 0 || skew > 0xfff) { /* FIXME: legit skews?? */

 		skew = 0; 

-		if(upxfn(src, ssize, dest, &dsize, exe_sections[i].rva, exe_sections[i + 1].rva, vep) >= 0)

-		    upx_success = 1;

-

 	    } else {

 		cli_dbgmsg("UPX: UPX1 seems skewed by %d bytes\n", skew);

-		if(upxfn(src + skew, ssize - skew, dest, &dsize, exe_sections[i].rva, exe_sections[i + 1].rva, vep-skew) >= 0 || upxfn(src, ssize, dest, &dsize, exe_sections[i].rva, exe_sections[i + 1].rva, vep) >= 0)

-		    upx_success = 1;

+	    }

+

+	    /* Try skewed first (skew may be zero) */

+	    if(upxfn(src + skew, ssize - skew, dest, &dsize, exe_sections[i].rva, exe_sections[i + 1].rva, vep-skew) >= 0) {

+		upx_success = 1;

+	    }

+	    /* If skew not successful and non-zero, try no skew */

+	    else if(skew && (upxfn(src, ssize, dest, &dsize, exe_sections[i].rva, exe_sections[i + 1].rva, vep) >= 0)) {

+		upx_success = 1;

 	    }

 	    if(upx_success)

@@ -2042,7 +2110,7 @@ int cli_scanpe(cli_ctx *ctx)

 	CLI_UNPSIZELIMITS("PEspin", fsize);

 	if((spinned = (char *) cli_malloc(fsize)) == NULL) {

-        cli_errmsg("PESping: Unable to allocate memory for spinned %u\n", fsize);

+        cli_errmsg("PESping: Unable to allocate memory for spinned %lu\n", (unsigned long)fsize);

 	    free(exe_sections);

 	    return CL_EMEM;

 	}

@@ -2106,7 +2174,7 @@ int cli_scanpe(cli_ctx *ctx)

 	    char *spinned;

 	    if((spinned = (char *) cli_malloc(fsize)) == NULL) {

-            cli_errmsg("yc: Unable to allocate memory for spinned %u\n", fsize);

+            cli_errmsg("yC: Unable to allocate memory for spinned %lu\n", (unsigned long)fsize);

 	      free(exe_sections);

 	      return CL_EMEM;

 	    }

@@ -1914,13 +1914,14 @@ static int cli_loadign(FILE *fs, struct cl_engine *engine, unsigned int options,

 #define MD5_TOKENS 5

 static int cli_loadhash(FILE *fs, struct cl_engine *engine, unsigned int *signo, unsigned int mode, unsigned int options, struct cli_dbio *dbio, const char *dbname)

 {

-	const char *tokens[MD5_TOKENS + 1];

-	char buffer[FILEBUFF], *buffer_cpy = NULL;

-	const char *pt, *virname;

-	int ret = CL_SUCCESS;

-	unsigned int size_field = 1, md5_field = 0, line = 0, sigs = 0, tokens_count;

-	struct cli_matcher *db;

-	unsigned long size;

+    const char *tokens[MD5_TOKENS + 1];

+    char buffer[FILEBUFF], *buffer_cpy = NULL;

+    const char *pt, *virname;

+    int ret = CL_SUCCESS;

+    unsigned int size_field = 1, md5_field = 0, line = 0, sigs = 0, tokens_count;

+    unsigned int req_fl = 0; 

+    struct cli_matcher *db;

+    unsigned long size;

     if(mode == MD5_MDB) {

@@ -1966,7 +1967,7 @@ static int cli_loadhash(FILE *fs, struct cl_engine *engine, unsigned int *signo,

 	    break;

 	}

 	if(tokens_count > MD5_TOKENS - 2) {

-	    unsigned int req_fl = atoi(tokens[MD5_TOKENS - 2]);

+	    req_fl = atoi(tokens[MD5_TOKENS - 2]);

 	    if(tokens_count > MD5_TOKENS) {

 		ret = CL_EMALFDB;

@@ -1976,17 +1977,28 @@ static int cli_loadhash(FILE *fs, struct cl_engine *engine, unsigned int *signo,

 	    if(cl_retflevel() < req_fl)

 		continue;

 	    if(tokens_count == MD5_TOKENS) {

-		req_fl = atoi(tokens[MD5_TOKENS - 1]);

-		if(cl_retflevel() > req_fl)

+		int max_fl = atoi(tokens[MD5_TOKENS - 1]);

+		if(cl_retflevel() > max_fl)

 		    continue;

 	    }

 	}

-	size = strtoul(tokens[size_field], (char **)&pt, 10);

-	if(*pt || !size || size >= 0xffffffff) {

-	    cli_errmsg("cli_loadhash: Invalid value for the size field\n");

-	    ret = CL_EMALFDB;

-	    break;

+	if((mode == MD5_MDB) || strcmp(tokens[size_field],"*")) {

+	    size = strtoul(tokens[size_field], (char **)&pt, 10);

+	    if(*pt || !size || size >= 0xffffffff) {

+		cli_errmsg("cli_loadhash: Invalid value for the size field\n");

+		ret = CL_EMALFDB;

+		break;

+	    }

+	}

+	else {

+	    size = 0;

+	    if((tokens_count < MD5_TOKENS - 1) || (req_fl < 73)) {

+		cli_errmsg("cli_loadhash: Minimum FLEVEL field must be at least 73 for wildcard size hash signatures."

+			" For reference, running FLEVEL is %d\n", cl_retflevel());

+		ret = CL_EMALFDB;

+		break;

+	    }

 	}

 	pt = tokens[2]; /* virname */

@@ -1348,7 +1348,7 @@ static int listdb(const char *filename, const regex_t *regex)

             line++;

             mprintf("%s\n", buffer);

         }

-    } else if(cli_strbcasestr(filename, ".hdb") || cli_strbcasestr(filename, ".hdu") || cli_strbcasestr(filename, ".mdb") || cli_strbcasestr(filename, ".mdu")) { /* hash database */

+    } else if(cli_strbcasestr(filename, ".hdb") || cli_strbcasestr(filename, ".hdu") || cli_strbcasestr(filename, ".mdb") || cli_strbcasestr(filename, ".mdu") || cli_strbcasestr(filename, ".hsb") || cli_strbcasestr(filename, ".hsu") || cli_strbcasestr(filename, ".msb") || cli_strbcasestr(filename, ".msu")) { /* hash database */

 	while(fgets(buffer, FILEBUFF, fh)) {

 	    cli_chomp(buffer);