@@ -306,6 +306,8 @@ int main(int argc, char **argv)

 	break;

     }

+    cache_init(256);

+

     if(!(engine = cl_engine_new())) {

 	logg("!Can't initialize antivirus engine\n");

 	ret = 1;

@@ -144,6 +144,8 @@ int main(int argc, char **argv)

     gettimeofday(&t1, NULL);

+    cache_init(256);

+

     ret = scanmanager(opts);

     if(!optget(opts, "no-summary")->enabled) {

@@ -346,6 +346,8 @@ libclamav_la_SOURCES = \

 	bytecode_api.h \

 	bytecode_api_impl.h \

 	bytecode_hooks.h

+	cache.c \

+	cache.h

 if !LINK_TOMMATH

 libclamav_la_SOURCES += bignum.c \

@@ -2231,6 +2231,8 @@ uninstall-am: uninstall-includeHEADERS uninstall-libLTLIBRARIES

 	tags tags-recursive uninstall uninstall-am \

 	uninstall-includeHEADERS uninstall-libLTLIBRARIES

+	cache.c \

+	cache.h

 .PHONY: version.h.tmp

 version.c: version.h

new file mode 100644

@@ -0,0 +1,187 @@

+#include <string.h>

+#include <stdlib.h>

+#include <pthread.h>

+

+#include "md5.h"

+#include "mpool.h"

+#include "clamav.h"

+#include "cache.h"

+#include "fmap.h"

+

+#if HAVE_CONFIG_H

+#include "clamav-config.h"

+#endif

+

+#define CACHE_PERTURB 10

+/* 1/10th */

+

+static mpool_t *mempool = NULL;

+static struct CACHE {

+    struct CACHE_ENTRY {

+	unsigned char hash[15];

+	uint32_t dbver;

+	uint32_t hits;

+    } *items;

+    pthread_mutex_t mutex;

+    uint32_t lastdb;

+} *cache = NULL;

+static unsigned int cache_entries = 0;

+

+int cl_cache_init(unsigned int entries) {

+    unsigned int i;

+

+    if(!(mempool = mpool_create())) {

+	cli_errmsg("mpool init fail\n");

+	return 1;

+    }

+    if(!(cache = mpool_malloc(mempool, sizeof(struct CACHE) * 256))) {

+	cli_errmsg("mpool malloc fail\n");

+	mpool_destroy(mempool);

+	return 1;

+    }

+

+    for(i=0; i<256; i++) {

+	struct CACHE_ENTRY *e = mpool_calloc(mempool, sizeof(struct CACHE_ENTRY), entries);

+	if(!e) {

+	    cli_errmsg("mpool calloc fail\n");

+	    mpool_destroy(mempool);

+	    return 1;

+	}

+	cache[i].items = e;

+	cache[i].lastdb = 0;

+	if(pthread_mutex_init(&cache[i].mutex, NULL)) {

+	    cli_errmsg("mutex init fail\n");

+	    mpool_destroy(mempool);

+	    return 1;

+	}

+    }

+    cache_entries = entries;

+    return 0;

+}

+

+void cache_swap(struct CACHE_ENTRY *e, unsigned int a) {

+    struct CACHE_ENTRY t;

+    unsigned int b = a-1;

+

+    if(!a || e[a].hits <= e[b].hits)

+	return;

+

+    do {

+	if(e[a].hits > e[b].hits)

+	    continue;

+	break;

+    } while(b--);

+    b++;

+

+    memcpy(&t, &e[a], sizeof(t));

+    memcpy(&e[a], &e[b], sizeof(t));

+    memcpy(&e[b], &t, sizeof(t));

+}

+

+static void updb(uint32_t db, unsigned int skip) {

+    unsigned int i;

+    for(i=0; i<256; i++) {

+	if(i==skip) continue;

+	if(pthread_mutex_lock(&cache[i].mutex)) {

+	    cli_errmsg("mutex lock fail\n");

+	    continue;

+	}

+	cache[i].lastdb = db;

+	pthread_mutex_unlock(&cache[i].mutex);	

+    }

+}

+

+static int cache_lookup_hash(unsigned char *md5, cli_ctx *ctx) {

+    unsigned int i;

+    int ret = CL_VIRUS;

+    struct CACHE_ENTRY *e;

+    struct CACHE *c;

+

+    if(!cache) return ret;

+

+    c = &cache[*md5];

+    e = c->items;

+    if(pthread_mutex_lock(&c->mutex)) {

+	cli_errmsg("mutex lock fail\n");

+	return ret;

+    }

+    if(c->lastdb <= ctx->engine->dbversion[0]) {

+	if(c->lastdb < ctx->engine->dbversion[0]) {

+	    c->lastdb = ctx->engine->dbversion[0];

+	    updb(c->lastdb, *md5);

+	} else {

+	    for(i=0; i<cache_entries; i++) {

+		if(!e[i].hits) break;

+		if(e[i].dbver == c->lastdb && !memcmp(e[i].hash, md5 + 1, 15)) {

+		    e[i].hits++;

+		    cache_swap(e, i);

+		    ret = CL_CLEAN;

+		    cli_warnmsg("cached\n");

+		    break;

+		}

+	    }

+	}

+    }

+    pthread_mutex_unlock(&c->mutex);

+    return ret;

+}

+

+void cache_add(unsigned char *md5, cli_ctx *ctx) {

+    unsigned int i, replace;

+    struct CACHE_ENTRY *e;

+    struct CACHE *c;

+

+    if(!cache) return;

+

+    c = &cache[*md5];

+    e = c->items;

+    if(pthread_mutex_lock(&c->mutex)) {

+	cli_errmsg("mutex lock fail\n");

+	return;

+    }

+    if(c->lastdb == ctx->engine->dbversion[0]) {

+	replace = cache_entries;

+	for(i=0; i<cache_entries; i++) {

+	    if(!e[i].hits) break;

+	    if(replace == cache_entries && e[i].dbver < c->lastdb) {

+		replace = i;

+	    } else if(e[i].hits && !memcmp(e[i].hash, md5 + 1, 15)) {

+		e[i].hits++;

+		cache_swap(e, i);

+		pthread_mutex_unlock(&c->mutex);

+		return;

+	    }

+	}

+	if(replace == cache_entries)

+	    replace = cache_entries - 1 - (rand() % (cache_entries / CACHE_PERTURB));

+	e[replace].hits = 1;

+	e[replace].dbver = c->lastdb;

+	memcpy(e[replace].hash, md5 + 1, 15);

+	cache_swap(e, replace);

+    }

+    pthread_mutex_unlock(&c->mutex);

+    return;

+}

+

+int cache_check(unsigned char *hash, cli_ctx *ctx) {

+    fmap_t *map = *ctx->fmap;

+    size_t todo = map->len, at = 0;;

+    cli_md5_ctx md5;

+

+    if(!cache) return CL_VIRUS;

+

+    cli_md5_init(&md5);

+    while(todo) {

+	void *buf;

+	size_t readme = todo < FILEBUFF ? todo : FILEBUFF;

+	if(!(buf = fmap_need_off_once(map, at, readme)) != readme) {

+	    lseek(desc, seekback, SEEK_SET);

+	    return CL_VIRUS;

+	}

+	todo -= readme;

+	at += readme;

+	cli_md5_update(&md5, buf, readme);

+    }

+    cli_md5_final(hash, &md5);

+    return cache_lookup_hash(hash, ctx);

+}

new file mode 100644

@@ -0,0 +1,30 @@

+/*

+ *  Copyright (C) 2010 Sourcefire, Inc.

+ *

+ *  Authors: aCaB <acab@clamav.net>

+ *

+ *  This program is free software; you can redistribute it and/or modify

+ *  it under the terms of the GNU General Public License version 2 as

+ *  published by the Free Software Foundation.

+ *

+ *  This program is distributed in the hope that it will be useful,

+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of

+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ *  GNU General Public License for more details.

+ *

+ *  You should have received a copy of the GNU General Public License

+ *  along with this program; if not, write to the Free Software

+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

+ *  MA 02110-1301, USA.

+ */

+

+#ifndef __CACHE_H

+#define __CACHE_H

+

+#include "clamav.h"

+#include "others.h"

+

+void cache_add(unsigned char *md5, cli_ctx *ctx);

+int cache_check(unsigned char *hash, cli_ctx *ctx);

+

+#endif

@@ -208,6 +208,7 @@ extern const char *cl_retver(void);

 /* others */

 extern const char *cl_strerror(int clerror);

+extern int cl_cache_init(unsigned int entries);

 #ifdef __cplusplus

 }

@@ -27,6 +27,7 @@ CLAMAV_PUBLIC {

     cl_statfree;

     cl_statinidir;

     cl_strerror;

+    cl_cache_init;

   local:

     *;

 };

@@ -344,7 +344,7 @@ int cli_scandesc(int desc, cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struc

     fmap_t *map = *ctx->fmap;

     if((*ctx->fmap = fmap(desc, 0, 0))) {

-	ret = cli_fmap_scandesc(ctx, ftype, ftonly, ftoffset, acmode);

+	ret = cli_fmap_scandesc(ctx, ftype, ftonly, ftoffset, acmode, NULL);

 	funmap(*ctx->fmap);

     }

     *ctx->fmap = map;

@@ -352,7 +352,7 @@ int cli_scandesc(int desc, cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struc

 }

-int cli_fmap_scandesc(cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli_matched_type **ftoffset, unsigned int acmode)

+int cli_fmap_scandesc(cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli_matched_type **ftoffset, unsigned int acmode, unsigned char *refhash)

 {

  	unsigned char *buff;

 	int ret = CL_CLEAN, type = CL_CLEAN, bytes;

@@ -420,7 +420,7 @@ int cli_fmap_scandesc(cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli

 	}

     }

-    if(!ftonly && ctx->engine->md5_hdb)

+    if(!refhash && !ftonly && ctx->engine->md5_hdb)

 	cli_md5_init(&md5ctx);

     while(offset < map->len) {

@@ -459,7 +459,7 @@ int cli_fmap_scandesc(cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli

 		    type = ret;

 	    }

-	    if(ctx->engine->md5_hdb)

+	    if(!refhash && ctx->engine->md5_hdb)

 		cli_md5_update(&md5ctx, buff + maxpatlen * (offset!=0), bytes - maxpatlen * (offset!=0));

 	}

@@ -534,8 +534,11 @@ int cli_fmap_scandesc(cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli

     if(!ftonly && ctx->engine->md5_hdb) {

 	    const struct cli_bm_patt *patt;

-	cli_md5_final(digest, &md5ctx);

-	if(cli_bm_scanbuff(digest, 16, ctx->virname, &patt, ctx->engine->md5_hdb, 0, NULL, NULL) == CL_VIRUS && patt->filesize == map->len && (cli_bm_scanbuff(digest, 16, NULL, &patt, ctx->engine->md5_fp, 0, NULL, NULL) != CL_VIRUS || patt->filesize != map->len))

+	if(!refhash) {

+	    cli_md5_final(digest, &md5ctx);

+	    refhash = digest;

+	}

+	if(cli_bm_scanbuff(refhash, 16, ctx->virname, &patt, ctx->engine->md5_hdb, 0, NULL, NULL) == CL_VIRUS && patt->filesize == map->len && (cli_bm_scanbuff(refhash, 16, NULL, &patt, ctx->engine->md5_fp, 0, NULL, NULL) != CL_VIRUS || patt->filesize != map->len))

 	    return CL_VIRUS;

     }

@@ -146,8 +146,7 @@ struct cli_target_info {

 int cli_scanbuff(const unsigned char *buffer, uint32_t length, uint32_t offset, cli_ctx *ctx, cli_file_t ftype, struct cli_ac_data **acdata);

 int cli_scandesc(int desc, cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli_matched_type **ftoffset, unsigned int acmode);

-int cli_fmap_scandesc(cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli_matched_type **ftoffset, unsigned int acmode);

-

+int cli_fmap_scandesc(cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli_matched_type **ftoffset, unsigned int acmode, unsigned char *digest);

 int cli_caloff(const char *offstr, struct cli_target_info *info, fmap_t *map, unsigned int target, uint32_t *offdata, uint32_t *offset_min, uint32_t *offset_max);

 int cli_checkfp(int fd, cli_ctx *ctx);

@@ -85,6 +85,7 @@

 #include "ishield.h"

 #include "7z.h"

 #include "fmap.h"

+#include "cache.h"

 #ifdef HAVE_BZLIB_H

 #include <bzlib.h>

@@ -1691,7 +1692,7 @@ static int cli_scanraw(cli_ctx *ctx, cli_file_t type, uint8_t typercg, cli_file_

     if(typercg)

 	acmode |= AC_SCAN_FT;

-    ret = cli_fmap_scandesc(ctx, type == CL_TYPE_TEXT_ASCII ? 0 : type, 0, &ftoffset, acmode);

+    ret = cli_fmap_scandesc(ctx, type == CL_TYPE_TEXT_ASCII ? 0 : type, 0, &ftoffset, acmode, NULL);

     if(ret >= CL_TYPENO) {

 	ctx->recursion++;

@@ -1840,6 +1841,7 @@ int cli_magic_scandesc(int desc, cli_ctx *ctx)

 	struct stat sb;

 	uint8_t typercg = 1;

 	cli_file_t current_container = ctx->container_type; /* TODO: container tracking code TBD - bb#1293 */

+	unsigned char hash[16];

     if(ctx->engine->maxreclevel && ctx->recursion > ctx->engine->maxreclevel) {

         cli_dbgmsg("cli_magic_scandesc: Archive recursion limit exceeded (%u, max: %u)\n", ctx->recursion, ctx->engine->maxreclevel);

@@ -1875,15 +1877,23 @@ int cli_magic_scandesc(int desc, cli_ctx *ctx)

 	return CL_EMEM;

     }

+    if(cache_check(hash, ctx) == CL_CLEAN)

+	return CL_CLEAN;

+    

     if(!ctx->options || (ctx->recursion == ctx->engine->maxreclevel)) { /* raw mode (stdin, etc.) or last level of recursion */

 	if(ctx->recursion == ctx->engine->maxreclevel)

 	    cli_dbgmsg("cli_magic_scandesc: Hit recursion limit, only scanning raw file\n");

 	else

 	    cli_dbgmsg("Raw mode: No support for special files\n");

-	if((ret = cli_fmap_scandesc(ctx, 0, 0, NULL, AC_SCAN_VIR)) == CL_VIRUS)

+

+	if((ret = cli_fmap_scandesc(ctx, 0, 0, NULL, AC_SCAN_VIR, hash)) == CL_VIRUS)

 	    cli_dbgmsg("%s found in descriptor %d\n", *ctx->virname, desc);

+	else

+	    cache_add(hash, ctx);

+

 	funmap(*ctx->fmap);

 	ctx->fmap--; 

+

 	return ret;

     }

@@ -2167,8 +2177,10 @@ int cli_magic_scandesc(int desc, cli_ctx *ctx)

 	case CL_EMAXSIZE:

 	case CL_EMAXFILES:

 	    cli_dbgmsg("Descriptor[%d]: %s\n", desc, cl_strerror(ret));

+	    cache_add(hash, ctx);

 	    return CL_CLEAN;

 	default:

+	    if(ret == CL_CLEAN) cache_add(hash, ctx);

 	    return ret;

     }

 }