@@ -34,313 +34,183 @@

 #include "fmap.h"

-/* struct cache_key { */

-/*     char digest[16]; */

-/*     uint32_t size; /\* 0 is used to mark an empty hash slot! *\/ */

-/*     struct cache_key *lru_next, *lru_prev; */

-/* }; */

-

-/* struct cache_set { */

-/*     struct cache_key *data; */

-/*     size_t capacity; */

-/*     size_t maxelements; /\* considering load factor *\/ */

-/*     size_t elements; */

-/*     size_t version; */

-/*     struct cache_key *lru_head, *lru_tail; */

-/*     pthread_mutex_t mutex; */

-/* }; */

-

-/* #define CACHE_INVALID_VERSION ~0u */

-/* #define CACHE_KEY_DELETED ~0u */

-/* #define CACHE_KEY_EMPTY 0 */

-

-/* /\* size must be power of 2! *\/ */

-/* static int cacheset_init(struct cache_set* map, size_t maxsize, uint8_t loadfactor) */

-/* { */

-/*     map->data = cli_calloc(maxsize, sizeof(*map->data)); */

-/*     if (!map->data) */

-/* 	return CL_EMEM; */

-/*     map->capacity = maxsize; */

-/*     map->maxelements = loadfactor*maxsize / 100; */

-/*     map->elements = 0; */

-/*     map->version = CACHE_INVALID_VERSION; */

-/*     map->lru_head = map->lru_tail = NULL; */

-/*     if (pthread_mutex_init(&map->mutex, NULL)) { */

-/* 	cli_errmsg("mutex init fail\n"); */

-/* 	return CL_EMEM; */

-/*     } */

-/* } */

-

-/* static void cacheset_destroy(struct cache_set *map) */

-/* { */

-/*     pthread_mutex_destroy(&map->mutex); */

-/*     free(map->data); */

-/* } */

-

-/* static void cacheset_acquire(struct cache_set *map) */

-/* { */

-/*     pthread_mutex_lock(&map->mutex); */

-/* } */

-

-/* static void cache_setversion(struct cache_set* map, uint32_t version) */

-/* { */

-/*     unsigned i; */

-/*     if (map->version == version) */

-/* 	return; */

-/*     map->version = version; */

-/*     map->elements = 0; /\* all elements have expired now *\/ */

-/*     for (i=0;i<map->capacity;i++) */

-/* 	map->data[i].size = 0; */

-/*     map->lru_head = map->lru_tail = NULL; */

-/* } */

-

-/* static void cacheset_lru_remove(struct cache_set *map, size_t howmany) */

-/* { */

-/*     while (howmany--) { */

-/* 	struct cache_key *old; */

-/* 	assert(map->lru_head); */

-/* 	assert(!old->lru_prev); */

-/* 	// Remove a key from the head of the list */

-/* 	old = map->lru_head; */

-/* 	map->lru_head = old->lru_next; */

-/* 	old->size = CACHE_KEY_DELETED; */

-/* 	/\* This slot is now deleted, it is not empty, */

-/* 	 * because previously we could have inserted a key that has seen this */

-/* 	 * slot as occupied, to find that key we need to ensure that all keys */

-/* 	 * that were occupied when the key was inserted, are seen as occupied */

-/* 	 * when searching too. */

-/* 	 * Of course when inserting a new value, we treat deleted slots as */

-/* 	 * empty. */

-/* 	 * We only replace old values with new values, but there is no guarantee */

-/* 	 * that the newly inserted value would hash to same place as the value */

-/* 	 * we remove due to LRU! *\/ */

-/* 	if (old == map->lru_tail) */

-/* 	    map->lru_tail = 0; */

-/*     } */

-/* } */

-

-/* static inline uint32_t hash32shift(uint32_t key) */

-/* { */

-/*   key = ~key + (key << 15); */

-/*   key = key ^ (key >> 12); */

-/*   key = key + (key << 2); */

-/*   key = key ^ (key >> 4); */

-/*   key = (key + (key << 3)) + (key << 11); */

-/*   key = key ^ (key >> 16); */

-/*   return key; */

-/* } */

-

-/* static inline size_t hash(const unsigned char* k,const size_t len,const size_t SIZE) */

-/* { */

-/*     size_t Hash = 1; */

-/*     size_t i; */

-/*     for(i=0;i<len;i++) { */

-/* 	/\* a simple add is good, because we use the mixing function below *\/ */

-/* 	Hash +=  k[i]; */

-/* 	/\* mixing function *\/ */

-/* 	Hash = hash32shift(Hash); */

-/*     } */

-/*     /\* SIZE is power of 2 *\/ */

-/*     return Hash & (SIZE - 1); */

-/* } */

-

-/* int cacheset_lookup_internal(struct cache_set *map, const struct cache_key *key, */

-/* 			     uint32_t *insert_pos, int deletedok) */

-/* { */

-/*     uint32_t idx = hash((const unsigned char*)key, sizeof(*key), map->capacity); */

-/*     uint32_t tries = 0; */

-/*     struct cache_key *k = &map->data[idx]; */

-/*     while (k->size != CACHE_KEY_EMPTY) { */

-/* 	if (k->size == key->size && */

-/* 	    !memcmp(k->digest, key, 16)) { */

-/* 	    /\* found key *\/ */

-/* 	    *insert_pos = idx; */

-/* 	    return 1; */

-/* 	} */

-/*        if (deletedok && k->size == CACHE_KEY_DELETED) { */

-/*            /\* treat deleted slot as empty *\/ */

-/*            *insert_pos = idx; */

-/*            return 0; */

-/*        } */

-/* 	idx = (idx + tries++)&(map->capacity-1); */

-/* 	k = &map->data[idx]; */

-/*     } */

-/*     /\* found empty pos *\/ */

-/*     *insert_pos = idx; */

-/*     return 0; */

-/* } */

-

-/* static inline void lru_remove(struct cache_set *map, struct cache_key *newkey) */

-/* { */

-/*     if (newkey->lru_next) */

-/* 	newkey->lru_next->lru_prev = newkey->lru_prev; */

-/*     if (newkey->lru_prev) */

-/* 	newkey->lru_prev->lru_next = newkey->lru_next; */

-/*     if (newkey == map->lru_head) */

-/* 	map->lru_head = newkey->lru_next; */

-/* } */

-

-/* static inline void lru_addtail(struct cache_set *map, struct cache_key *newkey) */

-/* { */

-/*     if (!map->lru_head) */

-/* 	map->lru_head = newkey; */

-/*     if (map->lru_tail) */

-/* 	map->lru_tail->lru_next = newkey; */

-/*     newkey->lru_next = NULL; */

-/*     newkey->lru_prev = map->lru_tail; */

-/*     map->lru_tail = newkey; */

-/* } */

-

-/* static void cacheset_add(struct cache_set *map, const struct cache_key *key) */

-/* { */

-/*     int ret; */

-/*     uint32_t pos; */

-/*     struct cache_key *newkey; */

-/*     if (map->elements >= map->maxelements) */

-/* 	cacheset_lru_remove(map, 1); */

-/*     assert(map->elements < map->maxelements); */

-

-/*     ret = cacheset_lookup_internal(map, key, &pos, 1); */

-/*     newkey = &map->data[pos]; */

-/*     if (ret) { */

-/* 	/\* was already added, remove from LRU list *\/ */

-/* 	lru_remove(map, newkey); */

-/*     } */

-/*     /\* add new key to tail of LRU list *\/ */

-/*     lru_addtail(map, newkey); */

-

-/*     map->elements++; */

-

-/*     assert(pos < map->maxelements); */

-

-/*     memcpy(&map->data[pos], key, sizeof(*key)); */

-/* } */

-

-/* static int cacheset_lookup(struct cache_set *map, const struct cache_key *key) */

-/* { */

-/*     struct cache_key *newkey; */

-/*     int ret; */

-/*     uint32_t pos; */

-/*     ret = cacheset_lookup_internal(map, key, &pos, 0); */

-/*     if (!ret) */

-/* 	return CACHE_INVALID_VERSION; */

-/*     newkey = &map->data[pos]; */

-/*     /\* update LRU position: move to tail *\/ */

-/*     lru_remove(map, newkey); */

-/*     lru_addtail(map, newkey); */

-

-/*     return map->version; */

-/* } */

-

-/* static void cacheset_release(struct cache_set *map) */

-/* { */

-/*     pthread_mutex_unlock(&map->mutex); */

-/* } */

-

-/* #if 0 */

-/* int main(int argc, char **argv) */

-/* { */

-/*     struct cache_key key; */

-/*     struct cache_set map; */

-/*     cacheset_init(&map, 256, 80); */

-/*     cacheset_acquire(&map); */

-/*     cache_setversion(&map, 10); */

-

-/*     key.size = 1024; */

-/*     memcpy(key.digest, "1234567890123456", 16); */

-/*     cacheset_add(&map, &key); */

-/*     memcpy(key.digest, "1234567890123457", 16); */

-/*     cacheset_add(&map, &key); */

-/*     memcpy(key.digest, "0123456789012345", 16); */

-/*     cacheset_add(&map, &key); */

-

-/*     key.size = 1024; */

-/*     memcpy(key.digest, "1234567890123456", 16); */

-/*     if (cacheset_lookup(&map, &key) != 10) */

-/* 	abort(); */

-/*     memcpy(key.digest, "1234567890123456", 16); */

-/*     if (cacheset_lookup(&map, &key) != 10) */

-/* 	abort(); */

-/*     memcpy(key.digest, "1234567890123457", 16); */

-/*     if (cacheset_lookup(&map, &key) != 10) */

-/* 	abort(); */

-/*     memcpy(key.digest, "0123456789012345", 16); */

-/*     if (cacheset_lookup(&map, &key) != 10) */

-/* 	abort(); */

-/*     memcpy(key.digest, "0123456789012346", 16); */

-/*     if (cacheset_lookup(&map, &key) == 10) */

-/* 	abort(); */

-

-/*     cache_setversion(&map, 1); */

-/*     memcpy(key.digest, "1234567890123456", 16); */

-/*     if (cacheset_lookup(&map, &key) != CACHE_INVALID_VERSION) */

-/* 	abort(); */

-/*     memcpy(key.digest, "1234567890123456", 16); */

-/*     if (cacheset_lookup(&map, &key) != CACHE_INVALID_VERSION) */

-/* 	abort(); */

-/*     memcpy(key.digest, "1234567890123457", 16); */

-/*     if (cacheset_lookup(&map, &key) != CACHE_INVALID_VERSION) */

-/* 	abort(); */

-/*     memcpy(key.digest, "0123456789012345", 16); */

-/*     if (cacheset_lookup(&map, &key) != CACHE_INVALID_VERSION) */

-/* 	abort(); */

-

-/*     cacheset_release(&map); */

-

-/*     cacheset_destroy(&map); */

-

-/*     cacheset_init(&map, 8, 50); */

-/*     cacheset_acquire(&map); */

-/*     cache_setversion(&map, 10); */

-

-/*     key.size = 416; */

-/*     memcpy(key.digest, "1234567890123456", 16); */

-/*     cacheset_add(&map, &key); */

-/*     memcpy(key.digest, "1234567890123457", 16); */

-/*     cacheset_add(&map, &key); */

-/*     memcpy(key.digest, "1234567890123459", 16); */

-/*     cacheset_add(&map, &key); */

-/*     key.size = 400; */

-/*     memcpy(key.digest, "1234567890123450", 16); */

-/*     cacheset_add(&map, &key); */

-

-/*     key.size = 416; */

-/*     memcpy(key.digest, "1234567890123456", 16); */

-/*     if (cacheset_lookup(&map, &key) != 10) */

-/* 	abort(); */

-/*     if (cacheset_lookup(&map, &key) != 10) */

-/* 	abort(); */

-/*     if (cacheset_lookup(&map, &key) != 10) */

-/* 	abort(); */

-

-/*     key.size = 500; */

-/*     cacheset_add(&map, &key); */

-/*     memcpy(key.digest, "1234567890123457", 16); */

-/*     if (cacheset_lookup(&map, &key) == 10) */

-/* 	abort(); */

-

-/*     cacheset_release(&map); */

-/*     cacheset_destroy(&map); */

-

-/*     return 0; */

-/* } */

-/* #endif */

-

-#define CACHE_PERTURB 8

-/* 1/10th */

+struct cache_key {

+    char digest[16];

+    uint32_t size; /* 0 is used to mark an empty hash slot! */

+    struct cache_key *lru_next, *lru_prev;

+};

+

+struct cache_set {

+    struct cache_key *data;

+    size_t capacity;

+    size_t maxelements; /* considering load factor */

+    size_t elements;

+    size_t version;

+    struct cache_key *lru_head, *lru_tail;

+};

+

+#define CACHE_INVALID_VERSION ~0u

+#define CACHE_KEY_DELETED ~0u

+#define CACHE_KEY_EMPTY 0

+

+static void cache_setversion(struct cache_set* map, uint32_t version)

+{

+    unsigned i;

+    if (map->version == version)

+	return;

+    map->version = version;

+    map->elements = 0; /* all elements have expired now */

+    for (i=0;i<map->capacity;i++)

+	map->data[i].size = 0;

+    map->lru_head = map->lru_tail = NULL;

+}

+

+static void cacheset_lru_remove(struct cache_set *map, size_t howmany)

+{

+    while (howmany--) {

+	struct cache_key *old;

+	assert(map->lru_head);

+	assert(!old->lru_prev);

+	// Remove a key from the head of the list

+	old = map->lru_head;

+	map->lru_head = old->lru_next;

+	old->size = CACHE_KEY_DELETED;

+	/* This slot is now deleted, it is not empty,

+	 * because previously we could have inserted a key that has seen this

+	 * slot as occupied, to find that key we need to ensure that all keys

+	 * that were occupied when the key was inserted, are seen as occupied

+	 * when searching too.

+	 * Of course when inserting a new value, we treat deleted slots as

+	 * empty.

+	 * We only replace old values with new values, but there is no guarantee

+	 * that the newly inserted value would hash to same place as the value

+	 * we remove due to LRU! */

+	if (old == map->lru_tail)

+	    map->lru_tail = 0;

+    }

+}

+

+static inline uint32_t hash32shift(uint32_t key)

+{

+  key = ~key + (key << 15);

+  key = key ^ (key >> 12);

+  key = key + (key << 2);

+  key = key ^ (key >> 4);

+  key = (key + (key << 3)) + (key << 11);

+  key = key ^ (key >> 16);

+  return key;

+}

+

+static inline size_t hash(const unsigned char* k,const size_t len,const size_t SIZE)

+{

+    size_t Hash = 1;

+    size_t i;

+    for(i=0;i<len;i++) {

+	/* a simple add is good, because we use the mixing function below */

+	Hash +=  k[i];

+	/* mixing function */

+	Hash = hash32shift(Hash);

+    }

+    /* SIZE is power of 2 */

+    return Hash & (SIZE - 1);

+}

+

+int cacheset_lookup_internal(struct cache_set *map, const struct cache_key *key,

+			     uint32_t *insert_pos, int deletedok)

+{

+    uint32_t idx = hash((const unsigned char*)key, sizeof(*key), map->capacity);

+    uint32_t tries = 0;

+    struct cache_key *k = &map->data[idx];

+    while (k->size != CACHE_KEY_EMPTY) {

+	if (k->size == key->size &&

+	    !memcmp(k->digest, key, 16)) {

+	    /* found key */

+	    *insert_pos = idx;

+	    return 1;

+	}

+       if (deletedok && k->size == CACHE_KEY_DELETED) {

+           /* treat deleted slot as empty */

+           *insert_pos = idx;

+           return 0;

+       }

+	idx = (idx + tries++)&(map->capacity-1);

+	k = &map->data[idx];

+    }

+    /* found empty pos */

+    *insert_pos = idx;

+    return 0;

+}

+

+static inline void lru_remove(struct cache_set *map, struct cache_key *newkey)

+{

+    if (newkey->lru_next)

+	newkey->lru_next->lru_prev = newkey->lru_prev;

+    if (newkey->lru_prev)

+	newkey->lru_prev->lru_next = newkey->lru_next;

+    if (newkey == map->lru_head)

+	map->lru_head = newkey->lru_next;

+}

+

+static inline void lru_addtail(struct cache_set *map, struct cache_key *newkey)

+{

+    if (!map->lru_head)

+	map->lru_head = newkey;

+    if (map->lru_tail)

+	map->lru_tail->lru_next = newkey;

+    newkey->lru_next = NULL;

+    newkey->lru_prev = map->lru_tail;

+    map->lru_tail = newkey;

+}

+

+static void cacheset_add(struct cache_set *map, const struct cache_key *key)

+{

+    int ret;

+    uint32_t pos;

+    struct cache_key *newkey;

+    if (map->elements >= map->maxelements)

+	cacheset_lru_remove(map, 1);

+    assert(map->elements < map->maxelements);

+

+    ret = cacheset_lookup_internal(map, key, &pos, 1);

+    newkey = &map->data[pos];

+    if (ret) {

+	/* was already added, remove from LRU list */

+	lru_remove(map, newkey);

+    }

+    /* add new key to tail of LRU list */

+    memcpy(&map->data[pos], key, sizeof(*key));

+    lru_addtail(map, newkey);

+

+    map->elements++;

+

+    assert(pos < map->maxelements);

+

+}

+

+static int cacheset_lookup(struct cache_set *map, const struct cache_key *key)

+{

+    struct cache_key *newkey;

+    int ret;

+    uint32_t pos;

+    ret = cacheset_lookup_internal(map, key, &pos, 0);

+    if (!ret)

+	return CACHE_INVALID_VERSION;

+    newkey = &map->data[pos];

+    /* update LRU position: move to tail */

+    lru_remove(map, newkey);

+    lru_addtail(map, newkey);

+

+    return map->version;

+}

+

 static mpool_t *mempool = NULL;

 static struct CACHE {

-    struct CACHE_ENTRY {

-	unsigned char hash[15];

-	uint32_t dbver;

-	uint32_t hits;

-    } *items;

+    struct cache_set cacheset;

     pthread_mutex_t mutex;

     uint32_t lastdb;

 } *cache = NULL;

 static unsigned int cache_entries = 0;

-

 #define TREES 256

 static inline unsigned int getkey(uint8_t *hash) { return *hash; }

@@ -367,15 +237,6 @@ int cl_cache_init(unsigned int entries) {

     }

     for(i=0; i<TREES; i++) {

-	struct CACHE_ENTRY *e = mpool_calloc(mempool, sizeof(struct CACHE_ENTRY), entries);

-	if(!e) {

-	    cli_errmsg("mpool calloc fail\n");

-	    mpool_destroy(mempool);

-	    mempool = NULL;

-	    return 1;

-	}

-	cache[i].items = e;

-	cache[i].lastdb = 0;

 	if(pthread_mutex_init(&cache[i].mutex, NULL)) {

 	    cli_errmsg("mutex init fail\n");

 	    mpool_destroy(mempool);

@@ -383,112 +244,57 @@ int cl_cache_init(unsigned int entries) {

 	    cache = NULL;

 	    return 1;

 	}

+

+	cache[i].cacheset.data = mpool_calloc(mempool, 256, sizeof(*cache[i].cacheset.data));

+	if (!cache[i].cacheset.data)

+	    return CL_EMEM;

+	cache_setversion(&cache[i].cacheset, 1337);

+	cache[i].cacheset.capacity = 256;

+	cache[i].cacheset.maxelements = 80*256 / 100;

+	cache[i].cacheset.elements = 0;

+	cache[i].cacheset.version = CACHE_INVALID_VERSION;

+	cache[i].cacheset.lru_head = cache[i].cacheset.lru_tail = NULL;

     }

     cache_entries = entries;

     return 0;

 }

-void cache_swap(struct CACHE_ENTRY *e, unsigned int a) {

-    struct CACHE_ENTRY t;

-    unsigned int b = a-1;

-

-    if(!a || e[a].hits <= e[b].hits)

-	return;

-

-    do {

-	if(e[a].hits > e[b].hits)

-	    continue;

-	break;

-    } while(b--);

-    b++;

-

-    memcpy(&t, &e[a], sizeof(t));

-    memcpy(&e[a], &e[b], sizeof(t));

-    memcpy(&e[b], &t, sizeof(t));

-}

-

-static void updb(uint32_t db, unsigned int skip) {

-    unsigned int i;

-    for(i=0; i<TREES; i++) {

-	if(i==skip) continue;

-	if(pthread_mutex_lock(&cache[i].mutex)) {

-	    cli_errmsg("mutex lock fail\n");

-	    continue;

-	}

-	cache[i].lastdb = db;

-	pthread_mutex_unlock(&cache[i].mutex);	

-    }

-}

-

 static int cache_lookup_hash(unsigned char *md5, cli_ctx *ctx) {

-    unsigned int i;

+    struct cache_key entry;

     int ret = CL_VIRUS;

     unsigned int key = getkey(md5);

-    struct CACHE_ENTRY *e;

     struct CACHE *c;

     if(!cache) return ret;

     c = &cache[key];

-    e = c->items;

     if(pthread_mutex_lock(&c->mutex)) {

 	cli_errmsg("mutex lock fail\n");

 	return ret;

     }

-    if(c->lastdb <= ctx->engine->dbversion[0]) {

-	if(c->lastdb < ctx->engine->dbversion[0]) {

-	    c->lastdb = ctx->engine->dbversion[0];

-	    updb(c->lastdb, key);

-	} else {

-	    for(i=0; i<cache_entries; i++) {

-		if(!e[i].hits) break;

-		if(e[i].dbver == c->lastdb && !memcmp(e[i].hash, md5 + 1, 15)) {

-		    e[i].hits++;

-		    cache_swap(e, i);

-		    ret = CL_CLEAN;

-		    break;

-		}

-	    }

-	}

-    }

+    entry.size = 1024;

+    memcpy(entry.digest, md5, 16);

+    ret = (cacheset_lookup(&c->cacheset, &entry) == 1337) ? CL_CLEAN : CL_VIRUS;

     pthread_mutex_unlock(&c->mutex);

+    if(ret == CL_CLEAN) cli_warnmsg("cached\n");

     return ret;

 }

 void cache_add(unsigned char *md5, cli_ctx *ctx) {

+    struct cache_key entry;

     unsigned int key = getkey(md5);

-    unsigned int i, replace;

-    struct CACHE_ENTRY *e;

     struct CACHE *c;

     if(!cache) return;

     c = &cache[key];

-    e = c->items;

     if(pthread_mutex_lock(&c->mutex)) {

 	cli_errmsg("mutex lock fail\n");

 	return;

     }

-    if(c->lastdb == ctx->engine->dbversion[0]) {

-	replace = cache_entries;

-	for(i=0; i<cache_entries; i++) {

-	    if(!e[i].hits) break;

-	    if(replace == cache_entries && e[i].dbver < c->lastdb) {

-		replace = i;

-	    } else if(e[i].hits && !memcmp(e[i].hash, md5 + 1, 15)) {

-		e[i].hits++;

-		cache_swap(e, i);

-		pthread_mutex_unlock(&c->mutex);

-		return;

-	    }

-	}

-	if(replace == cache_entries)

-	    replace = cache_entries - 1 - (rand() % (cache_entries / CACHE_PERTURB));

-	e[replace].hits = 1;

-	e[replace].dbver = c->lastdb;

-	memcpy(e[replace].hash, md5 + 1, 15);

-	cache_swap(e, replace);

-    }

+    entry.size = 1024;

+    memcpy(entry.digest, md5, 16);

+    cacheset_add(&c->cacheset, &entry);

     pthread_mutex_unlock(&c->mutex);

     return;

 }