Browse code
Merge branch 'cache' of git.clam.sourcefire.com:/var/lib/git/clamav-devel into cache
aCaB authored on 2010/01/08 07:38:33Showing 1 changed files
| ... | ... |
@@ -1,7 +1,7 @@ |
| 1 | 1 |
/* |
| 2 | 2 |
* Copyright (C) 2010 Sourcefire, Inc. |
| 3 | 3 |
* |
| 4 |
- * Authors: aCaB <acab@clamav.net> |
|
| 4 |
+ * Authors: aCaB <acab@clamav.net>, Török Edvin <edwin@clamav.net> |
|
| 5 | 5 |
* |
| 6 | 6 |
* This program is free software; you can redistribute it and/or modify |
| 7 | 7 |
* it under the terms of the GNU General Public License version 2 as |
| ... | ... |
@@ -18,14 +18,15 @@ |
| 18 | 18 |
* MA 02110-1301, USA. |
| 19 | 19 |
*/ |
| 20 | 20 |
|
| 21 |
-#include <string.h> |
|
| 22 |
-#include <stdlib.h> |
|
| 23 |
-#include <pthread.h> |
|
| 24 |
- |
|
| 25 | 21 |
#if HAVE_CONFIG_H |
| 26 | 22 |
#include "clamav-config.h" |
| 27 | 23 |
#endif |
| 28 | 24 |
|
| 25 |
+#include <string.h> |
|
| 26 |
+#include <stdlib.h> |
|
| 27 |
+#include <pthread.h> |
|
| 28 |
+#include <assert.h> |
|
| 29 |
+ |
|
| 29 | 30 |
#include "md5.h" |
| 30 | 31 |
#include "mpool.h" |
| 31 | 32 |
#include "clamav.h" |
| ... | ... |
@@ -34,7 +35,6 @@ |
| 34 | 34 |
|
| 35 | 35 |
#define CACHE_PERTURB 8 |
| 36 | 36 |
/* 1/10th */ |
| 37 |
- |
|
| 38 | 37 |
static mpool_t *mempool = NULL; |
| 39 | 38 |
static struct CACHE {
|
| 40 | 39 |
struct CACHE_ENTRY {
|
| ... | ... |
@@ -220,3 +220,295 @@ int cache_check(unsigned char *hash, cli_ctx *ctx) {
|
| 220 | 220 |
cli_md5_final(hash, &md5); |
| 221 | 221 |
return cache_lookup_hash(hash, ctx); |
| 222 | 222 |
} |
| 223 |
+ |
|
| 224 |
+/* struct cache_key { */
|
|
| 225 |
+/* char digest[16]; */ |
|
| 226 |
+/* uint32_t size; /\* 0 is used to mark an empty hash slot! *\/ */ |
|
| 227 |
+/* struct cache_key *lru_next, *lru_prev; */ |
|
| 228 |
+/* }; */ |
|
| 229 |
+ |
|
| 230 |
+/* struct cache_set { */
|
|
| 231 |
+/* struct cache_key *data; */ |
|
| 232 |
+/* size_t capacity; */ |
|
| 233 |
+/* size_t maxelements; /\* considering load factor *\/ */ |
|
| 234 |
+/* size_t elements; */ |
|
| 235 |
+/* size_t version; */ |
|
| 236 |
+/* struct cache_key *lru_head, *lru_tail; */ |
|
| 237 |
+/* pthread_mutex_t mutex; */ |
|
| 238 |
+/* }; */ |
|
| 239 |
+ |
|
| 240 |
+/* #define CACHE_INVALID_VERSION ~0u */ |
|
| 241 |
+/* #define CACHE_KEY_DELETED ~0u */ |
|
| 242 |
+/* #define CACHE_KEY_EMPTY 0 */ |
|
| 243 |
+ |
|
| 244 |
+/* /\* size must be power of 2! *\/ */ |
|
| 245 |
+/* static int cacheset_init(struct cache_set* map, size_t maxsize, uint8_t loadfactor) */ |
|
| 246 |
+/* { */
|
|
| 247 |
+/* map->data = cli_calloc(maxsize, sizeof(*map->data)); */ |
|
| 248 |
+/* if (!map->data) */ |
|
| 249 |
+/* return CL_EMEM; */ |
|
| 250 |
+/* map->capacity = maxsize; */ |
|
| 251 |
+/* map->maxelements = loadfactor*maxsize / 100; */ |
|
| 252 |
+/* map->elements = 0; */ |
|
| 253 |
+/* map->version = CACHE_INVALID_VERSION; */ |
|
| 254 |
+/* map->lru_head = map->lru_tail = NULL; */ |
|
| 255 |
+/* if (pthread_mutex_init(&map->mutex, NULL)) { */
|
|
| 256 |
+/* cli_errmsg("mutex init fail\n"); */
|
|
| 257 |
+/* return CL_EMEM; */ |
|
| 258 |
+/* } */ |
|
| 259 |
+/* } */ |
|
| 260 |
+ |
|
| 261 |
+/* static void cacheset_destroy(struct cache_set *map) */ |
|
| 262 |
+/* { */
|
|
| 263 |
+/* pthread_mutex_destroy(&map->mutex); */ |
|
| 264 |
+/* free(map->data); */ |
|
| 265 |
+/* } */ |
|
| 266 |
+ |
|
| 267 |
+/* static void cacheset_acquire(struct cache_set *map) */ |
|
| 268 |
+/* { */
|
|
| 269 |
+/* pthread_mutex_lock(&map->mutex); */ |
|
| 270 |
+/* } */ |
|
| 271 |
+ |
|
| 272 |
+/* static void cache_setversion(struct cache_set* map, uint32_t version) */ |
|
| 273 |
+/* { */
|
|
| 274 |
+/* unsigned i; */ |
|
| 275 |
+/* if (map->version == version) */ |
|
| 276 |
+/* return; */ |
|
| 277 |
+/* map->version = version; */ |
|
| 278 |
+/* map->elements = 0; /\* all elements have expired now *\/ */ |
|
| 279 |
+/* for (i=0;i<map->capacity;i++) */ |
|
| 280 |
+/* map->data[i].size = 0; */ |
|
| 281 |
+/* map->lru_head = map->lru_tail = NULL; */ |
|
| 282 |
+/* } */ |
|
| 283 |
+ |
|
| 284 |
+/* static void cacheset_lru_remove(struct cache_set *map, size_t howmany) */ |
|
| 285 |
+/* { */
|
|
| 286 |
+/* while (howmany--) { */
|
|
| 287 |
+/* struct cache_key *old; */ |
|
| 288 |
+/* assert(map->lru_head); */ |
|
| 289 |
+/* assert(!old->lru_prev); */ |
|
| 290 |
+/* // Remove a key from the head of the list */ |
|
| 291 |
+/* old = map->lru_head; */ |
|
| 292 |
+/* map->lru_head = old->lru_next; */ |
|
| 293 |
+/* old->size = CACHE_KEY_DELETED; */ |
|
| 294 |
+/* /\* This slot is now deleted, it is not empty, */ |
|
| 295 |
+/* * because previously we could have inserted a key that has seen this */ |
|
| 296 |
+/* * slot as occupied, to find that key we need to ensure that all keys */ |
|
| 297 |
+/* * that were occupied when the key was inserted, are seen as occupied */ |
|
| 298 |
+/* * when searching too. */ |
|
| 299 |
+/* * Of course when inserting a new value, we treat deleted slots as */ |
|
| 300 |
+/* * empty. */ |
|
| 301 |
+/* * We only replace old values with new values, but there is no guarantee */ |
|
| 302 |
+/* * that the newly inserted value would hash to same place as the value */ |
|
| 303 |
+/* * we remove due to LRU! *\/ */ |
|
| 304 |
+/* if (old == map->lru_tail) */ |
|
| 305 |
+/* map->lru_tail = 0; */ |
|
| 306 |
+/* } */ |
|
| 307 |
+/* } */ |
|
| 308 |
+ |
|
| 309 |
+/* static inline uint32_t hash32shift(uint32_t key) */ |
|
| 310 |
+/* { */
|
|
| 311 |
+/* key = ~key + (key << 15); */ |
|
| 312 |
+/* key = key ^ (key >> 12); */ |
|
| 313 |
+/* key = key + (key << 2); */ |
|
| 314 |
+/* key = key ^ (key >> 4); */ |
|
| 315 |
+/* key = (key + (key << 3)) + (key << 11); */ |
|
| 316 |
+/* key = key ^ (key >> 16); */ |
|
| 317 |
+/* return key; */ |
|
| 318 |
+/* } */ |
|
| 319 |
+ |
|
| 320 |
+/* static inline size_t hash(const unsigned char* k,const size_t len,const size_t SIZE) */ |
|
| 321 |
+/* { */
|
|
| 322 |
+/* size_t Hash = 1; */ |
|
| 323 |
+/* size_t i; */ |
|
| 324 |
+/* for(i=0;i<len;i++) { */
|
|
| 325 |
+/* /\* a simple add is good, because we use the mixing function below *\/ */ |
|
| 326 |
+/* Hash += k[i]; */ |
|
| 327 |
+/* /\* mixing function *\/ */ |
|
| 328 |
+/* Hash = hash32shift(Hash); */ |
|
| 329 |
+/* } */ |
|
| 330 |
+/* /\* SIZE is power of 2 *\/ */ |
|
| 331 |
+/* return Hash & (SIZE - 1); */ |
|
| 332 |
+/* } */ |
|
| 333 |
+ |
|
| 334 |
+/* int cacheset_lookup_internal(struct cache_set *map, const struct cache_key *key, */ |
|
| 335 |
+/* uint32_t *insert_pos, int deletedok) */ |
|
| 336 |
+/* { */
|
|
| 337 |
+/* uint32_t idx = hash((const unsigned char*)key, sizeof(*key), map->capacity); */ |
|
| 338 |
+/* uint32_t tries = 0; */ |
|
| 339 |
+/* struct cache_key *k = &map->data[idx]; */ |
|
| 340 |
+/* while (k->size != CACHE_KEY_EMPTY) { */
|
|
| 341 |
+/* if (k->size == key->size && */ |
|
| 342 |
+/* !memcmp(k->digest, key, 16)) { */
|
|
| 343 |
+/* /\* found key *\/ */ |
|
| 344 |
+/* *insert_pos = idx; */ |
|
| 345 |
+/* return 1; */ |
|
| 346 |
+/* } */ |
|
| 347 |
+/* if (deletedok && k->size == CACHE_KEY_DELETED) { */
|
|
| 348 |
+/* /\* treat deleted slot as empty *\/ */ |
|
| 349 |
+/* *insert_pos = idx; */ |
|
| 350 |
+/* return 0; */ |
|
| 351 |
+/* } */ |
|
| 352 |
+/* idx = (idx + tries++)&(map->capacity-1); */ |
|
| 353 |
+/* k = &map->data[idx]; */ |
|
| 354 |
+/* } */ |
|
| 355 |
+/* /\* found empty pos *\/ */ |
|
| 356 |
+/* *insert_pos = idx; */ |
|
| 357 |
+/* return 0; */ |
|
| 358 |
+/* } */ |
|
| 359 |
+ |
|
| 360 |
+/* static inline void lru_remove(struct cache_set *map, struct cache_key *newkey) */ |
|
| 361 |
+/* { */
|
|
| 362 |
+/* if (newkey->lru_next) */ |
|
| 363 |
+/* newkey->lru_next->lru_prev = newkey->lru_prev; */ |
|
| 364 |
+/* if (newkey->lru_prev) */ |
|
| 365 |
+/* newkey->lru_prev->lru_next = newkey->lru_next; */ |
|
| 366 |
+/* if (newkey == map->lru_head) */ |
|
| 367 |
+/* map->lru_head = newkey->lru_next; */ |
|
| 368 |
+/* } */ |
|
| 369 |
+ |
|
| 370 |
+/* static inline void lru_addtail(struct cache_set *map, struct cache_key *newkey) */ |
|
| 371 |
+/* { */
|
|
| 372 |
+/* if (!map->lru_head) */ |
|
| 373 |
+/* map->lru_head = newkey; */ |
|
| 374 |
+/* if (map->lru_tail) */ |
|
| 375 |
+/* map->lru_tail->lru_next = newkey; */ |
|
| 376 |
+/* newkey->lru_next = NULL; */ |
|
| 377 |
+/* newkey->lru_prev = map->lru_tail; */ |
|
| 378 |
+/* map->lru_tail = newkey; */ |
|
| 379 |
+/* } */ |
|
| 380 |
+ |
|
| 381 |
+/* static void cacheset_add(struct cache_set *map, const struct cache_key *key) */ |
|
| 382 |
+/* { */
|
|
| 383 |
+/* int ret; */ |
|
| 384 |
+/* uint32_t pos; */ |
|
| 385 |
+/* struct cache_key *newkey; */ |
|
| 386 |
+/* if (map->elements >= map->maxelements) */ |
|
| 387 |
+/* cacheset_lru_remove(map, 1); */ |
|
| 388 |
+/* assert(map->elements < map->maxelements); */ |
|
| 389 |
+ |
|
| 390 |
+/* ret = cacheset_lookup_internal(map, key, &pos, 1); */ |
|
| 391 |
+/* newkey = &map->data[pos]; */ |
|
| 392 |
+/* if (ret) { */
|
|
| 393 |
+/* /\* was already added, remove from LRU list *\/ */ |
|
| 394 |
+/* lru_remove(map, newkey); */ |
|
| 395 |
+/* } */ |
|
| 396 |
+/* /\* add new key to tail of LRU list *\/ */ |
|
| 397 |
+/* lru_addtail(map, newkey); */ |
|
| 398 |
+ |
|
| 399 |
+/* map->elements++; */ |
|
| 400 |
+ |
|
| 401 |
+/* assert(pos < map->maxelements); */ |
|
| 402 |
+ |
|
| 403 |
+/* memcpy(&map->data[pos], key, sizeof(*key)); */ |
|
| 404 |
+/* } */ |
|
| 405 |
+ |
|
| 406 |
+/* static int cacheset_lookup(struct cache_set *map, const struct cache_key *key) */ |
|
| 407 |
+/* { */
|
|
| 408 |
+/* struct cache_key *newkey; */ |
|
| 409 |
+/* int ret; */ |
|
| 410 |
+/* uint32_t pos; */ |
|
| 411 |
+/* ret = cacheset_lookup_internal(map, key, &pos, 0); */ |
|
| 412 |
+/* if (!ret) */ |
|
| 413 |
+/* return CACHE_INVALID_VERSION; */ |
|
| 414 |
+/* newkey = &map->data[pos]; */ |
|
| 415 |
+/* /\* update LRU position: move to tail *\/ */ |
|
| 416 |
+/* lru_remove(map, newkey); */ |
|
| 417 |
+/* lru_addtail(map, newkey); */ |
|
| 418 |
+ |
|
| 419 |
+/* return map->version; */ |
|
| 420 |
+/* } */ |
|
| 421 |
+ |
|
| 422 |
+/* static void cacheset_release(struct cache_set *map) */ |
|
| 423 |
+/* { */
|
|
| 424 |
+/* pthread_mutex_unlock(&map->mutex); */ |
|
| 425 |
+/* } */ |
|
| 426 |
+ |
|
| 427 |
+/* #if 0 */ |
|
| 428 |
+/* int main(int argc, char **argv) */ |
|
| 429 |
+/* { */
|
|
| 430 |
+/* struct cache_key key; */ |
|
| 431 |
+/* struct cache_set map; */ |
|
| 432 |
+/* cacheset_init(&map, 256, 80); */ |
|
| 433 |
+/* cacheset_acquire(&map); */ |
|
| 434 |
+/* cache_setversion(&map, 10); */ |
|
| 435 |
+ |
|
| 436 |
+/* key.size = 1024; */ |
|
| 437 |
+/* memcpy(key.digest, "1234567890123456", 16); */ |
|
| 438 |
+/* cacheset_add(&map, &key); */ |
|
| 439 |
+/* memcpy(key.digest, "1234567890123457", 16); */ |
|
| 440 |
+/* cacheset_add(&map, &key); */ |
|
| 441 |
+/* memcpy(key.digest, "0123456789012345", 16); */ |
|
| 442 |
+/* cacheset_add(&map, &key); */ |
|
| 443 |
+ |
|
| 444 |
+/* key.size = 1024; */ |
|
| 445 |
+/* memcpy(key.digest, "1234567890123456", 16); */ |
|
| 446 |
+/* if (cacheset_lookup(&map, &key) != 10) */ |
|
| 447 |
+/* abort(); */ |
|
| 448 |
+/* memcpy(key.digest, "1234567890123456", 16); */ |
|
| 449 |
+/* if (cacheset_lookup(&map, &key) != 10) */ |
|
| 450 |
+/* abort(); */ |
|
| 451 |
+/* memcpy(key.digest, "1234567890123457", 16); */ |
|
| 452 |
+/* if (cacheset_lookup(&map, &key) != 10) */ |
|
| 453 |
+/* abort(); */ |
|
| 454 |
+/* memcpy(key.digest, "0123456789012345", 16); */ |
|
| 455 |
+/* if (cacheset_lookup(&map, &key) != 10) */ |
|
| 456 |
+/* abort(); */ |
|
| 457 |
+/* memcpy(key.digest, "0123456789012346", 16); */ |
|
| 458 |
+/* if (cacheset_lookup(&map, &key) == 10) */ |
|
| 459 |
+/* abort(); */ |
|
| 460 |
+ |
|
| 461 |
+/* cache_setversion(&map, 1); */ |
|
| 462 |
+/* memcpy(key.digest, "1234567890123456", 16); */ |
|
| 463 |
+/* if (cacheset_lookup(&map, &key) != CACHE_INVALID_VERSION) */ |
|
| 464 |
+/* abort(); */ |
|
| 465 |
+/* memcpy(key.digest, "1234567890123456", 16); */ |
|
| 466 |
+/* if (cacheset_lookup(&map, &key) != CACHE_INVALID_VERSION) */ |
|
| 467 |
+/* abort(); */ |
|
| 468 |
+/* memcpy(key.digest, "1234567890123457", 16); */ |
|
| 469 |
+/* if (cacheset_lookup(&map, &key) != CACHE_INVALID_VERSION) */ |
|
| 470 |
+/* abort(); */ |
|
| 471 |
+/* memcpy(key.digest, "0123456789012345", 16); */ |
|
| 472 |
+/* if (cacheset_lookup(&map, &key) != CACHE_INVALID_VERSION) */ |
|
| 473 |
+/* abort(); */ |
|
| 474 |
+ |
|
| 475 |
+/* cacheset_release(&map); */ |
|
| 476 |
+ |
|
| 477 |
+/* cacheset_destroy(&map); */ |
|
| 478 |
+ |
|
| 479 |
+/* cacheset_init(&map, 8, 50); */ |
|
| 480 |
+/* cacheset_acquire(&map); */ |
|
| 481 |
+/* cache_setversion(&map, 10); */ |
|
| 482 |
+ |
|
| 483 |
+/* key.size = 416; */ |
|
| 484 |
+/* memcpy(key.digest, "1234567890123456", 16); */ |
|
| 485 |
+/* cacheset_add(&map, &key); */ |
|
| 486 |
+/* memcpy(key.digest, "1234567890123457", 16); */ |
|
| 487 |
+/* cacheset_add(&map, &key); */ |
|
| 488 |
+/* memcpy(key.digest, "1234567890123459", 16); */ |
|
| 489 |
+/* cacheset_add(&map, &key); */ |
|
| 490 |
+/* key.size = 400; */ |
|
| 491 |
+/* memcpy(key.digest, "1234567890123450", 16); */ |
|
| 492 |
+/* cacheset_add(&map, &key); */ |
|
| 493 |
+ |
|
| 494 |
+/* key.size = 416; */ |
|
| 495 |
+/* memcpy(key.digest, "1234567890123456", 16); */ |
|
| 496 |
+/* if (cacheset_lookup(&map, &key) != 10) */ |
|
| 497 |
+/* abort(); */ |
|
| 498 |
+/* if (cacheset_lookup(&map, &key) != 10) */ |
|
| 499 |
+/* abort(); */ |
|
| 500 |
+/* if (cacheset_lookup(&map, &key) != 10) */ |
|
| 501 |
+/* abort(); */ |
|
| 502 |
+ |
|
| 503 |
+/* key.size = 500; */ |
|
| 504 |
+/* cacheset_add(&map, &key); */ |
|
| 505 |
+/* memcpy(key.digest, "1234567890123457", 16); */ |
|
| 506 |
+/* if (cacheset_lookup(&map, &key) == 10) */ |
|
| 507 |
+/* abort(); */ |
|
| 508 |
+ |
|
| 509 |
+/* cacheset_release(&map); */ |
|
| 510 |
+/* cacheset_destroy(&map); */ |
|
| 511 |
+ |
|
| 512 |
+/* return 0; */ |
|
| 513 |
+/* } */ |
|
| 514 |
+/* #endif */ |