Browse code

enable prefiltering, and add to dconf.

Also downgrade some warnings to debug messages.

Török Edvin authored on 2010/02/15 22:01:37
Showing 12 changed files
... ...
@@ -110,6 +110,7 @@ static struct dconf_module modules[] = {
110 110
     { "OTHER",	    "CRYPTFF",	    OTHER_CONF_CRYPTFF,	    1 },
111 111
     { "OTHER",	    "DLP",	    OTHER_CONF_DLP,	    1 },
112 112
     { "OTHER",	    "MYDOOMLOG",    OTHER_CONF_MYDOOMLOG,   1 },
113
+    { "OTHER",      "PREFILTERING", OTHER_CONF_PREFILTERING,1 },
113 114
 
114 115
     { "PHISHING",   "ENGINE",       PHISHING_CONF_ENGINE,   1 },
115 116
     { "PHISHING",   "ENTCONV",      PHISHING_CONF_ENTCONV,  1 },
... ...
@@ -99,6 +99,7 @@ struct cli_dconf {
99 99
 #define OTHER_CONF_CRYPTFF  0x10
100 100
 #define OTHER_CONF_DLP	    0x20
101 101
 #define OTHER_CONF_MYDOOMLOG	0x40
102
+#define OTHER_CONF_PREFILTERING 0x80
102 103
 
103 104
 /* Phishing flags */
104 105
 #define PHISHING_CONF_ENGINE   0x1
... ...
@@ -228,7 +228,7 @@ int filter_add_static(struct filter *m, const unsigned char *pattern, unsigned l
228 228
 
229 229
 	assert(best_pos < len-1);
230 230
 	if (pattern[best_pos] == 0 && pattern[best_pos+1] == 0) {
231
-		cli_warnmsg("filter: subsignature begins with zero (static): %s\n", name);
231
+		cli_dbgmsg("!filter: subsignature begins with zero (static): %s\n", name);
232 232
 	}
233 233
 	pattern += best_pos;
234 234
 	len -= best_pos;
... ...
@@ -627,7 +627,7 @@ int  filter_add_acpatt(struct filter *m, const struct cli_ac_patt *pat)
627 627
 				unsigned char c0 = spec_ith_char(spec0, k0);
628 628
 				unsigned char c1 = spec_ith_char(spec1, k1);
629 629
 				if (!c0 && !c1 && !i) {
630
-					cli_warnmsg("filter: subsignature begins with zero: %s\n",pat->virname);
630
+					cli_dbgmsg("!filter: subsignature begins with zero: %s\n",pat->virname);
631 631
 				}
632 632
 				filter_set_atpos(m, i, c0 | (c1<<8));
633 633
 			}
... ...
@@ -360,11 +360,11 @@ int cli_ac_buildtrie(struct cli_matcher *root)
360 360
     }
361 361
 
362 362
     if (root->filter)
363
-	cli_warnmsg("Using filter for trie %d\n", root->type);
363
+	cli_dbgmsg("!Using filter for trie %d\n", root->type);
364 364
     return ac_maketrans(root);
365 365
 }
366 366
 
367
-int cli_ac_init(struct cli_matcher *root, uint8_t mindepth, uint8_t maxdepth)
367
+int cli_ac_init(struct cli_matcher *root, uint8_t mindepth, uint8_t maxdepth, uint8_t dconf_prefiltering)
368 368
 {
369 369
 #ifdef USE_MPOOL
370 370
     assert(root->mempool && "mempool must be initialized");
... ...
@@ -386,8 +386,7 @@ int cli_ac_init(struct cli_matcher *root, uint8_t mindepth, uint8_t maxdepth)
386 386
     root->ac_mindepth = mindepth;
387 387
     root->ac_maxdepth = maxdepth;
388 388
 
389
-    /* TODO: dconf here ?*/
390
-    if (cli_mtargets[root->type].enable_prefiltering && 0) {/* Disabled for now */
389
+    if (cli_mtargets[root->type].enable_prefiltering && dconf_prefiltering) {
391 390
 	root->filter = mpool_malloc(root->mempool, sizeof(*root->filter));
392 391
 	if (!root->filter) {
393 392
 	    cli_errmsg("cli_ac_init: Can't allocate memory for ac_root->filter\n");
... ...
@@ -94,7 +94,7 @@ int cli_ac_chklsig(const char *expr, const char *end, uint32_t *lsigcnt, unsigne
94 94
 void cli_ac_freedata(struct cli_ac_data *data);
95 95
 int cli_ac_scanbuff(const unsigned char *buffer, uint32_t length, const char **virname, void **customdata, struct cli_ac_result **res, const struct cli_matcher *root, struct cli_ac_data *mdata, uint32_t offset, cli_file_t ftype, struct cli_matched_type **ftoffset, unsigned int mode, const cli_ctx *ctx);
96 96
 int cli_ac_buildtrie(struct cli_matcher *root);
97
-int cli_ac_init(struct cli_matcher *root, uint8_t mindepth, uint8_t maxdepth);
97
+int cli_ac_init(struct cli_matcher *root, uint8_t mindepth, uint8_t maxdepth, uint8_t dconf_prefiltering);
98 98
 int cli_ac_caloff(const struct cli_matcher *root, struct cli_ac_data *data, fmap_t *map);
99 99
 void cli_ac_free(struct cli_matcher *root);
100 100
 int cli_ac_addsig(struct cli_matcher *root, const char *virname, const char *hexsig, uint32_t sigid, uint16_t parts, uint16_t partno, uint16_t rtype, uint16_t type, uint32_t mindist, uint32_t maxdist, const char *offset, const uint32_t *lsigid, unsigned int options);
... ...
@@ -56,7 +56,7 @@ int init_domainlist(struct cl_engine* engine)
56 56
 #ifdef USE_MPOOL
57 57
 		((struct regex_matcher*)engine->domainlist_matcher)->mempool = engine->mempool;
58 58
 #endif
59
-		return init_regex_list(engine->domainlist_matcher);
59
+		return init_regex_list(engine->domainlist_matcher, engine->dconf->other&OTHER_CONF_PREFILTERING);
60 60
 	}
61 61
 	else
62 62
 		return CL_ENULLARG;
... ...
@@ -57,7 +57,7 @@ int init_whitelist(struct cl_engine* engine)
57 57
 #endif
58 58
 		if(!engine->whitelist_matcher)
59 59
 			return CL_EMEM;
60
-		return	init_regex_list(engine->whitelist_matcher);
60
+		return	init_regex_list(engine->whitelist_matcher, engine->dconf->other&OTHER_CONF_PREFILTERING);
61 61
 	}
62 62
 	else
63 63
 		return CL_ENULLARG;
... ...
@@ -344,7 +344,7 @@ int cli_initroots(struct cl_engine *engine, unsigned int options)
344 344
 		root->ac_only = 1;
345 345
 
346 346
 	    cli_dbgmsg("Initialising AC pattern matcher of root[%d]\n", i);
347
-	    if((ret = cli_ac_init(root, engine->ac_mindepth, engine->ac_maxdepth))) {
347
+	    if((ret = cli_ac_init(root, engine->ac_mindepth, engine->ac_maxdepth, engine->dconf->other&OTHER_CONF_PREFILTERING))) {
348 348
 		/* no need to free previously allocated memory here */
349 349
 		cli_errmsg("cli_initroots: Can't initialise AC pattern matcher\n");
350 350
 		return ret;
... ...
@@ -817,7 +817,7 @@ static int cli_loadwdb(FILE *fs, struct cl_engine *engine, unsigned int options,
817 817
 	}
818 818
     }
819 819
 
820
-    if((ret = load_regex_matcher(engine->whitelist_matcher, fs, NULL, options, 1, dbio))) {
820
+    if((ret = load_regex_matcher(engine->whitelist_matcher, fs, NULL, options, 1, dbio, engine->dconf->other&OTHER_CONF_PREFILTERING))) {
821 821
 	return ret;
822 822
     }
823 823
 
... ...
@@ -838,7 +838,7 @@ static int cli_loadpdb(FILE *fs, struct cl_engine *engine, unsigned int *signo,
838 838
 	}
839 839
     }
840 840
 
841
-    if((ret = load_regex_matcher(engine->domainlist_matcher, fs, signo, options, 0, dbio))) {
841
+    if((ret = load_regex_matcher(engine->domainlist_matcher, fs, signo, options, 0, dbio, engine->dconf->other&OTHER_CONF_PREFILTERING))) {
842 842
 	return ret;
843 843
     }
844 844
 
... ...
@@ -251,7 +251,7 @@ int regex_list_match(struct regex_matcher* matcher,char* real_url,const char* di
251 251
 
252 252
 /* Initialization & loading */
253 253
 /* Initializes @matcher, allocating necesarry substructures */
254
-int init_regex_list(struct regex_matcher* matcher)
254
+int init_regex_list(struct regex_matcher* matcher, uint8_t dconf_prefiltering)
255 255
 {
256 256
 #ifdef USE_MPOOL
257 257
 	mpool_t *mp = matcher->mempool;
... ...
@@ -270,7 +270,7 @@ int init_regex_list(struct regex_matcher* matcher)
270 270
 	matcher->suffixes.mempool = mp;
271 271
 	assert(mp && "mempool must be initialized");
272 272
 #endif
273
-	if((rc = cli_ac_init(&matcher->suffixes, 2, 32))) {
273
+	if((rc = cli_ac_init(&matcher->suffixes, 2, 32, dconf_prefiltering))) {
274 274
 		return rc;
275 275
 	}
276 276
 #ifdef USE_MPOOL
... ...
@@ -385,7 +385,7 @@ static int add_hash(struct regex_matcher *matcher, char* pattern, const char fl,
385 385
 
386 386
 
387 387
 /* Load patterns/regexes from file */
388
-int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int *signo,unsigned int options,int is_whitelist,struct cli_dbio *dbio)
388
+int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int *signo,unsigned int options,int is_whitelist,struct cli_dbio *dbio, uint8_t dconf_prefiltering)
389 389
 {
390 390
 	int rc,line=0,entry=0;
391 391
 	char buffer[FILEBUFF];
... ...
@@ -401,7 +401,7 @@ int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int *sign
401 401
 
402 402
 	cli_dbgmsg("Loading regex_list\n");
403 403
 	if(!matcher->list_inited) {
404
-		rc = init_regex_list(matcher);
404
+		rc = init_regex_list(matcher, dconf_prefiltering);
405 405
 		if (!matcher->list_inited) {
406 406
 			cli_errmsg("Regex list failed to initialize!\n");
407 407
 			fatal_error(matcher);
... ...
@@ -60,8 +60,8 @@ struct regex_matcher {
60 60
 int cli_build_regex_list(struct regex_matcher* matcher);
61 61
 int regex_list_add_pattern(struct regex_matcher *matcher, char *pattern);
62 62
 int regex_list_match(struct regex_matcher* matcher, char* real_url,const char* display_url,const struct pre_fixup_info* pre_fixup, int hostOnly,const char **info, int is_whitelist);
63
-int init_regex_list(struct regex_matcher* matcher);
64
-int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int *signo,unsigned int options,int is_whitelist,struct cli_dbio *dbio);
63
+int init_regex_list(struct regex_matcher* matcher, uint8_t dconf_prefiltering);
64
+int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int *signo,unsigned int options,int is_whitelist,struct cli_dbio *dbio,uint8_t dconf_prefiltering);
65 65
 void regex_list_cleanup(struct regex_matcher* matcher);
66 66
 void regex_list_done(struct regex_matcher* matcher);
67 67
 int is_regex_ok(struct regex_matcher* matcher);
... ...
@@ -91,7 +91,7 @@ START_TEST (test_ac_scanbuff) {
91 91
 #ifdef USE_MPOOL
92 92
     root->mempool = mpool_create();
93 93
 #endif
94
-    ret = cli_ac_init(root, CLI_DEFAULT_AC_MINDEPTH, CLI_DEFAULT_AC_MAXDEPTH);
94
+    ret = cli_ac_init(root, CLI_DEFAULT_AC_MINDEPTH, CLI_DEFAULT_AC_MAXDEPTH, 1);
95 95
     fail_unless(ret == CL_SUCCESS, "cli_ac_init() failed");
96 96
 
97 97
 
... ...
@@ -157,7 +157,7 @@ static void rsetup(void)
157 157
 #ifdef USE_MPOOL
158 158
 	matcher.mempool = mpool_create();
159 159
 #endif
160
-	rc = init_regex_list(&matcher);
160
+	rc = init_regex_list(&matcher, 1);
161 161
 	fail_unless(rc == 0, "init_regex_list");
162 162
 }
163 163
 
... ...
@@ -298,7 +298,7 @@ static void psetup_impl(int load2)
298 298
 	f = fdopen(open_testfile("input/daily.pdb"),"r");
299 299
 	fail_unless(!!f, "fopen daily.pdb");
300 300
 
301
-	rc = load_regex_matcher(engine->domainlist_matcher,  f, &signo, 0, 0, NULL);
301
+	rc = load_regex_matcher(engine->domainlist_matcher,  f, &signo, 0, 0, NULL, 1);
302 302
 	fail_unless(rc == 0, "load_regex_matcher");
303 303
 	fclose(f);
304 304
 
... ...
@@ -309,7 +309,7 @@ static void psetup_impl(int load2)
309 309
 		fail_unless(!!f, "fopen daily.gdb");
310 310
 
311 311
 		signo = 0;
312
-		rc = load_regex_matcher(engine->domainlist_matcher,  f, &signo, 0, 0, NULL);
312
+		rc = load_regex_matcher(engine->domainlist_matcher,  f, &signo, 0, 0, NULL, 1);
313 313
 		fail_unless(rc == 0, "load_regex_matcher");
314 314
 		fclose(f);
315 315
 
... ...
@@ -322,7 +322,7 @@ static void psetup_impl(int load2)
322 322
 
323 323
 	f = fdopen(open_testfile("input/daily.wdb"),"r");
324 324
 	signo = 0;
325
-	rc = load_regex_matcher(engine->whitelist_matcher, f, &signo, 0, 1, NULL);
325
+	rc = load_regex_matcher(engine->whitelist_matcher, f, &signo, 0, 1, NULL, 1);
326 326
 	fail_unless(rc == 0,"load_regex_matcher");
327 327
 	fclose(f);
328 328