@@ -1,3 +1,9 @@

+Wed Mar 11 22:06:30 EET 2009 (edwin)

+------------------------------------

+ * libclamav/phishcheck.c, libclamav/regex_list.c,

+ libclamav/regex_list.h, unit_tests/check_regex.c,

+ unit_tests/input/daily.gdb: make use of hostkey prefix entries

+

 Wed Mar 11 21:27:32 EET 2009 (edwin)

 ------------------------------------

  * clamd/others.c, sigtool/Makefile.in: fix previous commit

@@ -1172,7 +1172,7 @@ static int whitelist_check(const struct cl_engine* engine,struct url_check* urls

 	return whitelist_match(engine,urls->realLink.data,urls->displayLink.data,hostOnly);

 }

-static int hash_match(const struct regex_matcher *rlist, const char *host, size_t hlen, const char *path, size_t plen)

+static int hash_match(const struct regex_matcher *rlist, const char *host, size_t hlen, const char *path, size_t plen, int *prefix_matched)

 {

 	const char *virname;

 #if 0

@@ -1198,9 +1198,15 @@ static int hash_match(const struct regex_matcher *rlist, const char *host, size_

 		h[2*i+1] = hexchars[sha256_dig[i]&0xf];

 	    }

 	    h[64]='\0';

-	    cli_dbgmsg("Looking up hash %s for %s%s\n", h, host, path);

-	    if(SO_search(&rlist->sha256_filter, sha256_dig, 32) != -1 &&

-	       cli_bm_scanbuff(sha256_dig, 32, &virname, &rlist->sha256_hashes,0,0,-1) == CL_VIRUS) {

+	    cli_dbgmsg("Looking up hash %s for %s(%u)%s(%u)\n", h, host, hlen, path, plen);

+	    if (prefix_matched) {

+		if (cli_bm_scanbuff(sha256_dig, 4, &virname, &rlist->hostkey_prefix,0,0,-1) == CL_VIRUS) {

+		    cli_dbgmsg("prefix matched\n", virname);

+		    *prefix_matched = 1;

+		} else

+		    return CL_SUCCESS;

+	    }

+	    if (cli_bm_scanbuff(sha256_dig, 32, &virname, &rlist->sha256_hashes,0,0,-1) == CL_VIRUS) {

 		switch(*virname) {

 		    case '1':

 			return CL_PHISH_HASH1;

@@ -1316,10 +1322,11 @@ static int url_hash_match(const struct regex_matcher *rlist, const char *inurl,

 	size_t path_len;

 	size_t host_len;

 	char *p;

-	int rc;

+	int rc, prefix_matched=0;

 	const char *lp[COMPONENTS+1];

 	size_t pp[COMPONENTS+2];

 	char urlbuff[URL_MAX_LEN+3];/* htmlnorm truncates at 1024 bytes + terminating null + slash + host end null */

+	unsigned count;

 	if(!rlist || !rlist->sha256_hashes.bm_patterns) {

 		return CL_SUCCESS;

@@ -1358,15 +1365,27 @@ static int url_hash_match(const struct regex_matcher *rlist, const char *inurl,

 		}

 	} else

 		k = 1;

-

-	for(ji=j;ji < COMPONENTS+1; ji++) {

-		for(ki=0;ki < k; ki++) {

-			assert(pp[ki] <= path_len);

-			rc = hash_match(rlist, lp[ji], host_begin + host_len - lp[ji] + 1, path_begin, pp[ki]);

-			if(rc) {

-				return rc;

-			}

+	count = 0;

+	for(ki=k;ki > 0;) {

+	    --ki;

+	    for(ji=COMPONENTS+1;ji > j;) {

+		/* lookup last 2 and 3 components of host, as hostkey prefix,

+		 * if not matched, shortcircuit lookups */

+		int need_prefixmatch = (count<2 && !prefix_matched) &&

+				       rlist->hostkey_prefix.bm_patterns;

+		--ji;

+		assert(pp[ki] <= path_len);

+		rc = hash_match(rlist, lp[ji], host_begin + host_len - lp[ji] + 1, path_begin, pp[ki], 

+				need_prefixmatch ? &prefix_matched : NULL);

+		if(rc) {

+		    return rc;

+		}

+		count++;

+		if (count == 2 && !prefix_matched && rlist->hostkey_prefix.bm_patterns) {

+		    cli_dbgmsg("hostkey prefix not matched, short-circuiting lookups\n");

+		    return CL_SUCCESS;

 		}

+	    }

 	}

 	return CL_SUCCESS;

 }

@@ -1394,8 +1413,11 @@ static enum phish_status phishingCheck(const struct cl_engine* engine,struct url

 	}

 	if(( rc = url_hash_match(engine->domainlist_matcher, urls->realLink.data, strlen(urls->realLink.data)) )) {

+	    if (rc == CL_PHISH_CLEAN)

+		cli_dbgmsg("not analyzing, not a real url: %s\n", urls->realLink.data);

+	    else

 		cli_dbgmsg("Hash matched for: %s\n", urls->realLink.data);

-		return rc;

+	    return rc;

 	}

 	if((rc = cleanupURLs(urls))) {

@@ -372,12 +372,15 @@ int init_regex_list(struct regex_matcher* matcher)

 	}

 #ifdef USE_MPOOL

 	matcher->sha256_hashes.mempool = mp;

+	matcher->hostkey_prefix.mempool = mp;

 #endif

 	if((rc = cli_bm_init(&matcher->sha256_hashes))) {

 		return rc;

 	}

+	if((rc = cli_bm_init(&matcher->hostkey_prefix))) {

+		return rc;

+	}

 	SO_init(&matcher->filter);

-	SO_init(&matcher->sha256_filter);

 	return CL_SUCCESS;

 }

@@ -424,10 +427,11 @@ static int functionality_level_check(char* line)

 	}

 }

-static int add_hash(struct regex_matcher *matcher, char* pattern, const char fl)

+static int add_hash(struct regex_matcher *matcher, char* pattern, const char fl, int is_prefix)

 {

 	int rc;

 	struct cli_bm_patt *pat = mpool_calloc(matcher->mempool, 1, sizeof(*pat));

+	struct cli_matcher *bm;

 	if(!pat)

 		return CL_EMEM;

 	pat->pattern = (unsigned char*)cli_mpool_hex2str(matcher->mempool, pattern);

@@ -440,8 +444,14 @@ static int add_hash(struct regex_matcher *matcher, char* pattern, const char fl)

 		return CL_EMEM;

 	}

 	*pat->virname = fl;

-	SO_preprocess_add(&matcher->sha256_filter, pat->pattern, pat->length);

-	if((rc = cli_bm_addpatt(&matcher->sha256_hashes, pat))) {

+	if (is_prefix) {

+	    pat->length=4;

+	    bm = &matcher->hostkey_prefix;

+	} else {

+	    bm = &matcher->sha256_hashes;

+	}

+

+	if((rc = cli_bm_addpatt(bm, pat))) {

 		cli_errmsg("add_hash: failed to add BM pattern\n");

 		free(pat->pattern);

 		free(pat->virname);

@@ -542,15 +552,12 @@ int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int *sign

 				return rc==CL_EMEM ? CL_EMEM : CL_EMALFDB;

 		} else if (buffer[0] == 'S' && !is_whitelist) {

 			pattern[pattern_len] = '\0';

-			if(*pattern=='F' && pattern[1]==':') {

+			if((pattern[0]=='F' || pattern[0]=='P') && pattern[1]==':') {

 			    pattern += 2;

-			    if (( rc = add_hash(matcher, pattern, flags[0]) )) {

+			    if (( rc = add_hash(matcher, pattern, flags[0], pattern[-2] == 'P') )) {

 				cli_errmsg("Error loading at line: %d\n", line);

 				return rc==CL_EMEM ? CL_EMEM : CL_EMALFDB;

 			    }

-			} else if (*pattern=='P' && pattern[1]==':') {

-			    pattern += 2;

-			    /* TODO: hostkey prefix */

 			} else {

 			    cli_errmsg("Error loading line: %d, %c\n", line, *pattern);

 			    return CL_EMALFDB;

@@ -617,6 +624,7 @@ void regex_list_done(struct regex_matcher* matcher)

 		}

 		hashtab_free(&matcher->suffix_hash);

 		cli_bm_free(&matcher->sha256_hashes);

+		cli_bm_free(&matcher->hostkey_prefix);

 	}

 }

@@ -52,7 +52,7 @@ struct regex_matcher {

 	regex_t **all_pregs;

 	struct cli_matcher suffixes;

 	struct cli_matcher sha256_hashes;

-	struct filter sha256_filter;

+	struct cli_matcher hostkey_prefix;

 	struct filter filter;

 #ifdef USE_MPOOL

 	mpool_t *mempool;

@@ -313,7 +313,7 @@ static void psetup_impl(int load2)

 		fail_unless(rc == 0, "load_regex_matcher");

 		fclose(f);

-		fail_unless_fmt(signo == 2, "Incorrect number of signatures: %u, expected %u", signo, 2);

+		fail_unless_fmt(signo == 4, "Incorrect number of signatures: %u, expected %u", signo, 4);

 	}

 	loaded_2 = load2;

@@ -1,2 +1,4 @@

+S:P:d1b8a025

 S:F:d1b8a0251d7555d016b6468ae623e4b1e830c7efccc54966d09447a3d0a85c60

+S2:P:7f6fd541

 S2:F:7f6fd541e625e7bc5d5a64f166e47ecfe13735464a74d160b48265c162a71089