@@ -1,3 +1,9 @@

+Thu Apr  2 23:35:36 EEST 2009 (edwin)

+-------------------------------------

+ * docs/phishsigs_howto.tex, libclamav/phishcheck.c,

+ libclamav/readdb.c, libclamav/regex_list.c: local.gdb whitelisting

+ of safebrowsing entries (bb #1482).

+

 Thu Apr  2 22:59:30 EEST 2009 (edwin)

 -------------------------------------

  * libclamav/htmlnorm.c, libclamav/htmlnorm.h, libclamav/mbox.c,

Binary files a/docs/phishsigs_howto.pdf and b/docs/phishsigs_howto.pdf differ

@@ -57,6 +57,7 @@ S1:P:HostPrefix[:FuncLevelSpec]

 S1:F:Sha256hash[:FuncLevelSpec]

 S2:P:HostPrefix[:FuncLevelSpec]

 S2:F:Sha256hash[:FuncLevelSpec]

+S:W:Sha256hash[:FuncLevelSpec]

 \end{verbatim}

 \begin{description}

@@ -67,6 +68,8 @@ S2:F:Sha256hash[:FuncLevelSpec]

  \item [{S1:}]

 	Hashes for blacklisting phishing sites.

 	Virus name: Phishing.URL.Blacklisted

+ \item [{S:W}]

+	Locally whitelisted hashes.

  \item [{HostPrefix}]

 	4-byte prefix of the sha256 hash of the last 2 or 3 components of the hostname.

 If prefix doesn't match, no further lookups are performed.

@@ -76,7 +79,8 @@ If prefix doesn't match, no further lookups are performed.

 To see which hash/URL matched, look at the \verb+clamscan --debug+ output, and look for the following strings:

 \verb+Looking up hash+, \verb+prefix matched+, and \verb+Hash matched+.

-Local whitelisting of .gdb entries can be done by creating .wdb entries.

+Local whitelisting of .gdb entries can be done by creating a local.gdb file, and

+adding a line \verb+S:W:<HASH>+.

 \subsection{WDB format}

 This file contains whitelisted url pairs

@@ -1206,6 +1206,9 @@ static int hash_match(const struct regex_matcher *rlist, const char *host, size_

 	    }

 	    if (cli_bm_scanbuff(sha256_dig, 32, &virname, &rlist->sha256_hashes,0,0,-1) == CL_VIRUS) {

 		switch(*virname) {

+		    case 'W':

+			cli_dbgmsg("Hash is whitelisted, skipping\n");

+			break;

 		    case '1':

 			return CL_PHISH_HASH1;

 		    case '2':

@@ -1413,7 +1416,7 @@ static enum phish_status phishingCheck(const struct cl_engine* engine,struct url

 {

 	struct url_check host_url;

 	int rc = CL_PHISH_NODECISION;

-	int phishy=0, blacklisted=0;

+	int phishy=0;

 	const struct phishcheck* pchk = (const struct phishcheck*) engine->phishcheck;

 	if(!urls->realLink.data)

@@ -1436,10 +1439,14 @@ static enum phish_status phishingCheck(const struct cl_engine* engine,struct url

 		return CL_PHISH_CLEAN;

 	    } else {

 		cli_dbgmsg("Hash matched for: %s\n", urls->realLink.data);

-		blacklisted = rc;

+		return rc;

 	    }

 	}

+	if (urls->displayLink.data[0] == '\0') {

+	    return CL_PHISH_CLEAN;

+	}

+

 	if((rc = cleanupURLs(urls))) {

 		/* it can only return an error, or say its clean;

 		 * it is not allowed to decide it is phishing */

@@ -1453,20 +1460,12 @@ static enum phish_status phishingCheck(const struct cl_engine* engine,struct url

 			( (phishy&PHISHY_NUMERIC_IP && !isNumericURL(pchk, urls->displayLink.data)) ||

 			  !(phishy&PHISHY_NUMERIC_IP))) {

 		cli_dbgmsg("Displayed 'url' is not url:%s\n",urls->displayLink.data);

-		if (!blacklisted)

-		    return CL_PHISH_CLEAN;

+		return CL_PHISH_CLEAN;

 	}

 	if(whitelist_check(engine, urls, 0))

 		return CL_PHISH_CLEAN;/* if url is whitelisted don't perform further checks */

-	if (blacklisted)

-	    return blacklisted;

-

-	if (urls->displayLink.data[0] == '\0') {

-	    return CL_PHISH_CLEAN;

-	}

-

 	url_check_init(&host_url);

 	if((rc = url_get_host(urls, &host_url, DOMAIN_DISPLAY, &phishy))) {

@@ -1585,6 +1585,13 @@ static int cli_loaddbdir(const char *dirname, struct cl_engine *engine, unsigned

 	return ret;

     }

+    /* try to load local.gdb next */

+    sprintf(dbfile, "%s/local.gdb", dirname);

+    if(!access(dbfile, R_OK) && (ret = cli_load(dbfile, engine, signo, options, NULL))) {

+	free(dbfile);

+	return ret;

+    }

+

     /* check for and load daily.cfg */

     sprintf(dbfile, "%s/daily.cfg", dirname);

     if(!access(dbfile, R_OK) && (ret = cli_load(dbfile, engine, signo, options, NULL))) {

@@ -432,18 +432,13 @@ static int add_hash(struct regex_matcher *matcher, char* pattern, const char fl,

 	int rc;

 	struct cli_bm_patt *pat = mpool_calloc(matcher->mempool, 1, sizeof(*pat));

 	struct cli_matcher *bm;

+	const char *vname = NULL;

 	if(!pat)

 		return CL_EMEM;

 	pat->pattern = (unsigned char*)cli_mpool_hex2str(matcher->mempool, pattern);

 	if(!pat->pattern)

 		return CL_EMALFDB;

 	pat->length = 32;

-	pat->virname = mpool_malloc(matcher->mempool, 1);

-	if(!pat->virname) {

-		free(pat);

-		return CL_EMEM;

-	}

-	*pat->virname = fl;

 	if (is_prefix) {

 	    pat->length=4;

 	    bm = &matcher->hostkey_prefix;

@@ -451,6 +446,23 @@ static int add_hash(struct regex_matcher *matcher, char* pattern, const char fl,

 	    bm = &matcher->sha256_hashes;

 	}

+	if (fl != 'W' && pat->length == 32 &&

+	    cli_bm_scanbuff(pat->pattern, 32, &vname, &matcher->sha256_hashes,0,0,-1) == CL_VIRUS) {

+	    if (*vname == 'W') {

+		/* hash is whitelisted in local.gdb */

+		cli_dbgmsg("Skipping hash %s\n", pattern);

+		mpool_free(matcher->mempool, pat->pattern);

+		mpool_free(matcher->mempool, pat);

+		return CL_SUCCESS;

+	    }

+	}

+	pat->virname = mpool_malloc(matcher->mempool, 1);

+	if(!pat->virname) {

+		free(pat);

+		return CL_EMEM;

+	}

+	*pat->virname = fl;

+

 	if((rc = cli_bm_addpatt(bm, pat))) {

 		cli_errmsg("add_hash: failed to add BM pattern\n");

 		free(pat->pattern);

@@ -550,9 +562,11 @@ int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int *sign

 			/*matches displayed host*/

 			if (( rc = add_static_pattern(matcher, pattern) ))

 				return rc==CL_EMEM ? CL_EMEM : CL_EMALFDB;

-		} else if (buffer[0] == 'S' && !is_whitelist) {

+		} else if (buffer[0] == 'S' && (!is_whitelist || pattern[0]=='W')) {

 			pattern[pattern_len] = '\0';

-			if((pattern[0]=='F' || pattern[0]=='P') && pattern[1]==':') {

+			if (pattern[0]=='W')

+			    flags[0]='W';

+			if((pattern[0]=='W' || pattern[0]=='F' || pattern[0]=='P') && pattern[1]==':') {

 			    pattern += 2;

 			    if (( rc = add_hash(matcher, pattern, flags[0], pattern[-2] == 'P') )) {

 				cli_errmsg("Error loading at line: %d\n", line);