@@ -1,3 +1,8 @@

+Sat Oct  7 12:47:32 CEST 2006 (tk)

+----------------------------------

+  * libclamav: make the experimental anti-phishing code more thread safe,

+	       patch from Edvin Torok <edwintorok*gmail.com>

+

 Thu Oct  5 22:46:19 CEST 2006 (tk)

 ----------------------------------

   * libclamav/sis.c: fix handling of compressed multiple language files (bb#42)

@@ -181,6 +181,10 @@ struct cl_engine {

     /* Hardware database handle */

     void *hwdb;

+

+    /* Phishing .pdb and .wdb databases*/

+    void *whitelist_matcher;

+    void *domainlist_matcher;

 };

 struct cl_limits {

@@ -19,6 +19,9 @@

  *  MA 02110-1301, USA.

  *

  *  $Log: phish_domaincheck_db.c,v $

+ *  Revision 1.3  2006/10/07 11:00:46  tkojm

+ *  make the experimental anti-phishing code more thread safe

+ *

  *  Revision 1.2  2006/09/26 18:55:36  njh

  *  Fixed portability issues

  *

@@ -91,13 +94,10 @@

 #include "regex_list.h"

 #include "matcher-ac.h"

-

-static struct regex_matcher domainlist_matcher;

-

-int domainlist_match(const char* real_url,const char* display_url,int hostOnly,unsigned short* flags)

+int domainlist_match(const struct cl_engine* engine,const char* real_url,const char* display_url,int hostOnly,unsigned short* flags)

 {

 	const char* info;

-	int rc = regex_list_match(&domainlist_matcher,real_url,display_url,hostOnly,&info);

+	int rc = engine->domainlist_matcher ? regex_list_match(engine->domainlist_matcher,real_url,display_url,hostOnly,&info) : 0;

 	if(rc && info && info[0]) {/*match successfull, and has custom flags*/

 		if(strlen(info)==3 && isxdigit(info[0]) && isxdigit(info[1]) && isxdigit(info[2])) {

 			unsigned short notwantedflags=0;

@@ -111,29 +111,38 @@ int domainlist_match(const char* real_url,const char* display_url,int hostOnly,u

 	return rc;

 }

-int init_domainlist(void)

+int init_domainlist(struct cl_engine* engine)

 {

-	return	init_regex_list(&domainlist_matcher);

+	if(engine) {

+		engine->domainlist_matcher = cli_malloc(sizeof(*engine->domainlist_matcher));

+		if(!engine->domainlist_matcher)

+			return CL_EMEM;

+		return init_regex_list(engine->domainlist_matcher);

 }

-

-int is_domainlist_ok(void)

-{

-	return is_regex_ok(&domainlist_matcher);

+	else

+		return CL_ENULLARG;

 }

-int cli_loadpdb(FILE* fd,unsigned int options)

+int is_domainlist_ok(const struct cl_engine* engine)

 {

-	return load_regex_matcher(&domainlist_matcher,fd,options);

+	return (engine && engine->domainlist_matcher) ? is_regex_ok(engine->domainlist_matcher) : 1;

 }

-void domainlist_cleanup(void)

+

+void domainlist_cleanup(const struct cl_engine* engine)

 {

-	regex_list_cleanup(&domainlist_matcher);

+	if(engine && engine->domainlist_matcher) {

+		regex_list_cleanup(engine->domainlist_matcher);

+	}

 }

-void domainlist_done(void)

+void domainlist_done(struct cl_engine* engine)

 {

-	regex_list_done(&domainlist_matcher);

+	if(engine && engine->domainlist_matcher) {

+		regex_list_done(engine->domainlist_matcher);

+		free(engine->domainlist_matcher);

+		engine->domainlist_matcher = NULL;

+	}

 }

 #endif

@@ -25,13 +25,11 @@

 #ifndef _PHISH_DOMAINCHECK_DB_H

 #define _PHISH_DOMAINCHECK_DB_H

-int cli_loadpdb(FILE* fd, unsigned int options);

-int build_domainlist(void);

-int init_domainlist(void);

-void domainlist_done(void);

-void domainlist_cleanup(void);

-int is_domainlist_ok(void);

-int domainlist_match(const char* real_url,const char* display_url,int hostOnly,unsigned short* flags);

+int init_domainlist(struct cl_engine* engine);

+void domainlist_done(struct cl_engine* engine);

+void domainlist_cleanup(const struct cl_engine* engine);

+int is_domainlist_ok(const struct cl_engine* engine);

+int domainlist_match(const struct cl_engine* engine,const char* real_url,const char* display_url,int hostOnly,unsigned short* flags);

 #endif

@@ -19,6 +19,9 @@

  *  MA 02110-1301, USA.

  *

  *  $Log: phish_whitelist.c,v $

+ *  Revision 1.4  2006/10/07 11:00:46  tkojm

+ *  make the experimental anti-phishing code more thread safe

+ *

  *  Revision 1.3  2006/09/26 18:55:36  njh

  *  Fixed portability issues

  *

@@ -93,78 +96,44 @@

 #include "regex_list.h"

 #include "matcher-ac.h"

-/*#define WHITELIST_TEST*/

-

-static struct regex_matcher whitelist_matcher;

-

-int whitelist_match(const char* real_url,const char* display_url,int hostOnly)

+int whitelist_match(const struct cl_engine* engine,const char* real_url,const char* display_url,int hostOnly)

 {

 	const char* info;/*unused*/

-	return	regex_list_match(&whitelist_matcher,real_url,display_url,hostOnly,&info);

+	return	engine->whitelist_matcher ? regex_list_match(engine->whitelist_matcher,real_url,display_url,hostOnly,&info) : 0;

 }

-int init_whitelist(void)

+int init_whitelist(struct cl_engine* engine)

 {

-	return	init_regex_list(&whitelist_matcher);

+	if(engine) {

+		engine->whitelist_matcher = cli_malloc(sizeof(*engine->whitelist_matcher));

+		if(!engine->whitelist_matcher)

+			return CL_EMEM;

+		return	init_regex_list(engine->whitelist_matcher);

 }

-

-int is_whitelist_ok(void)

-{

-	return is_regex_ok(&whitelist_matcher);

+	else

+		return CL_ENULLARG;

 }

-int cli_loadwdb(FILE* fd,unsigned int options)

+int is_whitelist_ok(const struct cl_engine* engine)

 {

-	return load_regex_matcher(&whitelist_matcher,fd,options);

+	return (engine && engine->whitelist_matcher) ? is_regex_ok(engine->whitelist_matcher) : 1;

 }

-void whitelist_cleanup(void)

-{

-	regex_list_cleanup(&whitelist_matcher);

-}

-void whitelist_done(void)

+void whitelist_cleanup(const struct cl_engine* engine)

 {

-	regex_list_done(&whitelist_matcher);

+	if(engine && engine->whitelist_matcher) {

+		regex_list_cleanup(engine->whitelist_matcher);

+	}

 }

-#ifdef WHITELIST_TEST

-int main(int argc,char* argv[])

+void whitelist_done(struct cl_engine* engine)

 {

-/*	struct tree_node* root=tree_node_alloc(NULL,1);

-	const  char* info;

-	const  unsigned char test[]="tesxt";

-	setup_matcher();

-	root->op=OP_ROOT;

-	root->c=0;

-	root->next=NULL;

-	root->listend=1;

-	dump_tree(root);

-	add_pattern(&root,"test","1");

-	dump_tree(root);

-	add_pattern(&root,"tesv","2");

-	dump_tree(root);

-	add_pattern(&root,"tert","3");

-	dump_tree(root);

-	add_pattern(&root,"terr+","4");

-	dump_tree(root);

-	add_pattern(&root,"tes[xy]t","5");

-	dump_tree(root);

-	match_node(root,test,sizeof(test),&info);

-	destroy_tree(root);

-	if(info)

-		printf("%s\n",info);

-	else printf("not found\n");*/

-	/*FILE* f=fopen("w.wdb","r");

-	init_whitelist();

-	load_whitelist(f);

-	fclose(f);

-	dump_tree(root_regex);

-	build_whitelist();

-	printf("%d\n",whitelist_match("http://www.google.ro","http://www.google.me.ro",0));

-	whitelist_done();*/

-	return 0;

+	if(engine && engine->whitelist_matcher) {

+		regex_list_done(engine->whitelist_matcher);	

+		free(engine->whitelist_matcher);

+		engine->whitelist_matcher = NULL;

+	}

 }

-#endif

 #endif

@@ -25,13 +25,11 @@

 #ifndef _PHISH_WHITELIST_H

 #define _PHISH_WHITELIST_H

-int cli_loadwdb(FILE* fd, unsigned int options);

-int build_whitelist(void);

-int init_whitelist(void);

-void whitelist_done(void);

-void whitelist_cleanup(void);

-int is_whitelist_ok(void);

-int whitelist_match(const char* real_url,const char* display_url,int hostOnly);

+int init_whitelist(struct cl_engine* engine);

+void whitelist_done(struct cl_engine* engine);

+void whitelist_cleanup(const struct cl_engine* engine);

+int is_whitelist_ok(const struct cl_engine* engine);

+int whitelist_match(const struct cl_engine* engine, const char* real_url,const char* display_url,int hostOnly);

 #endif

@@ -19,6 +19,9 @@

  *  MA 02110-1301, USA.

  *

  *  $Log: phishcheck.c,v $

+ *  Revision 1.11  2006/10/07 11:00:46  tkojm

+ *  make the experimental anti-phishing code more thread safe

+ *

  *  Revision 1.10  2006/09/27 14:23:14  njh

  *  Ported to VS2005

  *

@@ -147,6 +150,8 @@ case CL_PHISH_HOST_NOT_LISTED:

 #include <regex.h>

 #endif

+#include <pthread.h>

+

 #include "others.h"

 #include "defaults.h"

 #include "str.h"

@@ -247,6 +252,7 @@ For the Whitelist(.wdb)/Domainlist(.pdb) format see regex_list.c (search for Fla

  *

  */

 static char empty_string[]="";

+

 static	inline	void string_init_c(struct string* dest,char* data);

 static	void	string_assign_null(struct string* dest);

 static	char	*rfind(char *start, char c, size_t len);

@@ -347,7 +353,6 @@ void free_if_needed(struct url_check* url)

 	string_free(&url->displayLink);

 }

-static int phish_disabled = 0;/* disabled due to fatal startup error */

 static int build_regex(regex_t** preg,const char* regex,int nosub)

 {

@@ -372,7 +377,7 @@ static int build_regex(regex_t** preg,const char* regex,int nosub)

 #endif

 		free(*preg);

 		*preg=NULL;

-		phish_disabled=1;

+		phish_disable("problem in compiling regex");

 		return 1;

 	}

 	return 0;

@@ -468,10 +473,6 @@ static const char cctld_regex[] = "^"iana_cctld"$";

 int isCountryCode(const char* str)

 {

-	if(!preg_cctld) {

-		if(build_regex(&preg_cctld,cctld_regex,1))

-			return -1;

-	}

 	return str ? !regexec(preg_cctld,str,0,NULL,0) : 0;

 }

@@ -484,10 +485,6 @@ int isTLD(const char* str,int len)

 		int rc;

 		strncpy(s,str,len);

 		s[len]='\0';

-		if(!preg_tld) {

-			if(build_regex(&preg_tld,tld_regex,1))

-				return -1;

-		}

 		rc = !regexec(preg_tld,s,0,NULL,0);

 		free(s);

 		return rc;

@@ -599,15 +596,10 @@ int isSSL(const char* URL)

 	return URL ? !strncmp(https,URL,sizeof(https)-1) : 0;

 }

-static int hexinited=0;

-static short int hextable[256];

-static inline char hex2int(const unsigned char* src)

-{

-	assert(hexinited);

-	return hextable[src[0]]<<4 | hextable[src[1]];

-}

+static inline char hex2int(const unsigned char* src);

+

 /* deletes @what from the string @begin.

  * @what_len: length of @what, excluding the terminating \0 */

 static void

@@ -828,29 +820,32 @@ void get_redirected_URL(struct string* URL)

 	returns redirected URL*/

 }

-static inline int is_phish_disabled(void)

+

+/* ---- runtime disable ------*/

+static int phish_disabled = 0;

+static pthread_mutex_t phish_disabled_lock = PTHREAD_MUTEX_INITIALIZER;

+

+void phish_disable(const char* reason)

 {

-	if (phish_disabled)

-		return 1;

-	else if (!is_whitelist_ok()) {

+	cli_warnmsg("Disabling phishing checks, reason:%s\n",reason);

+	pthread_mutex_lock(&phish_disabled_lock);

 		phish_disabled = 1;

-		return 1;

-	}

-	else return 0;

+	pthread_mutex_unlock(&phish_disabled_lock);

 }

-static void init_hextable(void)

+static inline int is_phish_disabled(const struct cl_engine* engine)

 {

-	unsigned char c;

-	memset(hextable,0,256);

-	for(c='0';c<='9';c++)

-		hextable[c] = c-'0';

-	for(c='a';c<='z';c++)

-		hextable[c] = 10+c-'a';

-	for(c='A';c<='Z';c++)

-		hextable[c] = 10+c-'A';

-	hexinited=1;

+	int rc;

+	if (!is_whitelist_ok(engine)) 

+		phish_disable("whitelist is not ok");

+	if (!is_domainlist_ok(engine))

+		phish_disable("domainlist is not ok");

+	pthread_mutex_lock(&phish_disabled_lock);

+	rc = phish_disabled;

+	pthread_mutex_unlock(&phish_disabled_lock);

+	return rc;

 }

+/* -------end runtime disable---------*/

 int phishingScan(message* m,const char* dir,cli_ctx* ctx,tag_arguments_t* hrefs)

 {

@@ -859,12 +854,8 @@ int phishingScan(message* m,const char* dir,cli_ctx* ctx,tag_arguments_t* hrefs)

 	const size_t href_text_len = sizeof(href_text);

 	const size_t src_text_len = sizeof(src_text);

 	int i;

-	if(is_phish_disabled())

+	if(is_phish_disabled(ctx->engine))

 		return 0;

-	if(!hexinited) {

-		init_hextable();

-		atexit(phishing_done);/*TODO: replace this with a proper phishing_done call from manager.c*/

-	}

 	*ctx->virname=NULL;

 	for(i=0;i<hrefs->count;i++)

@@ -893,8 +884,8 @@ int phishingScan(message* m,const char* dir,cli_ctx* ctx,tag_arguments_t* hrefs)

 				urls.displayLink.data = url;

 			}

-			rc = phishingCheck(&urls);

-			if(phish_disabled)

+			rc = phishingCheck(ctx->engine,&urls);

+			if(is_phish_disabled(ctx->engine))

 				return 0;

 			free_if_needed(&urls);

 			cli_dbgmsg("Phishing scan result:%s\n",phishing_ret_toString(rc));

@@ -1015,28 +1006,89 @@ static char* str_compose(const char* a,const char* b,const char* c)

 /*Warning: take care when modifying this regex, it has been tweaked, and tuned, just don't break it please.

  * there is fragmentaddress1, and 2  to work around the ISO limitation of 509 bytes max length for string constants*/

-static char* url_regex = NULL;

 static const char numeric_url_regex[] = "^ *"URI_numeric_fragmentaddress" *$";

+static char* url_regex = NULL;

+

+static int hexinited=0;

+static short int hextable[256];

+

+static inline char hex2int(const unsigned char* src)

+{

+	assert(hexinited);

+	return hextable[src[0]]<<4 | hextable[src[1]];

+}

+

+static void free_regex(regex_t** p)

+{

+	if(p) {

+		if(*p) {

+			regfree(*p);

+			free(*p);

+			*p=NULL;

+		}

+	}

+}

+/* --------non-thread-safe functions--------*/

+static void init_hextable(void)

+{

+	unsigned char c;

+	memset(hextable,0,256);

+	for(c='0';c<='9';c++)

+		hextable[c] = c-'0';

+	for(c='a';c<='z';c++)

+		hextable[c] = 10+c-'a';

+	for(c='A';c<='Z';c++)

+		hextable[c] = 10+c-'A';

+	hexinited=1;

+}

+

+int phishing_init(engine)

+{

+	cli_dbgmsg("Initializing phishcheck module\n");

+	setup_matcher_engine();

+	if(build_regex(&preg_cctld,cctld_regex,1))

+		return -1;

+	if(build_regex(&preg_tld,tld_regex,1))

+		return -1;	

+	url_regex = str_compose("^ *("URI_fragmentaddress1,URI_fragmentaddress2,URI_fragmentaddress3"|"URI_CHECK_PROTOCOLS") *$");

+	if(build_regex(&preg,url_regex,1))

+		return -1;

+	if(build_regex(&preg_numeric,numeric_url_regex,1))

+		return -1;

+	init_hextable();

+	cli_dbgmsg("Phishcheck module initialized\n");

+	return 0;

+}

+

+

+void phishing_done(struct cl_engine* engine)

+{

+	cli_dbgmsg("Cleaning up phishcheck\n");

+	free_regex(&preg);

+	free_regex(&preg_cctld);

+	free_regex(&preg_tld);

+	free_regex(&preg_numeric);

+	if(url_regex)

+		free(url_regex);

+

+	whitelist_done(engine);

+	domainlist_done(engine);

+	matcher_engine_done();

+	cli_dbgmsg("Phishcheck cleaned up\n");

+}

+

+/* ---------------end of non-thread-safe function-----------*/

 /*

  * Only those URLs are identified as URLs for which phishing detection can be performed.

  * This means that no attempt is made to properly recognize 'cid:' URLs

  */

 int isURL(const char* URL)

 {

-	if(!preg) {

-		url_regex = str_compose("^ *("URI_fragmentaddress1,URI_fragmentaddress2,URI_fragmentaddress3"|"URI_CHECK_PROTOCOLS") *$");

-		if(build_regex(&preg,url_regex,1))

-			return -1;

-	}

 	return URL ? !regexec(preg,URL,0,NULL,0) : 0;

 }

 int isNumericURL(const char* URL)

 {

-	if(!preg_numeric) {

-		if(build_regex(&preg_numeric,numeric_url_regex,1))

-			return -1;

-	}

 	return URL ? !regexec(preg_numeric,URL,0,NULL,0) : 0;

 }

@@ -1115,36 +1167,15 @@ int isEncoded(const char* url)

 	return (cnt-1 >strlen(url)*7/10);/*more than 70% made up of &#;*/

 }

-static void free_regex(regex_t** p)

-{

-	if(p) {

-		if(*p) {

-			regfree(*p);

-			free(*p);

-			*p=NULL;

-		}

-	}

-}

-void phishing_done(void)

-{

-	free_regex(&preg);

-	free_regex(&preg_cctld);

-	free_regex(&preg_tld);

-	free_regex(&preg_numeric);

-	whitelist_done();

-	domainlist_done();

-	if(url_regex)

-		free(url_regex);

-}

-int whitelist_check(struct url_check* urls,int hostOnly)

+int whitelist_check(const struct cl_engine* engine,struct url_check* urls,int hostOnly)

 {

-	return whitelist_match(urls->realLink.data,urls->displayLink.data,hostOnly);

+	return whitelist_match(engine,urls->realLink.data,urls->displayLink.data,hostOnly);

 }

 /* urls can't contain null pointer, caller must ensure this */

-enum phish_status phishingCheck(struct url_check* urls)

+enum phish_status phishingCheck(const struct cl_engine* engine,struct url_check* urls)

 {

 	struct url_check host_url;

 	const char cid[] = "cid:";

@@ -1166,10 +1197,10 @@ enum phish_status phishingCheck(struct url_check* urls)

 		return rc;/* URLs identical after cleanup */

 	}

-	if(whitelist_check(urls,0))

+	if(whitelist_check(engine,urls,0))

 		return CL_PHISH_WHITELISTED;/* if url is whitelist don't perform further checks */

-	if(urls->flags&DOMAINLIST_REQUIRED && domainlist_match(urls->realLink.data,urls->displayLink.data,0,&urls->flags))

+	if(urls->flags&DOMAINLIST_REQUIRED && domainlist_match(engine,urls->realLink.data,urls->displayLink.data,0,&urls->flags))

 		phishy |= DOMAIN_LISTED;

 	else {

 		/* although entire url is not listed, the host might be,

@@ -1184,14 +1215,14 @@ enum phish_status phishingCheck(struct url_check* urls)

 		return rc;

 	}

-	if(whitelist_check(&host_url,1)) {

+	if(whitelist_check(engine,&host_url,1)) {

 		free_if_needed(&host_url);

 		return CL_PHISH_HOST_WHITELISTED;

 	}

 	if(urls->flags&DOMAINLIST_REQUIRED) {

 		if(!(phishy&DOMAIN_LISTED)) {

-			if(domainlist_match(urls->displayLink.data,urls->realLink.data,1,&urls->flags))

+			if(domainlist_match(engine,urls->displayLink.data,urls->realLink.data,1,&urls->flags))

 				phishy |= DOMAIN_LISTED;

 			else {

 				free_if_needed(&host_url);

@@ -61,9 +61,9 @@ struct url_check {

 };

 int phishingScan(message* m,const char* dir,cli_ctx* ctx,tag_arguments_t* hrefs);

-enum phish_status phishingCheck(struct url_check* urls);

+enum phish_status phishingCheck(const struct cl_engine* engine,struct url_check* urls);

-int whitelist_check(struct url_check* urls,int hostOnly);

+int whitelist_check(const struct cl_engine* engine,struct url_check* urls,int hostOnly);

 void url_check_init(struct url_check* urls);

 void get_host(struct string* dest,const char* URL,int isReal,int* phishy);

 void string_free(struct string* str);

@@ -89,7 +89,13 @@ enum phish_status url_get_host(struct url_check* url,struct url_check* host_url,

 void url_get_domain(struct url_check* url,struct url_check* domains);

 enum phish_status phishy_map(int phishy,enum phish_status fallback);

 int isEncoded(const char* url);

-void phishing_done(void);

+

+void phish_disable(const char* reason);

+/* Global, non-thread-safe functions, call only once! */

+void phishint_init(struct cl_engine* engine);

+void phishing_done(struct cl_engine* engine);

+/* end of non-thread-safe functions */

+

 static inline int isPhishing(enum phish_status rc)

 {

@@ -44,6 +44,7 @@

 #ifdef CL_EXPERIMENTAL

 #include "phish_whitelist.h"

 #include "phish_domaincheck_db.h"

+#include "regex_list.h"

 #endif

@@ -469,6 +470,7 @@ int cli_parse_add(struct cli_matcher *root, const char *virname, const char *hex

 static int cli_initengine(struct cl_engine **engine, unsigned int options)

 {

+    int rc;

     if(!*engine) {

 	cli_dbgmsg("Initializing the engine structure\n");

@@ -487,6 +489,10 @@ static int cli_initengine(struct cl_engine **engine, unsigned int options)

 	    cli_errmsg("Can't allocate memory for roots!\n");

 	    return CL_EMEM;

 	}

+#ifdef CL_EXPERIMENTAL

+	if(rc =phishing_init(*engine))

+		return rc;

+#endif

     }

     return CL_SUCCESS;

@@ -593,6 +599,44 @@ static int cli_loaddb(FILE *fd, struct cl_engine **engine, unsigned int *signo,

     return CL_SUCCESS;

 }

+#ifdef CL_EXPERIMENTAL

+static int cli_loadwdb(struct cl_engine** engine,FILE* fd,unsigned int options)

+{

+	int ret = 0;

+

+	if((ret = cli_initengine(engine, options))) {

+		cl_free(*engine);

+		return ret;

+	}

+

+	if(!(*engine)->whitelist_matcher)

+		if(ret = init_whitelist(*engine)) {

+	            whitelist_done(*engine);

+		    cl_free(*engine);

+		    return ret;

+		}

+

+	return load_regex_matcher((*engine)->whitelist_matcher,fd,options);

+}

+

+static int cli_loadpdb(struct cl_engine** engine,FILE* fd,unsigned int options)

+{

+	int ret = 0;

+

+	if((ret = cli_initengine(engine, options))) {

+		cl_free(*engine);

+		return ret;

+	}

+

+	if(!(*engine)->domainlist_matcher)

+		if(ret = init_domainlist(*engine)) {

+	            domainlist_done(*engine);

+		    cl_free(*engine);

+		    return ret;

+		}

+	return load_regex_matcher((*engine)->domainlist_matcher,fd,options);

+}

+#endif

 static int cli_loadndb(FILE *fd, struct cl_engine **engine, unsigned int *signo, unsigned short sdb, unsigned int options)

 {

 	char buffer[FILEBUFF], *sig, *virname, *offset, *pt;

@@ -1118,12 +1162,12 @@ static int cli_load(const char *filename, struct cl_engine **engine, unsigned in

 #ifdef CL_EXPERIMENTAL

     } else if(cli_strbcasestr(filename, ".wdb")) {

 	if(!(options & CL_SCAN_NOPHISHING))

-	    ret = cli_loadwdb(fd, options);

+	    ret = cli_loadwdb(engine, fd, options);

 	else

 	    skipped = 1;

     } else if(cli_strbcasestr(filename, ".pdb")) {

 	if(!(options & CL_SCAN_NOPHISHING))

-	    ret = cli_loadpdb(fd, options);

+	    ret = cli_loadpdb(engine, fd, options);

 	else

 	    skipped = 1;

 #endif

@@ -1548,6 +1592,9 @@ void cl_free(struct cl_engine *engine)

 	free(metah);

     }

+#ifdef CL_EXPERIMENTAL

+   phishing_done(engine);

+#endif

     free(engine);

 }

@@ -19,6 +19,9 @@

  *  MA 02110-1301, USA.

  *

  *  $Log: regex_list.c,v $

+ *  Revision 1.7  2006/10/07 11:00:46  tkojm

+ *  make the experimental anti-phishing code more thread safe

+ *

  *  Revision 1.6  2006/09/27 19:14:49  njh

  *  Fix segfault on Solaris

  *

@@ -190,15 +193,47 @@ static struct std_classmap {

 	{"[:xdigit:]",XDIGIT}

 };

+static int cli_iswctype(const char c,const enum wctype_t type);

+

+/* -------------- NON_THREAD_SAFE BEGIN --------------*/

+/* Global variables and functions accessing them, not thread-safe!

+ * they should be called on application startup/shutdown once! */

 static const size_t std_class_cnt =  sizeof(std_class)/sizeof(std_class[0]);

 #define STD_CLASS_CNT sizeof(std_class)/sizeof(std_class[0])

 typedef char char_bitmap_t[32];

 static char_bitmap_p char_class_bitmap[STD_CLASS_CNT];

 static unsigned short int char_class[256];

+static int engine_ok = 0;

+

+void setup_matcher_engine(void)

+{

+	/*Set up std character classes*/

+	size_t i;

+	size_t j;

+	memset(char_class,0,256);

+	for(i=0;i<std_class_cnt;i++) {

+		enum wctype_t type = std_class[i].type;

+		char_class_bitmap[i]=cli_calloc(256>>3,1);

+		for(j=0;j<256;j++)

+			if(cli_iswctype(j,type)) {

+				char_class[j] |= 1<<i;

+				char_class_bitmap[i][j>>3] |= 1<<(j&0x07);

+			}

+	}	

+	engine_ok  = 1;

+}

+

+void matcher_engine_done(void)

+{

+	size_t i;

+	for(i=0;i<std_class_cnt;i++)

+			free(char_class_bitmap[i]);

+	engine_ok = 0;

+}

+

+/* -------------- NON_THREAD_SAFE END --------------*/

 /* Prototypes */

-static void setup_matcher_engine(void);

-static void matcher_engine_done(void);

 static int add_pattern(struct regex_matcher* matcher,const unsigned char* pat,const char* info);

 static int match_node(struct tree_node* node,const unsigned char* c,size_t len,const char** info);

 static void destroy_tree(struct regex_matcher* matcher);

@@ -236,6 +271,7 @@ int regex_list_match(struct regex_matcher* matcher,const char* real_url,const ch

 	assert(real_url);

 	assert(display_url);

 	assert(info);

+	assert(engine_ok);

 	if(!matcher->list_inited)

 		return 0;

 	assert(matcher->list_built);

@@ -328,9 +364,6 @@ static inline struct tree_node* stack_pop(struct node_stack* stack)

 int init_regex_list(struct regex_matcher* matcher)

 {

 	assert(matcher);

-	

-	setup_matcher_engine();

-

 	matcher->list_inited = 0;

 	matcher->root_hosts = (struct cli_matcher*) cli_calloc(1,sizeof(*matcher->root_hosts));

 	if(!matcher->root_hosts)

@@ -424,7 +457,6 @@ static int add_regex_list_element(struct cli_matcher* root,const char* pattern,c

 #ifndef NDEBUG

 void dump_tree(struct tree_node* root);

 #endif

-static int matcher_engine_refcount=0;

 static int build_regex_list(struct regex_matcher* matcher);

 /* Load patterns/regexes from file */

@@ -511,7 +543,6 @@ int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int optio

 		return CL_EMALFDB;

 	}

 	regex_list_cleanup(matcher);

-	matcher_engine_refcount++;

 	return CL_SUCCESS;

 }

@@ -665,7 +696,6 @@ void regex_list_done(struct regex_matcher* matcher)

 		matcher->list_loaded=0;

 	}

 	if(matcher->list_inited) {

-		matcher_engine_done();

 		matcher->list_inited=0;

 	}

 	stack_destroy(&matcher->node_stack);

@@ -708,38 +738,7 @@ static int cli_iswctype(const char c,const enum wctype_t type)

 	}

 }

-static int engine_inited=0;

-

-static void setup_matcher_engine(void)

-{

-	/*Set up std character classes*/

-	size_t i;

-	size_t j;

-	if(engine_inited)

-		return;

-	memset(char_class,0,256);

-	for(i=0;i<std_class_cnt;i++) {

-		enum wctype_t type = std_class[i].type;

-		char_class_bitmap[i]=cli_calloc(256>>3,1);

-		for(j=0;j<256;j++)

-			if(cli_iswctype(j,type)) {

-				char_class[j] |= 1<<i;

-				char_class_bitmap[i][j>>3] |= 1<<(j&0x07);

-			}

-	}	

-	engine_inited=1;

-}

-static void matcher_engine_done(void)

-{

-	size_t i;

-	matcher_engine_refcount--;

-	if(!matcher_engine_refcount) {

-		for(i=0;i<std_class_cnt;i++)

-			free(char_class_bitmap[i]);

-	}

-	engine_inited=0;

-}

 struct token_t

 {

@@ -48,6 +48,8 @@ int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int optio

 void regex_list_cleanup(struct regex_matcher* matcher);

 void regex_list_done(struct regex_matcher* matcher);

 int is_regex_ok(struct regex_matcher* matcher);

+

+void setup_matcher_engine(void);/* global, non thread-safe */

 #endif

 #endif