@@ -1,3 +1,8 @@

+Tue Jul 29 13:18:24 EEST 2008 (edwin)

+------------------------------------

+  * libclamav/regex_*.[ch]: handle multiple matches (bb #1110)

+  * unit_tests: update tests for regex, reenable test 

+

 Tue Jul 29 10:47:23 CEST 2008 (tk)

 ----------------------------------

   * libclamav/matcher-ac.c: add support for returning multiple matches in

@@ -58,6 +58,7 @@

 #include "jsparse/textbuf.h"

 #include "regex_suffix.h"

 /* Prototypes */

+static regex_t *new_preg(struct regex_matcher *matcher);

 static size_t reverse_string(char *pattern);

 static int add_pattern_suffix(void *cbdata, const char *suffix, size_t suffix_len, struct regex_list *regex);

 static int add_static_pattern(struct regex_matcher *matcher, char* pattern);

@@ -240,6 +241,7 @@ int regex_list_match(struct regex_matcher* matcher,char* real_url,const char* di

 {

 	char* orig_real_url = real_url;

 	struct regex_list *regex;

+	struct regex_list *last_match;

 	assert(matcher);

 	assert(real_url);

@@ -259,6 +261,7 @@ int regex_list_match(struct regex_matcher* matcher,char* real_url,const char* di

 		char *bufrev;

 		int rc = 0;

 		struct cli_ac_data mdata;

+		struct cli_ac_result *res = NULL;

 		if(!buffer)

 			return CL_EMEM;

@@ -281,30 +284,37 @@ int regex_list_match(struct regex_matcher* matcher,char* real_url,const char* di

 		reverse_string(bufrev);

 		rc = SO_search(&matcher->filter, (const unsigned char*)bufrev, buffer_len) != -1;

 		if(!rc) {

+			free(buffer);

+			free(bufrev);

 			/* filter says this suffix doesn't match.

 			 * The filter has false positives, but no false

 			 * negatives */

 			return 0;

 		}

-		rc = cli_ac_scanbuff((const unsigned char*)bufrev,buffer_len, NULL, &regex, NULL, &matcher->suffixes,&mdata,0,0,-1,NULL,AC_SCAN_VIR,NULL);

+		rc = cli_ac_scanbuff((const unsigned char*)bufrev,buffer_len, NULL, (void*)&regex, &res, &matcher->suffixes,&mdata,0,0,-1,NULL,AC_SCAN_VIR,NULL);

 		free(bufrev);

 		cli_ac_freedata(&mdata);

-		if(rc) {

-			/* TODO loop over multiple virusnames here */

-			do {

+		rc = 0;

+		while(res) {

+			struct cli_ac_result *q;

+			regex = res->customdata;

+			while(!rc && regex) {

 				/* loop over multiple regexes corresponding to

 				 * this suffix */

-				if (!regex->preg.re_magic) {

+				if (!regex->preg) {

 					/* we matched a static pattern */

 					rc = validate_subdomain(regex, pre_fixup, buffer, buffer_len, real_url, real_len, orig_real_url);

 				} else {

-					rc = !cli_regexec(&regex->preg, buffer, 0, NULL, 0);

+					rc = !cli_regexec(regex->preg, buffer, 0, NULL, 0);

 				}

 				if(rc) *info = regex->pattern;

 				regex = regex->nxt;

-			 } while(!rc && regex);

+			}

+			q = res;

+			res = res->next;

+			free(q);

 		}

 		free(buffer);

 		if(!rc)

@@ -510,20 +520,22 @@ void regex_list_done(struct regex_matcher* matcher)

 			for(i=0;i<matcher->suffix_cnt;i++) {

 				struct regex_list *r = matcher->suffix_regexes[i];

 				while(r) {

-					cli_regfree(&r->preg);

+					struct regex_list *q = r;

 					r = r->nxt;

+					free(q->pattern);

+					free(q);

 				}

 			}

 			free(matcher->suffix_regexes);

 			matcher->suffix_regexes = NULL;

 		}

-		if(matcher->all_regexes) {

+		if(matcher->all_pregs) {

 			for(i=0;i<matcher->regex_cnt;i++) {

-				struct regex_list *r = matcher->all_regexes[i];

-				free(r->pattern);

+				regex_t *r = matcher->all_pregs[i];

+				cli_regfree(r);

 				free(r);

 			}

-			free(matcher->all_regexes);

+			free(matcher->all_pregs);

 		}

 		hashtab_free(&matcher->suffix_hash);

 		matcher->list_built=0;

@@ -589,12 +601,18 @@ static int add_newsuffix(struct regex_matcher *matcher, struct regex_list *info,

 /* ------ load a regex, determine suffix, determine suffix2regexlist map ---- */

 /* returns 0 on success, clamav error code otherwise */

-static int add_pattern_suffix(void *cbdata, const char *suffix, size_t suffix_len, struct regex_list *regex)

+static int add_pattern_suffix(void *cbdata, const char *suffix, size_t suffix_len, struct regex_list *iregex)

 {

 	struct regex_matcher *matcher = cbdata;

+	struct regex_list *regex = cli_malloc(sizeof(*regex));

 	const struct element *el;

 	assert(matcher);

+	if(!regex)

+		return CL_EMEM;

+	regex->pattern = iregex->pattern ? cli_strdup(iregex->pattern) : NULL;

+	regex->preg = iregex->preg;

+	regex->nxt = NULL;

 	el = hashtab_find(&matcher->suffix_hash, suffix, suffix_len);

 	/* TODO: what if suffixes are prefixes of eachother and only one will

 	 * match? */

@@ -630,45 +648,43 @@ static size_t reverse_string(char *pattern)

 	return len;

 }

-static struct regex_list *new_regex(struct regex_matcher *matcher)

+static regex_t *new_preg(struct regex_matcher *matcher)

 {

-	struct regex_list *r;

-	matcher->all_regexes = cli_realloc(matcher->all_regexes, ++matcher->regex_cnt * sizeof(*matcher->all_regexes));

-	if(!matcher->all_regexes)

+	regex_t *r;

+	matcher->all_pregs = cli_realloc(matcher->all_pregs, ++matcher->regex_cnt * sizeof(*matcher->all_pregs));

+	if(!matcher->all_pregs)

 		return NULL;

 	r = cli_malloc(sizeof(*r));

 	if(!r)

 		return NULL;

-	matcher->all_regexes[matcher->regex_cnt-1] = r;

+	matcher->all_pregs[matcher->regex_cnt-1] = r;

 	return r;

 }

 static int add_static_pattern(struct regex_matcher *matcher, char* pattern)

 {

 	size_t len;

-	struct regex_list *regex = new_regex(matcher);

-	if(!regex)

-		return CL_EMEM;

+	struct regex_list regex;

+	int rc;

+

 	len = reverse_string(pattern);

-	regex->nxt = NULL;

-	regex->pattern = cli_strdup(pattern);

-	regex->preg.re_magic = 0;

-	return add_pattern_suffix(matcher, pattern, len, regex);

+	regex.nxt = NULL;

+	regex.pattern = cli_strdup(pattern);

+	regex.preg = NULL;

+	rc = add_pattern_suffix(matcher, pattern, len, &regex);

+	free(regex.pattern);

+	return rc;

 }

 int regex_list_add_pattern(struct regex_matcher *matcher, char *pattern)

 {

 	int rc;

-	struct regex_list *regex = new_regex(matcher);

+	struct regex_list regex;

 	size_t len;

 	/* we only match the host, so remove useless stuff */

 	const char remove_end[] = "([/?].*)?/";

 	const char remove_end2[] = "([/?].*)/";

-

-	if(!regex)

-		return CL_EMEM;

-

 	len = strlen(pattern);

 	if(len > sizeof(remove_end)) {

 		if(strncmp(&pattern[len - sizeof(remove_end)+1], remove_end, sizeof(remove_end)-1) == 0) {

@@ -681,12 +697,15 @@ int regex_list_add_pattern(struct regex_matcher *matcher, char *pattern)

 		}

 	}

 	pattern[len] = '\0';

-	regex->pattern = NULL;

+	regex.pattern = NULL;

+

+	regex.preg = new_preg(matcher);

+	if(!regex.preg)

+		return CL_EMEM;

-	rc = cli_regex2suffix(pattern, regex, add_pattern_suffix, matcher);

+	rc = cli_regex2suffix(pattern, &regex, add_pattern_suffix, matcher);

 	if(rc) {

-		cli_regfree(&regex->preg);

-		free(regex);

+		cli_regfree(regex.preg);

 	}

 	return rc;

@@ -41,7 +41,7 @@ struct regex_matcher {

 	size_t suffix_cnt;

 	struct regex_list **suffix_regexes;

 	size_t regex_cnt;

-	struct regex_list **all_regexes;

+	regex_t **all_pregs;

 	struct cli_matcher suffixes;

 	struct filter filter;

 	int list_inited:2;

@@ -415,12 +415,12 @@ int cli_regex2suffix(const char *pattern, struct regex_list *regex, suffix_callb

 	assert(regex && pattern);

-	rc = cli_regcomp(&regex->preg, pattern, REG_EXTENDED);

+	rc = cli_regcomp(regex->preg, pattern, REG_EXTENDED);

 	if(rc) {

-		size_t buflen = cli_regerror(rc, &regex->preg, NULL, 0);

+		size_t buflen = cli_regerror(rc, regex->preg, NULL, 0);

 		char *errbuf = cli_malloc(buflen);

 		if(errbuf) {

-			cli_regerror(rc, &regex->preg, errbuf, buflen);

+			cli_regerror(rc, regex->preg, errbuf, buflen);

 			cli_errmsg(MODULE "Error compiling regular expression %s: %s\n", pattern, errbuf);

 			free(errbuf);

 		} else {

@@ -428,10 +428,8 @@ int cli_regex2suffix(const char *pattern, struct regex_list *regex, suffix_callb

 		}

 		return rc;

 	}

-#ifdef CL_DEBUG

-	regex->pattern = cli_strdup(pattern);

-#endif

 	regex->nxt = NULL;

+	regex->pattern = cli_strdup(pattern);

 	n = parse_regex(pattern, &last);

 	if(!n)

@@ -441,6 +439,7 @@ int cli_regex2suffix(const char *pattern, struct regex_list *regex, suffix_callb

 	n->parent = &root_node;

 	rc = build_suffixtree_descend(n, &buf, cb, cbdata, regex);

+	free(regex->pattern);

 	free(buf.data);

 	destroy_tree(n);

 	return rc;

@@ -25,7 +25,7 @@

 struct regex_list {

 	char *pattern;

-	regex_t preg;

+	regex_t *preg;

 	struct regex_list *nxt;

 };

 typedef int (*suffix_callback)(void *cbdata, const char *suffix, size_t len, struct regex_list *regex);

@@ -333,7 +333,7 @@ int main(int argc, char **argv)

     srunner_add_suite(sr, test_cli_suite());

     srunner_add_suite(sr, test_jsnorm_suite());

     srunner_add_suite(sr, test_str_suite());

-    /*    srunner_add_suite(sr, test_regex_suite()); */

+    srunner_add_suite(sr, test_regex_suite());

     srunner_add_suite(sr, test_disasm_suite());

     srunner_set_log(sr, "test.log");

@@ -61,7 +61,10 @@ START_TEST (empty)

 	const char pattern[] = "";

 	int rc;

 	errmsg_expected();

+	regex.preg = malloc(sizeof(*regex.preg));

+	fail_unless(!!regex.preg, "malloc");

 	rc = cli_regex2suffix(pattern, &regex, cb_fail, NULL);

+	free(regex.preg);

 	fail_unless(rc == REG_EMPTY, "empty pattern");

 	fail_unless(cb_called == 0, "callback shouldn't be called");

 }

@@ -71,9 +74,12 @@ START_TEST (one)

 {

 	const char pattern[] = "a";

 	int rc;

+	regex.preg = malloc(sizeof(*regex.preg));

+	fail_unless(!!regex.preg, "malloc");

 	rc = cli_regex2suffix(pattern, &regex, cb_expect_single, "a");

 	fail_unless(rc == 0, "single character pattern");

-	cli_regfree(&regex.preg);

+	cli_regfree(regex.preg);

+	free(regex.preg);

 	fail_unless(cb_called == 1, "callback should be called once");

 }

 END_TEST

@@ -111,9 +117,12 @@ START_TEST (test_suffix)

 	const char **p=tests[_i];

 	fail_unless(!!pattern, "test pattern");

+	regex.preg = malloc(sizeof(*regex.preg));

+	fail_unless(!!regex.preg, "malloc");

 	rc = cli_regex2suffix(pattern, &regex, cb_expect_multi, tests[_i]);

 	fail_unless(rc == 0, "single character pattern");

-	cli_regfree(&regex.preg);

+	cli_regfree(regex.preg);

+	free(regex.preg);

 	p++;

 	while(*p++) n++;

 	fail_unless(cb_called == n,

@@ -128,7 +137,6 @@ static void setup(void)

 static void teardown(void)

 {

-	free(regex.pattern);

 }

 static struct regex_matcher matcher;

@@ -154,7 +162,11 @@ static const struct rtest {

 	{".+\\.ebayrtm\\.com([/?].*)?:.+\\.ebay\\.(de|com|co\\.uk)([/?].*)?/",

 		"http://srx.main.ebayrtm.com",

 		"pages.ebay.de",

-		1 /* should be whitelisted */}

+		1 /* should be whitelisted */},

+	{".+\\.ebayrtm\\.com([/?].*)?:.+\\.ebay\\.(de|com|co\\.uk)([/?].*)?/",

+		"http://srx.main.ebayrtm.com.evil.example.com",

+		"pages.ebay.de",

+		0}

 };

 START_TEST (regex_list_match_test)