@@ -1,3 +1,8 @@

+Tue May  1 17:20:53 CEST 2007 (tk)

+----------------------------------

+  * prepare for enabling phishing code in non-experimental builds - apply

+    patches from Edwin

+

 Tue May  1 13:13:56 CEST 2007 (tk)

 ----------------------------------

   * configure: remove libcurl checks

@@ -302,12 +302,10 @@ int main(int argc, char **argv)

     else

 	logg("Not loading phishing signatures.\n");

-#ifdef CL_EXPERIMENTAL

     if(cfgopt(copt,"PhishingScanURLs")->enabled)

 	dboptions |= CL_DB_PHISHING_URLS;

     else

 	logg("Disabling URL based phishing detection.\n");

-#endif

     if(cfgopt(copt, "NodalCoreAcceleration")->enabled) {

 #ifdef HAVE_NCORE

@@ -431,7 +431,6 @@ int acceptloop_th(int *socketds, int nsockets, struct cl_engine *engine, unsigne

 	logg("HTML support disabled.\n");

     }

-#ifdef CL_EXPERIMENTAL

     if(cfgopt(copt,"PhishingScanURLs")->enabled) {

 	if(cfgopt(copt,"PhishingRestrictedScan")->enabled) {

@@ -457,7 +456,6 @@ int acceptloop_th(int *socketds, int nsockets, struct cl_engine *engine, unsigne

 	    logg("Phishing: Always checking for ssl mismatches\n");

 	}

     }

-#endif /* CL_EXPERIMENTAL */

     selfchk = cfgopt(copt, "SelfCheck")->numarg;

     if(!selfchk) {

@@ -300,12 +300,10 @@ void help(void)

     mprintf("\n");

     mprintf("    --no-mail                            Disable mail file support\n");

     mprintf("    --no-phishing-sigs                   Disable signature-based phishing detection\n");

-#ifdef CL_EXPERIMENTAL

     mprintf("    --no-phishing-scan-urls              Disable url-based phishing detection\n");

     mprintf("    --no-phishing-restrictedscan         Enable phishing detection for all domains (might lead to false positives!)\n");

     mprintf("    --phishing-ssl                       Always block SSL mismatches in URLs (phishing module)\n");

     mprintf("    --phishing-cloak                     Always block cloaked URLs (phishing module)\n");

-#endif

     mprintf("    --no-algorithmic                     Disable algorithmic detection\n");

     mprintf("    --no-pe                              Disable PE analysis\n");

     mprintf("    --no-elf                             Disable ELF support\n");

@@ -75,12 +75,10 @@ static struct option clamscan_longopt[] = {

     {"no-mail", 0, 0, 0},

     {"mail-follow-urls", 0, 0, 0},

     {"no-phishing-sigs", 0, 0, 0},

-#ifdef CL_EXPERIMENTAL

     {"no-phishing-scan-urls", 0, 0, 0},

     {"no-phishing-restrictedscan", 0, 0, 0},

     {"phishing-ssl", 0, 0, 0},

     {"phishing-cloak", 0, 0, 0},

-#endif

     {"no-algorithmic", 0, 0, 0},

     {"unzip", 2, 0, 0},

     {"unrar", 2, 0, 0},

@@ -171,7 +171,6 @@ int scanmanager(const struct optstruct *opt)

     if(!opt_check(opt, "no-phishing-sigs"))

 	dboptions |= CL_DB_PHISHING;

-#ifdef CL_EXPERIMENTAL

     if(!opt_check(opt,"no-phishing-scan-urls"))

 	dboptions |= CL_DB_PHISHING_URLS;

     if(!opt_check(opt,"no-phishing-restrictedscan")) {

@@ -184,7 +183,6 @@ int scanmanager(const struct optstruct *opt)

     if(opt_check(opt,"phishing-cloak")) {

 	options |= CL_SCAN_PHISHING_BLOCKCLOAK;

     }

-#endif

     if(opt_check(opt, "dev-ac-only"))

 	dboptions |= CL_DB_ACONLY;

@@ -75,7 +75,7 @@ extern "C"

 #define CL_DB_PHISHING_URLS 0x8

 /* recommended db settings */

-#define CL_DB_STDOPT	    CL_DB_PHISHING

+#define CL_DB_STDOPT	    (CL_DB_PHISHING | CL_DB_PHISHING_URLS)

 /* scan options */

 #define CL_SCAN_RAW		    0x0

@@ -39,6 +39,12 @@ struct dconf_module {

     uint8_t	state;	    /* default state (on/off) */

 };

+#ifdef CL_EXPERIMENTAL

+#define DCONF_ENABLE_EXPERIMENTAL 1

+#else

+#define DCONF_ENABLE_EXPERIMENTAL 0

+#endif

+

 static struct dconf_module modules[] = {

     { "PE",	    "PARITE",	    PE_CONF_PARITE,	    1 },

@@ -88,6 +94,9 @@ static struct dconf_module modules[] = {

     { "OTHER",	    "JPEG",	    OTHER_CONF_JPEG,	    1 },

     { "OTHER",	    "CRYPTFF",	    OTHER_CONF_CRYPTFF,	    1 },

+    { "PHISHING",   "ENGINE",       PHISHING_CONF_ENGINE,   1 },

+    { "PHISHING",   "ENTCONV",      PHISHING_CONF_ENTCONV,  DCONF_ENABLE_EXPERIMENTAL }, /* exp */

+

     { NULL,	    NULL,	    0,			    0 }

 };

@@ -125,6 +134,9 @@ struct cli_dconf *cli_dconf_init(void)

 	} else if(!strcmp(modules[i].mname, "OTHER")) {

 	    if(modules[i].state)

 		dconf->other |= modules[i].bflag;

+	} else if(!strcmp(modules[i].mname, "PHISHING")) {

+	    if(modules[i].state)

+		dconf->phishing |= modules[i].bflag;

 	}

     }

@@ -133,7 +145,7 @@ struct cli_dconf *cli_dconf_init(void)

 void cli_dconf_print(struct cli_dconf *dconf)

 {

-	uint8_t pe = 0, elf = 0, arch = 0, doc = 0, mail = 0, other = 0;

+	uint8_t pe = 0, elf = 0, arch = 0, doc = 0, mail = 0, other = 0, phishing=0;

 	unsigned int i;

@@ -196,6 +208,15 @@ void cli_dconf_print(struct cli_dconf *dconf)

 		cli_dbgmsg("   * Submodule %10s:\t%s\n", modules[i].sname, (dconf->other & modules[i].bflag) ? "On" : "** Off **");

 	    else

 		continue;

+	} else if(!strcmp(modules[i].mname, "PHISHING")) {

+	    if(!phishing) {

+		cli_dbgmsg("Module PHISHING %s\n", dconf->phishing ? "On" : "Off");

+		phishing = 1;

+	    }

+	    if(dconf->phishing)

+		cli_dbgmsg("   * Submodule %10s:\t%s\n", modules[i].sname, (dconf->phishing & modules[i].bflag) ? "On" : "** Off **");

+	    else

+		continue;

 	}

     }

 }

@@ -309,6 +330,15 @@ int cli_dconf_load(FILE *fd, struct cl_engine **engine, unsigned int options)

 		break;

 	    }

 	}

+

+	if(!strncmp(buffer, "PHISHING:", 9) && chkflevel(buffer, 2)) {

+	    if(sscanf(buffer + 9, "0x%x", &val) == 1) {

+		dconf->phishing = val;

+	    } else {

+		ret = CL_EMALFDB;

+		break;

+	    }

+	}

     }

     if(ret) {

@@ -31,6 +31,7 @@ struct cli_dconf {

     uint32_t doc;

     uint32_t mail;

     uint32_t other;

+    uint32_t phishing;

 };

 /* PE flags */

@@ -80,6 +81,9 @@ struct cli_dconf {

 #define OTHER_CONF_JPEG	    0x8

 #define OTHER_CONF_CRYPTFF  0x10

+/* Phishing flags */

+#define PHISHING_CONF_ENGINE   0x1

+#define PHISHING_CONF_ENTCONV  0x2

 struct cli_dconf *cli_dconf_init(void);

 void cli_dconf_print(struct cli_dconf *dconf);

@@ -37,10 +37,8 @@

 #include "matcher-ac.h"

 #include "str.h"

-#ifdef CL_EXPERIMENTAL

 #include "htmlnorm.h"

 #include "entconv.h"

-#endif

 struct cli_magic_s {

     size_t offset;

@@ -287,8 +285,7 @@ cli_file_t cli_filetype2(int desc, const struct cl_engine *engine)

 	    }

 	    cli_ac_freedata(&mdata);

-#ifdef CL_EXPERIMENTAL

-	    if(ret != CL_TYPE_HTML_UTF16) {

+	    if((((struct cli_dconf*) engine->dconf)->phishing & PHISHING_CONF_ENTCONV) && ret != CL_TYPE_HTML_UTF16) {

 		    struct entity_conv conv;

 		    const size_t conv_size = 2*bread < 256 ? 256 : 2*bread;

@@ -324,7 +321,6 @@ cli_file_t cli_filetype2(int desc, const struct cl_engine *engine)

 		    cli_warnmsg("cli_filetype2: Error initializing entity converter\n");

 		}

 	    }

-#endif /* CL_EXPERIMENTAL */

 	}

     }

@@ -55,12 +55,10 @@

 #include "lockdb.h"

 #include "readdb.h"

-#ifdef CL_EXPERIMENTAL

 #include "phishcheck.h"

 #include "phish_whitelist.h"

 #include "phish_domaincheck_db.h"

 #include "regex_list.h"

-#endif

 #if defined(HAVE_READDIR_R_3) || defined(HAVE_READDIR_R_2)

 #include <limits.h>

@@ -275,9 +273,7 @@ int cli_parse_add(struct cli_matcher *root, const char *virname, const char *hex

 int cli_initengine(struct cl_engine **engine, unsigned int options)

 {

-#ifdef CL_EXPERIMENTAL

 	int ret;

-#endif

     if(!*engine) {

@@ -309,11 +305,9 @@ int cli_initengine(struct cl_engine **engine, unsigned int options)

 	}

     }

-#ifdef CL_EXPERIMENTAL

-    if(options & CL_DB_PHISHING_URLS)

+    if((options & CL_DB_PHISHING_URLS) && (((struct cli_dconf*) (*engine)->dconf)->phishing & PHISHING_CONF_ENGINE))

 	if((ret = phishing_init(*engine)))

 	    return ret;

-#endif

     return CL_SUCCESS;

 }

@@ -418,7 +412,6 @@ static int cli_loaddb(FILE *fd, struct cl_engine **engine, unsigned int *signo,

     return CL_SUCCESS;

 }

-#ifdef CL_EXPERIMENTAL

 static int cli_loadwdb(FILE *fd, struct cl_engine **engine, unsigned int options)

 {

 	int ret = 0;

@@ -429,6 +422,9 @@ static int cli_loadwdb(FILE *fd, struct cl_engine **engine, unsigned int options

 	return ret;

     }

+    if(!(((struct cli_dconf *) (*engine)->dconf)->phishing & PHISHING_CONF_ENGINE))

+	return CL_SUCCESS;

+

     if(!(*engine)->whitelist_matcher) {

 	if((ret = init_whitelist(*engine))) {

 	    phishing_done(*engine);

@@ -456,6 +452,9 @@ static int cli_loadpdb(FILE *fd, struct cl_engine **engine, unsigned int options

 	return ret;

     }

+    if(!(((struct cli_dconf *) (*engine)->dconf)->phishing & PHISHING_CONF_ENGINE))

+	return CL_SUCCESS;

+

     if(!(*engine)->domainlist_matcher) {

 	if((ret = init_domainlist(*engine))) {

 	    phishing_done(*engine);

@@ -472,7 +471,6 @@ static int cli_loadpdb(FILE *fd, struct cl_engine **engine, unsigned int options

     return CL_SUCCESS;

 }

-#endif

 #define NDB_TOKENS 6

 static int cli_loadndb(FILE *fd, struct cl_engine **engine, unsigned int *signo, unsigned short sdb, unsigned int options)

@@ -1017,7 +1015,6 @@ static int cli_load(const char *filename, struct cl_engine **engine, unsigned in

 	else

 #endif

 	    skipped = 1;

-#ifdef CL_EXPERIMENTAL

     } else if(cli_strbcasestr(filename, ".wdb")) {

 	if(options & CL_DB_PHISHING_URLS)

 	    ret = cli_loadwdb(fd, engine, options);

@@ -1028,7 +1025,6 @@ static int cli_load(const char *filename, struct cl_engine **engine, unsigned in

 	    ret = cli_loadpdb(fd, engine, options);

 	else

 	    skipped = 1;

-#endif

     } else {

 	cli_dbgmsg("cli_load: unknown extension - assuming old database format\n");

 	ret = cli_loaddb(fd, engine, signo, options);

@@ -1108,10 +1104,8 @@ static int cli_loaddbdir_l(const char *dirname, struct cl_engine **engine, unsig

 	     cli_strbcasestr(dent->d_name, ".sdb")  ||

 	     cli_strbcasestr(dent->d_name, ".zmd")  ||

 	     cli_strbcasestr(dent->d_name, ".rmd")  ||

-#ifdef CL_EXPERIMENTAL

 	     cli_strbcasestr(dent->d_name, ".pdb")  ||

 	     cli_strbcasestr(dent->d_name, ".wdb")  ||

-#endif

 	     cli_strbcasestr(dent->d_name, ".ncdb") ||

 	     cli_strbcasestr(dent->d_name, ".inc")  ||

 	     cli_strbcasestr(dent->d_name, ".cvd"))) {

@@ -1265,10 +1259,8 @@ int cl_statinidir(const char *dirname, struct cl_stat *dbstat)

 	    cli_strbcasestr(dent->d_name, ".zmd")  || 

 	    cli_strbcasestr(dent->d_name, ".rmd")  || 

 	    cli_strbcasestr(dent->d_name, ".cfg")  ||

-#ifdef CL_EXPERIMENTAL

 	    cli_strbcasestr(dent->d_name, ".pdb")  ||

 	    cli_strbcasestr(dent->d_name, ".wdb")  ||

-#endif

 	    cli_strbcasestr(dent->d_name, ".ncdb")  ||

 	    cli_strbcasestr(dent->d_name, ".inc")   ||

 	    cli_strbcasestr(dent->d_name, ".cvd"))) {

@@ -1379,10 +1371,8 @@ int cl_statchkdir(const struct cl_stat *dbstat)

 	    cli_strbcasestr(dent->d_name, ".zmd")  || 

 	    cli_strbcasestr(dent->d_name, ".rmd")  || 

 	    cli_strbcasestr(dent->d_name, ".cfg")  ||

-#ifdef CL_EXPERIMENTAL

 	    cli_strbcasestr(dent->d_name, ".pdb")  ||

 	    cli_strbcasestr(dent->d_name, ".wdb")  ||

-#endif

 	    cli_strbcasestr(dent->d_name, ".ncdb")  ||

 	    cli_strbcasestr(dent->d_name, ".inc")   ||

 	    cli_strbcasestr(dent->d_name, ".cvd"))) {

@@ -1553,9 +1543,8 @@ void cl_free(struct cl_engine *engine)

 	free(metah);

     }

-#ifdef CL_EXPERIMENTAL

-    phishing_done(engine);

-#endif

+    if(((struct cli_dconf *) engine->dconf)->phishing & PHISHING_CONF_ENGINE)

+	phishing_done(engine);

     if(engine->dconf)

 	free(engine->dconf);

@@ -49,14 +49,12 @@ struct cfgoption cfg_options[] = {

     {"MailFollowURLs", OPT_BOOL, 0, NULL, 0, OPT_CLAMD},

     {"MailMaxRecursion", OPT_NUM, 64, NULL, 0, OPT_CLAMD},

     {"PhishingSignatures", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},

-#ifdef CL_EXPERIMENTAL

     {"PhishingScanURLs",OPT_BOOL, 1, NULL, 0, OPT_CLAMD},

     /* these are FP prone options, if default isn't used */

     {"PhishingAlwaysBlockCloak", OPT_BOOL, 0, NULL, 0, OPT_CLAMD},

     {"PhishingAlwaysBlockSSLMismatch", OPT_BOOL, 0, NULL, 0, OPT_CLAMD},

     {"PhishingRestrictedScan", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},

     /* end of FP prone options */

-#endif

     {"AlgorithmicDetection", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},

     {"ScanHTML", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},

     {"ScanOLE2", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},

@@ -140,7 +140,7 @@ static int htmlnorm(struct optstruct *opt)

 	return -1;

     }

-    html_normalise_fd(fd, ".", NULL);

+    html_normalise_fd(fd, ".", NULL, NULL);

     close(fd);

     return 0;