@@ -1,3 +1,9 @@

+Thu Jul 31 18:09:58 CEST 2008 (tk)

+----------------------------------

+  * libclamav: add support for loading specific PUA categories (bb#939)

+  * clamd: new options ExcludePUA and IncludePUA

+  * clamscan: new switches --exclude-pua and --include-pua

+

 Thu Jul 31 13:35:11 EEST 2008 (edwin)

 -------------------------------------

   * clamd, clamscan, libclamav: new option HeuristicScanPrecedence (bb #649)

@@ -97,7 +97,8 @@ int main(int argc, char **argv)

 	time_t currtime;

 	struct cl_engine *engine = NULL;

 	const char *dbdir, *cfgfile;

-	int ret, tcpsock = 0, localsock = 0;

+	char *pua_cats = NULL;

+	int ret, tcpsock = 0, localsock = 0, i;

 	unsigned int sigs = 0;

 	int lsockets[2], nlsockets = 0;

 	unsigned int dboptions = 0;

@@ -302,10 +303,74 @@ int main(int argc, char **argv)

     dbdir = cfgopt(copt, "DatabaseDirectory")->strarg;

     logg("#Reading databases from %s\n", dbdir);

-    if(cfgopt(copt, "DetectPUA")->enabled)

+    if(cfgopt(copt, "DetectPUA")->enabled) {

 	dboptions |= CL_DB_PUA;

-    else

+

+	if((cpt = cfgopt(copt, "ExcludePUA"))->enabled) {

+	    dboptions |= CL_DB_PUA_EXCLUDE;

+	    i = 0;

+	    logg("#Excluded PUA categories:");

+	    while(cpt) {

+		if(!(pua_cats = realloc(pua_cats, i + strlen(cpt->strarg) + 3))) {

+		    logg("!Can't allocate memory for pua_cats\n");

+		    logg_close();

+		    freecfg(copt);

+		    return 1;

+		}

+		logg("# %s", cpt->strarg);

+		sprintf(pua_cats + i, ".%s", cpt->strarg);

+		i += strlen(cpt->strarg) + 1;

+		pua_cats[i] = 0;

+		cpt = cpt->nextarg;

+	    }

+	    logg("#\n");

+	    pua_cats[i] = '.';

+	    pua_cats[i + 1] = 0;

+	}

+

+	if((cpt = cfgopt(copt, "IncludePUA"))->enabled) {

+	    if(pua_cats) {

+		logg("!ExcludePUA and IncludePUA cannot be used at the same time\n");

+		logg_close();

+		freecfg(copt);

+		free(pua_cats);

+		return 1;

+	    }

+	    dboptions |= CL_DB_PUA_INCLUDE;

+	    i = 0;

+	    logg("#Included PUA categories:");

+	    while(cpt) {

+		if(!(pua_cats = realloc(pua_cats, i + strlen(cpt->strarg) + 3))) {

+		    logg("!Can't allocate memory for pua_cats\n");

+		    logg_close();

+		    freecfg(copt);

+		    return 1;

+		}

+		logg("# %s", cpt->strarg);

+		sprintf(pua_cats + i, ".%s", cpt->strarg);

+		i += strlen(cpt->strarg) + 1;

+		pua_cats[i] = 0;

+		cpt = cpt->nextarg;

+	    }

+	    logg("#\n");

+	    pua_cats[i] = '.';

+	    pua_cats[i + 1] = 0;

+	}

+

+	if(pua_cats) {

+	    /* FIXME with the new API */

+	    if((ret = cli_initengine(&engine, dboptions))) {

+		logg("!cli_initengine() failed: %s\n", cl_strerror(ret));

+		logg_close();

+		freecfg(copt);

+		free(pua_cats);

+		return 1;

+	    }

+	    engine->pua_cats = pua_cats;

+	}

+    } else {

 	logg("#Not loading PUA signatures.\n");

+    }

     if(cfgopt(copt, "PhishingSignatures")->enabled)

 	dboptions |= CL_DB_PHISHING;

@@ -194,6 +194,7 @@ static struct cl_engine *reload_db(struct cl_engine *engine, unsigned int dbopti

 	const char *dbdir;

 	int retval;

 	unsigned int sigs = 0;

+	char *pua_cats = NULL;

     *ret = 0;

     if(do_check) {

@@ -211,12 +212,6 @@ static struct cl_engine *reload_db(struct cl_engine *engine, unsigned int dbopti

 	}

     }

-    /* release old structure */

-    if(engine) {

-	cl_free(engine);

-	engine = NULL;

-    }

-

     dbdir = cfgopt(copt, "DatabaseDirectory")->strarg;

     logg("Reading databases from %s\n", dbdir);

@@ -238,6 +233,26 @@ static struct cl_engine *reload_db(struct cl_engine *engine, unsigned int dbopti

 	return NULL;

     }

+    /* release old structure */

+    if(engine) {

+	if(engine->pua_cats)

+	    if(!(pua_cats = strdup(engine->pua_cats)))

+		logg("^Can't make a copy of pua_cats\n");

+

+	cl_free(engine);

+	engine = NULL;

+    }

+

+    if(pua_cats) {

+	if((retval = cli_initengine(&engine, dboptions))) {

+	    logg("!cli_initengine() failed: %s\n", cl_strerror(retval));

+	    *ret = 1;

+	    free(pua_cats);

+	    return NULL;

+	}

+	engine->pua_cats = pua_cats;

+    }

+

     if((retval = cl_load(dbdir, &engine, &sigs, dboptions))) {

 	logg("!reload db failed: %s\n", cl_strerror(retval));

 	*ret = 1;

@@ -315,6 +315,8 @@ void help(void)

 #endif

     mprintf("\n");

     mprintf("    --detect-pua                         Detect Possibly Unwanted Applications\n");

+    mprintf("    --exclude-pua=CAT                    Skip PUA sigs of category CAT\n");

+    mprintf("    --include-pua=CAT                    Load PUA sigs of category CAT\n");

     mprintf("    --detect-structured                  Detect structured data (SSN, Credit Card)\n");

     mprintf("    --structured-ssn-format=X            SSN format (0=normal,1=stripped,2=both)\n");

     mprintf("    --structured-ssn-count=N             Min SSN count to generate a detect\n");

@@ -59,6 +59,8 @@ static struct option clamscan_longopt[] = {

     {"max-recursion", 1, 0, 0},

     {"max-dir-recursion", 1, 0, 0},

     {"detect-pua", 0, 0, 0},

+    {"exclude-pua", 1, 0, 0},

+    {"include-pua", 1, 0, 0},

     {"detect-structured", 0, 0, 0},

     {"structured-ssn-format", 1, 0, 0},

     {"structured-ssn-count", 1, 0, 0},

@@ -344,7 +344,8 @@ int scanmanager(const struct optstruct *opt)

 	struct cl_engine *engine = NULL;

 	struct cl_limits limits;

 	struct stat sb;

-	char *file, cwd[1024];

+	char *file, cwd[1024], *pua_cats = NULL, *argument;

+	const struct optnode *optnode;

 #ifndef C_WINDOWS

 	struct rlimit rlim;

 #endif

@@ -371,9 +372,61 @@ int scanmanager(const struct optstruct *opt)

     if(opt_check(opt, "dev-ac-depth"))

 	cli_ac_setdepth(AC_DEFAULT_MIN_DEPTH, atoi(opt_arg(opt, "dev-ac-depth")));

-    if(opt_check(opt, "detect-pua"))

+    if(opt_check(opt, "detect-pua")) {

 	dboptions |= CL_DB_PUA;

+	if(opt_check(opt, "exclude-pua")) {

+	    dboptions |= CL_DB_PUA_EXCLUDE;

+	    argument = opt_firstarg(opt, "exclude-pua", &optnode);

+	    i = 0;

+	    while(argument) {

+		if(!(pua_cats = realloc(pua_cats, i + strlen(argument) + 3))) {

+		    logg("!Can't allocate memory for pua_cats\n");

+		    return 70;

+		}

+		sprintf(pua_cats + i, ".%s", argument);

+		i += strlen(argument) + 1;

+		pua_cats[i] = 0;

+		argument = opt_nextarg(&optnode, "exclude-pua");

+	    }

+	    pua_cats[i] = '.';

+	    pua_cats[i + 1] = 0;

+	}

+

+	if(opt_check(opt, "include-pua")) {

+	    if(pua_cats) {

+		logg("!--exclude-pua and --include-pua cannot be used at the same time\n");

+		free(pua_cats);

+		return 40;

+	    }

+	    dboptions |= CL_DB_PUA_INCLUDE;

+	    argument = opt_firstarg(opt, "include-pua", &optnode);

+	    i = 0;

+	    while(argument) {

+		if(!(pua_cats = realloc(pua_cats, i + strlen(argument) + 3))) {

+		    logg("!Can't allocate memory for pua_cats\n");

+		    return 70;

+		}

+		sprintf(pua_cats + i, ".%s", argument);

+		i += strlen(argument) + 1;

+		pua_cats[i] = 0;

+		argument = opt_nextarg(&optnode, "include-pua");

+	    }

+	    pua_cats[i] = '.';

+	    pua_cats[i + 1] = 0;

+	}

+

+	if(pua_cats) {

+	    /* FIXME with the new API */

+	    if((ret = cli_initengine(&engine, dboptions))) {

+		logg("!cli_initengine() failed: %s\n", cl_strerror(ret));

+		free(pua_cats);

+		return 50;

+	    }

+	    engine->pua_cats = pua_cats;

+	}

+    }

+

     if(opt_check(opt, "database")) {

 	if((ret = cl_load(opt_arg(opt, "database"), &engine, &info.sigs, dboptions))) {

 	    logg("!%s\n", cl_strerror(ret));

@@ -72,6 +72,9 @@ extern "C"

 #define CL_DB_PUA	    0x10

 #define CL_DB_CVDNOTMP	    0x20

 #define CL_DB_OFFICIAL	    0x40

+#define CL_DB_PUA_MODE	    0x80

+#define CL_DB_PUA_INCLUDE   0x100

+#define CL_DB_PUA_EXCLUDE   0x200

 /* recommended db settings */

 #define CL_DB_STDOPT	    (CL_DB_PHISHING | CL_DB_PHISHING_URLS)

@@ -146,6 +149,9 @@ struct cl_engine {

     /* Ignored signatures */

     void *ignored;

+

+    /* PUA categories (to be included or excluded) */

+    char *pua_cats;

 };

 struct cl_limits {

@@ -448,6 +448,42 @@ static int cli_chkign(const struct cli_ignored *ignored, const char *dbname, uns

     return 0;

 }

+static int cli_chkpua(const char *signame, const char *pua_cats, unsigned int options)

+{

+	char cat[32], *pt;

+	const char *sig;

+	int ret;

+

+    if(strncmp(signame, "PUA.", 4)) {

+	cli_dbgmsg("Skipping signature %s - no PUA prefix\n", signame);

+	return 1;

+    }

+    sig = signame + 3;

+    if(!(pt = strchr(sig + 1, '.'))) {

+	cli_dbgmsg("Skipping signature %s - bad syntax\n", signame);

+	return 1;

+    }

+

+    if(pt - sig + 2 > sizeof(cat)) {

+	cli_dbgmsg("Skipping signature %s - too long category name\n", signame);

+	return 1;

+    }

+

+    strncpy(cat, sig, pt - signame + 1);

+    cat[pt - sig + 1] = 0;

+    pt = strstr(pua_cats, cat);

+

+    if(options & CL_DB_PUA_INCLUDE)

+	ret = pt ? 0 : 1;

+    else

+	ret = pt ? 1 : 0;

+

+    if(ret)

+	cli_dbgmsg("Skipping PUA signature %s - excluded category\n", signame);

+

+    return ret;

+}

+

 static int cli_loaddb(FILE *fs, struct cl_engine **engine, unsigned int *signo, unsigned int options, struct cli_dbio *dbio, const char *dbname)

 {

 	char buffer[FILEBUFF], *pt, *start;

@@ -613,6 +649,10 @@ static int cli_loadndb(FILE *fs, struct cl_engine **engine, unsigned int *signo,

 	    break;

 	}

+	if((*engine)->pua_cats && (options & CL_DB_PUA_MODE) && (options & (CL_DB_PUA_INCLUDE | CL_DB_PUA_EXCLUDE)))

+	    if(cli_chkpua(virname, (*engine)->pua_cats, options))

+		continue;

+

 	if((*engine)->ignored && cli_chkign((*engine)->ignored, dbname, line, virname))

 	    continue;

@@ -891,6 +931,10 @@ static int cli_loadldb(FILE *fs, struct cl_engine **engine, unsigned int *signo,

 	    break;

 	}

+	if((*engine)->pua_cats && (options & CL_DB_PUA_MODE) && (options & (CL_DB_PUA_INCLUDE | CL_DB_PUA_EXCLUDE)))

+	    if(cli_chkpua(virname, (*engine)->pua_cats, options))

+		continue;

+

 	if((*engine)->ignored && cli_chkign((*engine)->ignored, dbname, line, virname))

 	    continue;

@@ -1307,9 +1351,21 @@ static int cli_loadmd5(FILE *fs, struct cl_engine **engine, unsigned int *signo,

     while(cli_dbgets(buffer, FILEBUFF, fs, dbio)) {

 	line++;

 	cli_chomp(buffer);

-

 	cli_strtokenize(buffer, ':', MD5_TOKENS, tokens);

+	if(!(pt = tokens[2])) { /* virname */

+	    ret = CL_EMALFDB;

+	    break;

+	}

+

+	if((*engine)->pua_cats && (options & CL_DB_PUA_MODE) && (options & (CL_DB_PUA_INCLUDE | CL_DB_PUA_EXCLUDE)))

+	    if(cli_chkpua(pt, (*engine)->pua_cats, options))

+		continue;

+

+	if((*engine)->ignored && cli_chkign((*engine)->ignored, dbname, line, pt))

+	    continue;

+

+

 	new = (struct cli_bm_patt *) cli_calloc(1, sizeof(struct cli_bm_patt));

 	if(!new) {

 	    ret = CL_EMEM;

@@ -1345,13 +1401,6 @@ static int cli_loadmd5(FILE *fs, struct cl_engine **engine, unsigned int *signo,

 	    break;

 	}

-	if((*engine)->ignored && cli_chkign((*engine)->ignored, dbname, line, new->virname)) {

-	    free(new->virname);

-	    free(new->pattern);

-	    free(new);

-	    continue;

-	}

-

 	MD5_DB;

 	if(!db && (ret = cli_md5db_init(engine, mode))) {

 	    free(new->pattern);

@@ -1627,7 +1676,7 @@ int cli_load(const char *filename, struct cl_engine **engine, unsigned int *sign

     } else if(cli_strbcasestr(dbname, ".hdu")) {

 	if(options & CL_DB_PUA)

-	    ret = cli_loadmd5(fs, engine, signo, MD5_HDB, options, dbio, dbname);

+	    ret = cli_loadmd5(fs, engine, signo, MD5_HDB, options | CL_DB_PUA_MODE, dbio, dbname);

 	else

 	    skipped = 1;

@@ -1639,7 +1688,7 @@ int cli_load(const char *filename, struct cl_engine **engine, unsigned int *sign

     } else if(cli_strbcasestr(dbname, ".mdu")) {

 	if(options & CL_DB_PUA)

-	    ret = cli_loadmd5(fs, engine, signo, MD5_MDB, options, dbio, dbname);

+	    ret = cli_loadmd5(fs, engine, signo, MD5_MDB, options | CL_DB_PUA_MODE, dbio, dbname);

 	else

 	    skipped = 1;

@@ -1650,14 +1699,14 @@ int cli_load(const char *filename, struct cl_engine **engine, unsigned int *sign

 	if(!(options & CL_DB_PUA))

 	    skipped = 1;

 	else

-	    ret = cli_loadndb(fs, engine, signo, 0, options, dbio, dbname);

+	    ret = cli_loadndb(fs, engine, signo, 0, options | CL_DB_PUA_MODE, dbio, dbname);

     } else if(cli_strbcasestr(filename, ".ldb")) {

        ret = cli_loadldb(fs, engine, signo, options, dbio, dbname);

     } else if(cli_strbcasestr(filename, ".ldu")) {

 	if(options & CL_DB_PUA)

-	    ret = cli_loadldb(fs, engine, signo, options, dbio, dbname);

+	    ret = cli_loadldb(fs, engine, signo, options | CL_DB_PUA_MODE, dbio, dbname);

 	else

 	    skipped = 1;

@@ -2145,6 +2194,9 @@ void cl_free(struct cl_engine *engine)

     if(engine->dconf)

 	free(engine->dconf);

+    if(engine->pua_cats)

+	free(engine->pua_cats);

+

     cli_ftfree(engine->ftypes);

     cli_freeign(engine);

     free(engine);

@@ -56,6 +56,8 @@ struct cfgoption cfg_options[] = {

     {"HeuristicScanPrecedence", OPT_BOOL, 0, NULL, 0, OPT_CLAMD},

     /* end of FP prone options */

     {"DetectPUA", OPT_BOOL, 0, NULL, 0, OPT_CLAMD},

+    {"ExcludePUA", OPT_QUOTESTR, -1, NULL, 1, OPT_CLAMD},

+    {"IncludePUA", OPT_QUOTESTR, -1, NULL, 1, OPT_CLAMD},

     {"StructuredDataDetection", OPT_BOOL, 0, NULL, 0, OPT_CLAMD},

     {"StructuredMinCreditCardCount", OPT_NUM, 1, NULL, 0, OPT_CLAMD},

     {"StructuredMinSSNCount", OPT_NUM, 1, NULL, 0, OPT_CLAMD},