@@ -1,7 +1,7 @@

 /*

  *  ClamAV bytecode handler tool.

  *

- *  Copyright (C) 2009 Sourcefire, Inc.

+ *  Copyright (C) 2009-2012 Sourcefire, Inc.

  *

  *  Authors: Török Edvin

  *

@@ -306,7 +306,8 @@ int main(int argc, char *argv[])

     bcs.all_bcs = bc;

     bcs.count = 1;

-    rc = cli_bytecode_load(bc, f, NULL, optget(opts, "trust-bytecode")->enabled);

+    rc = cli_bytecode_load(bc, f, NULL, optget(opts, "trust-bytecode")->enabled, 

+			   optget(opts, "bytecode-statistics")->enabled);

     if (rc != CL_SUCCESS) {

 	fprintf(stderr,"Unable to load bytecode: %s\n", cl_strerror(rc));

 	optfree(opts);

@@ -1,5 +1,5 @@

 /*

- *  Copyright (C) 2007-2009 Sourcefire, Inc.

+ *  Copyright (C) 2007-2012 Sourcefire, Inc.

  *

  *  Authors: Tomasz Kojm

  *

@@ -231,6 +231,7 @@ void help(void)

     mprintf("    --bytecode[=yes(*)/no]               Load bytecode from the database\n");

     mprintf("    --bytecode-unsigned[=yes/no(*)]      Load unsigned bytecode\n");

     mprintf("    --bytecode-timeout=N                 Set bytecode timeout (in milliseconds)\n");

+    mprintf("    --bytecode-statistics[=yes/no(*)]    Collect and print bytecode statistics\n");

     mprintf("    --detect-pua[=yes/no(*)]             Detect Possibly Unwanted Applications\n");

     mprintf("    --exclude-pua=CAT                    Skip PUA sigs of category CAT\n");

     mprintf("    --include-pua=CAT                    Load PUA sigs of category CAT\n");

@@ -1,5 +1,5 @@

 /*

- *  Copyright (C) 2007-2009 Sourcefire, Inc.

+ *  Copyright (C) 2007-2012 Sourcefire, Inc.

  *

  *  Authors: Tomasz Kojm

  *

@@ -635,6 +635,9 @@ int scanmanager(const struct optstruct *opts)

     if(optget(opts, "bytecode-unsigned")->enabled)

 	dboptions |= CL_DB_BYTECODE_UNSIGNED;

+    if(optget(opts, "bytecode-statistics")->enabled)

+	dboptions |= CL_DB_BYTECODE_STATS;

+

     if((opt = optget(opts,"bytecode-timeout"))->enabled)

 	cl_engine_set_num(engine, CL_ENGINE_BYTECODE_TIMEOUT, opt->numarg);

     if((opt = optget(opts,"bytecode-mode"))->enabled) {

@@ -938,6 +941,11 @@ int scanmanager(const struct optstruct *opts)

 	}

     }

+    if(optget(opts, "bytecode-statistics")->enabled) {

+	cli_sigperf_print();

+	cli_sigperf_events_destroy();

+    }

+

     /* free the engine */

     cl_engine_free(engine);

@@ -96,6 +96,9 @@ Allow loading bytecode from outside digitally signed .c[lv]d files.

 \fB\-\-bytecode\-timeout=N\fR

 Set bytecode timeout in milliseconds (default: 60000 = 60s)

 .TP 

+\fB\-\-bytecode\-statistics[=yes/no(*)]\fR

+Collect and print bytecode statistics.

+.TP 

 \fB\-\-detect\-pua[=yes/no(*)]\fR

 Detect Possibly Unwanted Applications.

 .TP 

@@ -1,7 +1,7 @@

 /*

  *  Load, and verify ClamAV bytecode.

  *

- *  Copyright (C) 2009-2010 Sourcefire, Inc.

+ *  Copyright (C) 2009-2012 Sourcefire, Inc.

  *

  *  Authors: Török Edvin

  *

@@ -40,6 +40,13 @@

 #include "builtin_bytecodes.h"

 #include <string.h>

+#define MAX_BC 64

+#define BC_EVENTS_PER_SIG 2

+#define MAX_BC_SIGEVENT_ID MAX_BC*BC_EVENTS_PER_SIG

+

+cli_events_t * g_sigevents = NULL;

+unsigned int g_sigid;

+

 /* dummy values */

 static const uint32_t nomatch[64] = {

     0xdeadbeef, 0xdeaddead, 0xbeefdead, 0xdeaddead, 0xdeadbeef, 0, 0, 0,

@@ -1409,7 +1416,112 @@ enum parse_state {

     PARSE_SKIP

 };

-int cli_bytecode_load(struct cli_bc *bc, FILE *f, struct cli_dbio *dbio, int trust)

+struct sigperf_elem {

+    const char * bc_name;

+    uint64_t usecs;

+    unsigned long run_count;

+    unsigned long match_count;

+};

+

+static int sigelem_comp(const void * a, const void * b)

+{

+    const struct sigperf_elem *ela = a;

+    const struct sigperf_elem *elb = b;

+    return elb->usecs/elb->run_count - ela->usecs/ela->run_count;

+}

+

+void cli_sigperf_print()

+{

+    struct sigperf_elem stats[MAX_BC], *elem = stats;

+    int i, elems = 0, max_name_len = 0, name_len;

+

+    memset(stats, 0, sizeof(stats));

+    for (i=0;i<MAX_BC;i++) {

+	union ev_val val;

+	uint32_t count;

+	const char * name = cli_event_get_name(g_sigevents, i*BC_EVENTS_PER_SIG);

+	cli_event_get(g_sigevents, i*BC_EVENTS_PER_SIG, &val, &count);

+	if (!count) {

+	    if (name)

+		cli_dbgmsg("No event triggered for %s\n", name);

+	    continue;

+	}

+	name_len = strlen(name);

+	if (name_len > max_name_len)

+	    max_name_len = name_len;

+	elem->bc_name = name?name:"\"noname\"";

+	elem->usecs = val.v_int;

+	elem->run_count = count;

+	cli_event_get(g_sigevents, i*BC_EVENTS_PER_SIG+1, &val, &count);

+	elem->match_count = count;

+	elem++, elems++;

+    }

+

+    qsort(stats, elems, sizeof(struct sigperf_elem), sigelem_comp);

+

+    elem = stats;

+    /* name runs matches microsecs avg */

+    cli_infomsg (NULL, "%-*s %*s %*s %*s %*s\n", max_name_len, "Bytecode name",

+	    8, "#runs", 8, "#matches", 12, "usecs total", 9, "usecs avg");

+    cli_infomsg (NULL, "%-*s %*s %*s %*s %*s\n", max_name_len, "=============",

+	    8, "=====", 8, "========", 12, "===========", 9, "=========");

+    while (elem->run_count) {

+	cli_infomsg (NULL, "%-*s %*lu %*lu %*lu %*.2f\n", max_name_len, elem->bc_name,

+		     8, elem->run_count, 8, elem->match_count, 

+		12, elem->usecs, 9, (double)elem->usecs/elem->run_count);

+	elem++;

+    }

+}

+

+static void sigperf_events_init(struct cli_bc *bc)

+{

+    int ret;

+

+    if (!g_sigevents)

+	g_sigevents = cli_events_new(MAX_BC_SIGEVENT_ID);

+

+    if (!g_sigevents) {

+	cli_errmsg("No memory for events table\n");

+	return;

+    }

+

+    if (g_sigid > MAX_BC_SIGEVENT_ID - BC_EVENTS_PER_SIG - 1) {

+	cli_errmsg("sigperf_events_init: events table full. Increase MAX_BC\n");

+	return;

+    }

+

+    cli_dbgmsg("sigperf_events_init(): adding sig ids starting %u for %s\n", g_sigid, bc->lsig);

+

+    if (!bc->lsig) {

+	cli_dbgmsg("cli_event_define error for time event id %d\n", bc->sigtime_id);

+	return;

+    }

+

+    /* register time event */

+    bc->sigtime_id = g_sigid;

+    ret = cli_event_define(g_sigevents, g_sigid++, bc->lsig, ev_time, multiple_sum);

+    if (ret) {

+	cli_errmsg("sigperf_events_init: cli_event_define() error for time event id %d\n", bc->sigtime_id);

+	bc->sigtime_id = MAX_BC_SIGEVENT_ID+1;

+	return;

+    }

+

+    /* register match count */

+    bc->sigmatch_id = g_sigid;

+    ret = cli_event_define(g_sigevents, g_sigid++, bc->lsig, ev_int, multiple_sum);

+    if (ret) {

+	cli_errmsg("sigperf_events_init: cli_event_define() error for matches event id %d\n", bc->sigmatch_id);

+	bc->sigtime_id = MAX_BC_SIGEVENT_ID+1;

+	return;

+    }

+}

+

+void cli_sigperf_events_destroy()

+{

+    cli_events_free(g_sigevents);

+}

+

+int cli_bytecode_load(struct cli_bc *bc, FILE *f, struct cli_dbio *dbio, int trust, int sigperf)

 {

     unsigned row = 0, current_func = 0, bb=0;

     char *buffer;

@@ -1565,6 +1677,8 @@ int cli_bytecode_load(struct cli_bc *bc, FILE *f, struct cli_dbio *dbio, int tru

     }

     free(buffer);

     cli_dbgmsg("Parsed %d functions\n", current_func);

+    if (sigperf)

+	sigperf_events_init(bc);

     if (current_func != bc->num_func && bc->state != bc_skip) {

 	cli_errmsg("Loaded less functions than declared: %u vs. %u\n",

 		   current_func, bc->num_func);

@@ -1652,6 +1766,7 @@ int cli_bytecode_run(const struct cli_all_bc *bcs, const struct cli_bc *bc, stru

 	    return CL_EBYTECODE_TESTFAIL;

 	}

     }

+    cli_event_time_start(g_sigevents, bc->sigtime_id);

     if (bc->state == bc_interp || test_mode) {

 	ctx->bc_events = interp_ev;

 	memset(&func, 0, sizeof(func));

@@ -1705,6 +1820,9 @@ int cli_bytecode_run(const struct cli_all_bc *bcs, const struct cli_bc *bc, stru

 	if (ctx->outfd)

 	    cli_bcapi_extract_new(ctx, -1);

     }

+    cli_event_time_stop(g_sigevents, bc->sigtime_id);

+    if (ctx->virname)

+	cli_event_count(g_sigevents, bc->sigmatch_id);

     if (test_mode) {

 	unsigned interp_errors = cli_event_errors(interp_ev);

@@ -1738,7 +1856,8 @@ int cli_bytecode_run(const struct cli_all_bc *bcs, const struct cli_bc *bc, stru

 	    ok = 0;

 	}

 	/*cli_event_debug(jit_ev, BCEV_EXEC_TIME);

-	cli_event_debug(interp_ev, BCEV_EXEC_TIME);*/

+        cli_event_debug(interp_ev, BCEV_EXEC_TIME);

+	cli_event_debug(g_sigevents, bc->sigtime_id);*/

 	if (!ok) {

 	    cli_events_free(jit_ev);

 	    cli_events_free(interp_ev);

@@ -2398,7 +2517,7 @@ static int run_builtin_or_loaded(struct cli_all_bc *bcs, uint8_t kind, const cha

 	    return CL_EMALFDB;

 	}

-	rc = cli_bytecode_load(bc, NULL, &dbio, 1);

+	rc = cli_bytecode_load(bc, NULL, &dbio, 1, 0);

 	if (rc) {

 	    cli_errmsg("Failed to load builtin %s bytecode\n", desc);

 	    free(bc);

@@ -2692,7 +2811,7 @@ int cli_bytecode_runhook(cli_ctx *cctx, const struct cl_engine *engine, struct c

 	    continue;

 	}

 	if (ctx->virname) {

-	    cli_dbgmsg("Bytecode found virus: %s\n", ctx->virname);

+	    cli_dbgmsg("Bytecode runhook found virus: %s\n", ctx->virname);

 	    cli_append_virus(cctx, ctx->virname);

 	    if (!(cctx->options & CL_SCAN_ALLMATCHES)) {

 		cli_bytecode_context_clear(ctx);

@@ -1,7 +1,7 @@

 /*

  *  Load, verify and execute ClamAV bytecode.

  *

- *  Copyright (C) 2009-2010 Sourcefire, Inc.

+ *  Copyright (C) 2009-2012 Sourcefire, Inc.

  *

  *  Authors: Török Edvin

  *

@@ -47,29 +47,30 @@ enum bc_state {

 };

 struct cli_bc {

-  struct bytecode_metadata metadata;

-  unsigned id;

-  unsigned kind;

-  unsigned num_types;

-  unsigned num_func;

-  struct cli_bc_func *funcs;

-  struct cli_bc_type *types;

-  uint64_t **globals;

-  uint16_t *globaltys;

-  size_t num_globals;

-  enum bc_state state;

-  struct bitset_tag *uses_apis;

-  char *lsig;

-  char *vnameprefix;

-  char **vnames;

-  unsigned vnames_cnt;

-  uint16_t start_tid;

-  struct cli_bc_dbgnode *dbgnodes;

-  unsigned dbgnode_cnt;

-  unsigned hook_lsig_id;

-  unsigned trusted;

-  uint32_t numGlobalBytes;

-  uint8_t *globalBytes;

+    struct bytecode_metadata metadata;

+    unsigned id;

+    unsigned kind;

+    unsigned num_types;

+    unsigned num_func;

+    struct cli_bc_func *funcs;

+    struct cli_bc_type *types;

+    uint64_t **globals;

+    uint16_t *globaltys;

+    size_t num_globals;

+    enum bc_state state;

+    struct bitset_tag *uses_apis;

+    char *lsig;

+    char *vnameprefix;

+    char **vnames;

+    unsigned vnames_cnt;

+    uint16_t start_tid;

+    struct cli_bc_dbgnode *dbgnodes;

+    unsigned dbgnode_cnt;

+    unsigned hook_lsig_id;

+    unsigned trusted;

+    uint32_t numGlobalBytes;

+    uint8_t *globalBytes;

+    uint32_t sigtime_id, sigmatch_id;

 };

 struct cli_all_bc {

@@ -107,7 +108,7 @@ extern int have_clamjit;

 }

 #endif

 int cli_bytecode_init(struct cli_all_bc *allbc);

-int cli_bytecode_load(struct cli_bc *bc, FILE *f, struct cli_dbio *dbio, int security);

+int cli_bytecode_load(struct cli_bc *bc, FILE *f, struct cli_dbio *dbio, int security, int sigperf);

 int cli_bytecode_prepare2(struct cl_engine *engine, struct cli_all_bc *allbc, unsigned dconfmask);

 int cli_bytecode_run(const struct cli_all_bc *bcs, const struct cli_bc *bc, struct cli_bc_ctx *ctx);

 void cli_bytecode_destroy(struct cli_bc *bc);

@@ -141,7 +142,8 @@ void cli_bytecode_context_set_trace(struct cli_bc_ctx*, unsigned level,

 				    bc_dbg_callback_trace_op,

 				    bc_dbg_callback_trace_val,

 				    bc_dbg_callback_trace_ptr);

-

+void cli_sigperf_print(void);

+void cli_sigperf_events_destroy(void);

 #ifdef __cplusplus

 }

 #endif

@@ -120,6 +120,7 @@ typedef enum {

 #define CL_DB_SIGNED	    0x4000  /* internal */

 #define CL_DB_BYTECODE_UNSIGNED	0x8000

 #define CL_DB_UNSIGNED	    0x10000 /* internal */

+#define CL_DB_BYTECODE_STATS 0x20000

 /* recommended db settings */

 #define CL_DB_STDOPT	    (CL_DB_PHISHING | CL_DB_PHISHING_URLS | CL_DB_BYTECODE)

@@ -1,7 +1,7 @@

 /*

  *  (bytecode) events

  *

- *  Copyright (C) 2010 Sourcefire, Inc.

+ *  Copyright (C) 2010-2012 Sourcefire, Inc.

  *

  *  Authors: Török Edvin

  *

@@ -138,6 +138,14 @@ static inline void ev_chain(cli_events_t *ctx, struct cli_event *ev, union ev_va

     ev->count++;

 }

+const char * cli_event_get_name(cli_events_t *ctx, unsigned id)

+{

+    struct cli_event *ev = get_event(ctx, id);

+    if (!ev)

+	return NULL;

+    return ev->name;

+}

+

 void cli_event_int(cli_events_t *ctx, unsigned id, uint64_t arg)

 {

     struct cli_event *ev = get_event(ctx, id);

@@ -1,7 +1,7 @@

 /*

  *  (bytecode) events

  *

- *  Copyright (C) 2010 Sourcefire, Inc.

+ *  Copyright (C) 2010-2012 Sourcefire, Inc.

  *

  *  Authors: Török Edvin

  *

@@ -83,6 +83,8 @@ void cli_event_count(cli_events_t *ctx, unsigned id);

 void cli_event_get(cli_events_t* ctx, unsigned id, union ev_val *val, uint32_t *count);

+const char * cli_event_get_name(cli_events_t* ctx, unsigned id);

+

 /* print all recorded events */

 void cli_event_debug_all(cli_events_t *ctx);

@@ -44,6 +44,9 @@ CLAMAV_PUBLIC {

 };

 CLAMAV_PRIVATE {

   global:

+    cli_sigperf_print; 

+    cli_sigperf_events_destroy; 

+

     cli_gettmpdir;

     cli_strtok;

     cli_strtokenize;

@@ -1,5 +1,5 @@

 /*

- *  Copyright (C) 2007-2010 Sourcefire, Inc.

+ *  Copyright (C) 2007-2012 Sourcefire, Inc.

  *

  *  Authors: Tomasz Kojm

  *

@@ -1487,7 +1487,7 @@ static int cli_loadcbc(FILE *fs, struct cl_engine *engine, unsigned int *signo,

 	    security_trust = 0;

     }

-    rc = cli_bytecode_load(bc, fs, dbio, security_trust);

+    rc = cli_bytecode_load(bc, fs, dbio, security_trust, options&CL_DB_BYTECODE_STATS);

     /* read remainder of DB, needed because cvd.c checks that we read the entire

      * file */

     while (cli_dbgets(buf, sizeof(buf), fs, dbio)) {}

@@ -1,5 +1,5 @@

 /*

- *  Copyright (C) 2008-2009 Sourcefire, Inc.

+ *  Copyright (C) 2008-2012 Sourcefire, Inc.

  *

  *  Author: Tomasz Kojm <tkojm@clamav.net>

  *

@@ -276,7 +276,9 @@ const struct clam_option __clam_options[] = {

     { "BytecodeMode", "bytecode-mode", 0, TYPE_STRING, "^(Auto|ForceJIT|ForceInterpreter|Test)$", -1, "Auto", FLAG_REQUIRED, OPT_CLAMD | OPT_CLAMSCAN,

 	"Set bytecode execution mode.\nPossible values:\n\tAuto - automatically choose JIT if possible, fallback to interpreter\nForceJIT - always choose JIT, fail if not possible\nForceIntepreter - always choose interpreter\nTest - run with both JIT and interpreter and compare results. Make all failures fatal.","Auto"},

-    { "DetectPUA", "detect-pua", 0, TYPE_BOOL, MATCH_BOOL, 0, NULL, 0, OPT_CLAMD | OPT_CLAMSCAN, "Detect Potentially Unwanted Applications.", "yes" },

+    { "BytecodeStatistics", "bytecode-statistics", 0, TYPE_BOOL, MATCH_BOOL, 0, NULL, 0, OPT_CLAMSCAN, "Collect and print bytecode execution statistics.", "no" },

+

+   { "DetectPUA", "detect-pua", 0, TYPE_BOOL, MATCH_BOOL, 0, NULL, 0, OPT_CLAMD | OPT_CLAMSCAN, "Detect Potentially Unwanted Applications.", "yes" },

     { "ExcludePUA", "exclude-pua", 0, TYPE_STRING, NULL, -1, NULL, FLAG_MULTIPLE, OPT_CLAMD | OPT_CLAMSCAN, "Exclude a specific PUA category. This directive can be used multiple times.\nSee http://www.clamav.net/support/pua for the complete list of PUA\ncategories.", "NetTool\nPWTool" },

@@ -1,7 +1,7 @@

 /*

  *  Unit tests for bytecode functions. 

  *

- *  Copyright (C) 2009 Sourcefire, Inc.

+ *  Copyright (C) 2009-2012 Sourcefire, Inc.

  *

  *  Authors: Török Edvin

  *

@@ -87,7 +87,7 @@ static void runtest(const char *file, uint64_t expected, int fail, int nojit,

     bcs.all_bcs = &bc;

     bcs.count = 1;

-    rc = cli_bytecode_load(&bc, f, NULL, 1);

+    rc = cli_bytecode_load(&bc, f, NULL, 1, 0);

     fail_unless(rc == CL_SUCCESS, "cli_bytecode_load failed");

     fclose(f);