@@ -2733,9 +2733,15 @@ static int cli_loadopenioc(FILE *fs, const char *dbname, struct cl_engine *engin

 }

 #ifndef _WIN32

-#define YARATARGET0 "Target:0;"

-#define YARATARGET1 "Target:1;"

-#define EPSTR "EP+0:"

+#define YARA_DEBUG 1

+#if YARA_DEBUG == 2

+#define cli_yaramsg(...) cli_errmsg(__VA_ARGS__)

+#elseif YARA_DEBUG == 1

+#define cli_yaramsg(...) cli_dbgmsg(__VA_ARGS__)

+#else

+#define cli_yaramsg(...) 

+#endif

+

 static char *parse_yara_hex_string(YR_STRING *string);

 static char *parse_yara_hex_string(YR_STRING *string)

@@ -2805,250 +2811,362 @@ static char *parse_yara_hex_string(YR_STRING *string)

     return res;

 }

+int ytable_add(char ***ytablep, uint32_t *ytbl_cntp)

+{

+}

+

+uint32_t ytable_lookup(char **ytable, uint32_t ytbl_cnt)

+{

+}

+

+void ytable_delete(char **ytable, uint32_t ytbl_cnt)

+{

+    uint32_t i;

+

+    if (ytable) {

+        for (i = 0; i < ytbl_cnt; ++i)

+            free(ytable[i]);

+        free(ytable);

+    }

+}

+

 static unsigned int yara_total, yara_malform, yara_complex; 

+#define YARATARGET0 "Target:0"

+#define YARATARGET1 "Target:1"

+#define EPSTR "EP+0:"

-static int cli_loadyara(FILE *fs, const char *dbname, struct cl_engine *engine, unsigned int options, struct cli_dbio *dbio)

+/* yara has no apparent cap on the number of strings; TODO - should we have one? */

+/* function base off load_oneldb */

+static int load_oneyara(YR_RULE *rule, struct cl_engine *engine, unsigned int options, unsigned int *sigs)

 {

-    char buffer[FILEBUFF];

-    char * current_rule = NULL;

-    char * current_meta = NULL;

-    char * current_string = NULL;

-    char * current_condition = NULL;

-    int rc = CL_SUCCESS;

-    uint32_t line = 0;

-    uint8_t is_comment;

-    uint8_t rule_state;

-    YR_COMPILER compiler = {0};

-    YR_RULE * rule;

-    YR_STRING * string;

-    size_t nstrings, allstringsize, totsize;

-    char *rulestr, *ruledup;

-    unsigned int sigs, i;

-    uint8_t has_short_string;

-    char * exp_op = "|";

+    YR_STRING *string;

+    int str_error = 0, i = 0, ret = CL_SUCCESS;

+    struct cli_lsig_tdb tdb;

+    uint32_t lsigid[2];

+    struct cli_matcher *root;

+    struct cli_ac_lsig **newtable, *lsig;

+    unsigned short target = 0;

+    size_t lsize;

+    char *logic = NULL;

+    char *exp_op = "|";

+    char *offset;

-    if((rc = cli_initroots(engine, options)))

-        return rc;

+    char **ytable = NULL;

+    uint32_t ytbl_cnt = 0;

-    compiler.last_result = ERROR_SUCCESS;

-    STAILQ_INIT(&compiler.rules);

-    STAILQ_INIT(&compiler.current_rule_strings);

+    cli_yaramsg("called load_oneyara()\n");

-    //    cli_errmsg("Loading yara signatures\n");

-#if 1 /* for compilation */

-    yr_lex_parse_rules_file(fs, &compiler);

-#endif

-    while (!STAILQ_EMPTY(&compiler.rules)) {

-        yara_total++;

-        sigs=0;

-        nstrings=0;

-        rule = STAILQ_FIRST(&compiler.rules);

-        STAILQ_REMOVE(&compiler.rules, rule, _yc_rule, link);

+    if (!rule) {

+        cli_errmsg("load_oneyara: empty rule passed as argument\n");

+        return CL_ENULLARG;

+    }

-        /* First find out how long our dynamically-build ldb string should be */

-        allstringsize = 0;

-        totsize = strlen(rule->id) + 2 + strlen(YARATARGET0);

-        STAILQ_FOREACH(string, &rule->strings, link) {

-            nstrings++;

-            if (STRING_IS_HEX(string)) {

-                size_t len = strlen(string->string);

-                for (i=0; i < len; i++) {

-                    int ch = string->string[i];

-                    if (!isspace(ch))

-                        allstringsize++;

-                }

-            } else if (STRING_IS_REGEXP(string)) {

-                char *trigger = PCRE_BYPASS;

-                allstringsize += strlen(trigger);

-                allstringsize += (strlen(string->string)+2); /* 2 from delimiters */

-            } else if (STRING_IS_NO_CASE(string)) {

-                allstringsize += (strlen(string->string)+2); /* 2 from delimiter+option */

-            } else {

-                allstringsize += strlen(string->string);

+    /* PUA stuff, for the future

+    if (chkpua && cli_chkpua(rule->id, engine->pua_cats, options))

+        return CL_SUCCESS;

+

+    if (engine->ignored && cli_chkign(engine->ignored, rule->id, buffer_cpy ? buffer_cpy : rule->id)) {

+        if(skip)

+            *skip = 1;

+        return CL_SUCCESS;

+    }

+    */

+

+    /*** rule specific checks ***/

+    if (RULE_IS_PRIVATE(rule) || RULE_IS_GLOBAL(rule) || RULE_IS_NULL(rule) ||

+        ((rule->g_flags) & RULE_GFLAGS_REQUIRE_FILE) || ((rule->g_flags) & RULE_GFLAGS_REQUIRE_EXECUTABLE)) {

+

+        cli_warnmsg("load_oneyara: skipping %s due to unsupported rule gflags\n", rule->id);

+        (*sigs)--;

+        return CL_SUCCESS;

+    }

+

+    if(engine->cb_sigload && engine->cb_sigload("yara", rule->id, ~options & CL_DB_OFFICIAL, engine->cb_sigload_ctx)) {

+        cli_dbgmsg("load_oneyara: skipping %s due to callback\n", rule->id);

+        (*sigs)--;

+        return CL_SUCCESS;

+    }

+

+    /*** verification step - can clamav load it?       ***/

+    /*** initial population pass for the strings table ***/

+    STAILQ_FOREACH(string, &rule->strings, link) {

+        char *substr;

+

+        /* string type handler */

+        if (STRING_IS_NULL(string)) {

+            cli_warnmsg("load_oneyara: skipping NULL string %s\n", rule->id);

+            //str_error++; /* kill the insertion? */

+            continue;

+        } else if (STRING_IS_HEX(string)) {

+            substr = parse_yara_hex_string(string);

+            if (substr) {

+                /*

+                if (strlen(substr)/2 <= CLI_DEFAULT_AC_MINDEPTH)  //FIXME: Yara has no length minimum

+                    has_short_string = 1;

+                snprintf(rulestr+len, totsize-len, "%s", substr);

+                free(substr);

+                */

             }

-        }

-        if (!nstrings) {

-            cli_errmsg("Rule %s contains to readable strings\n", rule->id);

-#ifdef YARA_PROTO

-            yara_malform++;

-            free(rule->id);

-            free(rule);

+            cli_yaramsg("Yara hex string: \"%s\"\n", substr);

+        } else if (STRING_IS_LITERAL(string)) {

+        } else if (STRING_IS_REGEXP(string)) {

+        } else {

+            cli_warnmsg("load_oneyara: skipping undefined typed string %s\n", rule->id);

+            //str_error++; /* kill the insertion? */

             continue;

-#else

-            return CL_EMALFDB;

-#endif

         }

-        /* ldb can only handle MAX_LDB_SUBSIGS(currently 64) substrings */

-        if (nstrings > MAX_LDB_SUBSIGS) {

-            cli_errmsg("cli_loadyara: rule %s exceeds MAX_LDB_SUBSIGS maximum strings - skipping.\n", rule->id);

-            free(rule->id);

-            free(rule);

+        /* modifier handler */

+        if (STRING_IS_NO_CASE(string)) {

+        }

+        if (STRING_IS_ASCII(string)) {

+        }

+        if (STRING_IS_WIDE(string)) {

+        }

+        if (STRING_IS_FULL_WORD(string)) {

+        }

+

+        /* special modifier handler */

+        if (STRING_IS_ANONYMOUS(string)) { /* empty */ }

+

+        /* unsupported(?) modifier handler */

+        if (STRING_IS_REFERENCED(string) || STRING_IS_SINGLE_MATCH(string) || STRING_IS_FAST_HEX_REGEXP(string) ||

+            STRING_IS_CHAIN_PART(string) || STRING_IS_CHAIN_TAIL(string) || STRING_FITS_IN_ATOM(string)) {

+            cli_warnmsg("load_oneyara: skipping unsupported string %s\n", rule->id);

+            str_error++;

+            free(substr);

             continue;

         }

+    }

-        allstringsize *= 2; /* For converting to hex */

-        totsize += allstringsize;

-        totsize += (nstrings * 3); /* 3 for |; */

-        if (nstrings > 10)

-            totsize += nstrings-10;

-        totsize++;

+    if (str_error > 0) {

+        cli_errmsg("load_oneyara: clamav does not support %d input strings\n", str_error);

+        ytable_delete(ytable, ytbl_cnt);

+        (*sigs)--;

+        return CL_SUCCESS; /* TODO - kill signature instead? */

+    } else if (ytbl_cnt == 0) {

+        cli_errmsg("load_oneyara: yara contains no supported strings\n");

+        ytable_delete(ytable, ytbl_cnt);

+        (*sigs)--;

+        return CL_SUCCESS; /* TODO - kill signature instead? */

+    }

-#ifdef YARA_PROTO

-        if (rule->g_flags & RULE_EP && nstrings == 1)

-            totsize += strlen(EPSTR);

-#endif

+    /*** conditional verification step (ex. do we define too many strings versus used?)  ***/

+    /*** additional string table population (ex. offsets), second translation table pass ***/

-        rulestr = cli_calloc(totsize, sizeof(char));

-        if (!rulestr) {

-            free(rule->id);

-            free(rule);

-            rc = CL_EMEM;

-            break;

+    lsize = 3*ytbl_cnt;

+    logic = cli_calloc(lsize, sizeof(char));

+    if (!logic) {

+        cli_errmsg("load_oneyara: cannot allocate memory for logic statement\n");

+        ytable_delete(ytable, ytbl_cnt);

+        return CL_EMEM;

+    }

+

+    if (rule->g_flags & RULE_ALL && rule->g_flags & RULE_THEM)

+        exp_op = "&";

+    else {

+        exp_op = "|";

+        if ((!(rule->g_flags & RULE_ANY && rule->g_flags & RULE_THEM) && ytbl_cnt > 1) &&

+            !(rule->g_flags & RULE_EP && ytbl_cnt == 1))

+            yara_complex++;

+    }

+

+    for (i=0; i<ytbl_cnt; i++) {

+        size_t len=strlen(logic);

+        snprintf(logic+len, lsize-len, "%u%s", i, (i+1 == ytbl_cnt) ? "" : exp_op);

+    }    

+

+    /*** END CONDITIONAL HANDLING ***/

+

+    /* TDB */

+    memset(&tdb, 0, sizeof(tdb));

+#ifdef USE_MPOOL

+    tdb.mempool = engine->mempool;

+#endif

+    if((ret = lsigattribs(YARATARGET0, &tdb))) {

+        FREE_TDB(tdb);

+        if(ret == 1) {

+            cli_dbgmsg("load_oneyara: Not supported attribute(s) in logical signature for %s, skipping\n", rule->id);

+            ytable_delete(ytable, ytbl_cnt);

+            free(logic);

+            (*sigs)--;

+            return CL_SUCCESS;

         }

+        return CL_EMALFDB;

+    }

-#ifdef YARA_PROTO

-        if (rule->g_flags & RULE_ALL && rule->g_flags & RULE_THEM)

-            exp_op = "&";

-        else {

-            exp_op = "|";

-            if ((!(rule->g_flags & RULE_ANY && rule->g_flags & RULE_THEM) && nstrings > 1) &&

-                !(rule->g_flags & RULE_EP && nstrings == 1))

-                yara_complex++;

+    if(tdb.engine) {

+        if(tdb.engine[0] > cl_retflevel()) {

+            cli_dbgmsg("load_oneyara: Signature for %s not loaded (required f-level: %u)\n", rule->id, tdb.engine[0]);

+            FREE_TDB(tdb);

+            ytable_delete(ytable, ytbl_cnt);

+            free(logic);

+            (*sigs)--;

+            return CL_SUCCESS;

+        } else if(tdb.engine[1] < cl_retflevel()) {

+            FREE_TDB(tdb);

+            ytable_delete(ytable, ytbl_cnt);

+            free(logic);

+            (*sigs)--;

+            return CL_SUCCESS;

         }

-        if (rule->g_flags & RULE_EP)

-            sprintf(rulestr, "%s;%s(", rule->id, YARATARGET1);

-        else

-#endif

-        sprintf(rulestr, "%s;%s(", rule->id, YARATARGET0);

-        for (i=0; i<nstrings; i++) {

-            size_t len=strlen(rulestr);

-            snprintf(rulestr+len, totsize-len, "%u%s", i, (i+1 == nstrings) ? "" : exp_op);

+    }

+

+    if(!tdb.target) {

+        cli_errmsg("load_oneyara: No target specified in TDB\n");

+        FREE_TDB(tdb);

+        ytable_delete(ytable, ytbl_cnt);

+        free(logic);

+        return CL_EMALFDB;

+    } else if(tdb.target[0] >= CLI_MTARGETS) {

+        cli_dbgmsg("load_oneyara: Not supported target type in logical signature for %s, skipping\n", rule->id);

+        FREE_TDB(tdb);

+        ytable_delete(ytable, ytbl_cnt);

+        free(logic);

+        (*sigs)--;

+        return CL_SUCCESS;

+    }

+

+    if((tdb.icongrp1 || tdb.icongrp2) && tdb.target[0] != 1) {

+        cli_errmsg("load_oneyara: IconGroup is only supported in PE (target 1) signatures\n");

+        FREE_TDB(tdb);

+        ytable_delete(ytable, ytbl_cnt);

+        free(logic);

+        return CL_EMALFDB;

+    }

+

+    if((tdb.ep || tdb.nos) && tdb.target[0] != 1 && tdb.target[0] != 6 && tdb.target[0] != 9) {

+        cli_errmsg("load_oneyara: EntryPoint/NumberOfSections is only supported in PE/ELF/Mach-O signatures\n");

+        FREE_TDB(tdb);

+        ytable_delete(ytable, ytbl_cnt);

+        free(logic);

+        return CL_EMALFDB;

+    }

+

+    /*** populating lsig ***/

+    root = engine->root[tdb.target[0]];

+

+    lsig = (struct cli_ac_lsig *) mpool_calloc(engine->mempool, 1, sizeof(struct cli_ac_lsig));

+    if(!lsig) {

+        cli_errmsg("load_oneyara: Can't allocate memory for lsig\n");

+        FREE_TDB(tdb);

+        ytable_delete(ytable, ytbl_cnt);

+        free(logic);

+        return CL_EMEM;

+    }

+

+    if (logic) {

+        lsig->type = CLI_NORMAL_LSIG;

+        lsig->u.logic = cli_mpool_strdup(engine->mempool, logic);

+        if(!lsig->u.logic) {

+            cli_errmsg("load_oneyara: Can't allocate memory for lsig->logic\n");

+            FREE_TDB(tdb);

+            ytable_delete(ytable, ytbl_cnt);

+            free(logic);

+            mpool_free(engine->mempool, lsig);

+            return CL_EMEM;

         }

+    } else {

+            cli_errmsg("load_oneyara: Unsupported logic type\n");

+            FREE_TDB(tdb);

+            ytable_delete(ytable, ytbl_cnt);

+            free(logic);

+            mpool_free(engine->mempool, lsig);

+            return CL_EMEM;

+    }

+    free(logic);

-        strcat(rulestr, ");");

+    lsigid[0] = lsig->id = root->ac_lsigs;

-        has_short_string = 0;

-        int dcount = 0;

-        while (!STAILQ_EMPTY(&rule->strings)) {

-            string = STAILQ_FIRST(&rule->strings);

-            STAILQ_REMOVE(&rule->strings, string, _yc_string, link);

+    root->ac_lsigs++;

+    newtable = (struct cli_ac_lsig **) mpool_realloc(engine->mempool, root->ac_lsigtable, root->ac_lsigs * sizeof(struct cli_ac_lsig *));

+    if(!newtable) {

+        root->ac_lsigs--;

+        cli_errmsg("cli_loadldb: Can't realloc root->ac_lsigtable\n");

+        FREE_TDB(tdb);

+        ytable_delete(ytable, ytbl_cnt);

+        mpool_free(engine->mempool, lsig);

+        return CL_EMEM;

+    }

-#if 0

-            cli_errmsg("%i:", ++dcount);

-#endif

-#ifdef YARA_PROTO

-            if (rule->g_flags & RULE_EP && nstrings == 1) {

-                size_t len = strlen(rulestr);

-                snprintf(rulestr+len, totsize-len, "%s", EPSTR);

-            }

-#endif

+    newtable[root->ac_lsigs - 1] = lsig;

+    root->ac_lsigtable = newtable;

+    tdb.subsigs = ytbl_cnt;

-            if (STRING_IS_HEX(string)) {

-                char *substr;

-                size_t len = strlen(rulestr);

+    /*** loading step - put things into the AC trie ***/

+    for (i = 0; i < ytbl_cnt; ++i) {

+        lsigid[1] = i;

-                substr = parse_yara_hex_string(string);

-#if 0

-                cli_errmsg("Yara hex string: \"%s\"\n", substr);

-#endif

-                if (substr) {

-                    if (strlen(substr)/2 <= CLI_DEFAULT_AC_MINDEPTH)  //FIXME: Yara has no length minimum

-                        has_short_string = 1;

-                    snprintf(rulestr+len, totsize-len, "%s", substr);

-                    free(substr);

-                }

-            } else if (STRING_IS_REGEXP(string)) {

-                size_t len = strlen(rulestr);

-#if 0

-                cli_errmsg("Yara regex string: \"%s\"\n", string->string);

-#endif

-                snprintf(rulestr+len, totsize-len, "%s/%s/", PCRE_BYPASS, string->string);

-            } else if (STRING_IS_NO_CASE(string)) {

-                size_t len;

-#if 0

-                cli_errmsg("Yara nocase string: \"%s\"\n", string->string);

-#endif

-                if (strlen(string->string) <= CLI_DEFAULT_AC_MINDEPTH) //FIXME: Yara has no length minimum

-                    has_short_string = 1;

-                for (i=0; i < strlen(string->string); i++) {

-                    len = strlen(rulestr);

-                    snprintf(rulestr+len, totsize-len, "%02x", string->string[i]);

-                }

+        /* TODO - offsets as separate table or integrated into ytable[i]? */

+        offset = "*";

-                len = strlen(rulestr);

-                snprintf(rulestr+len, totsize-len, "/i");

-            } else {

-#if 0

-                cli_errmsg("Yara literal string: \"%s\"\n", string->string);

-#endif

-                if (strlen(string->string) <= CLI_DEFAULT_AC_MINDEPTH) //FIXME: Yara has no length minimum

-                    has_short_string = 1;

-                for (i=0; i < strlen(string->string); i++) {

-                    size_t len = strlen(rulestr);

-                    snprintf(rulestr+len, totsize-len, "%02x", string->string[i]);

-                }

-            }

+        /* TODO - options as separate table or integrated into ytable[i]? */

-            if (!STAILQ_EMPTY(&rule->strings))

-                strcat(rulestr, ";");

+        if((ret = cli_parse_add(root, rule->id, ytable[i], NULL, 0, 0, offset, target, lsigid, options)))

+            return ret;

+    }

-            free(string->id);

-            free(string->string);

-            free(string);

-        }

+    memcpy(&lsig->tdb, &tdb, sizeof(tdb));

+    ytable_delete(ytable, ytbl_cnt);

+    return CL_SUCCESS;

+}

-        if (rulestr[strlen(rulestr)-1] == ';')

-            rulestr[strlen(rulestr)-1] = '\0';

+//TODO - pua? dbio?

+static int cli_loadyara(FILE *fs, struct cl_engine *engine, unsigned int *signo, unsigned int options, struct cli_dbio *dbio, const char *dbname)

+{

+    YR_COMPILER compiler = {0};

+    YR_RULE *rule;

+    unsigned int sigs = 0, rules = 0;

+    int rc;

-#if 0

-        printf("[+] computed ldb: \"%s\"\n", rulestr);

-#endif

-        ruledup = cli_malloc(strlen(rulestr)+1);

-        if (!ruledup) {

-            free(rulestr);

-            free(rule->id);

-            free(rule);

-            rc = CL_EMEM;

-            break;

-        }

+    UNUSEDPARAM(dbio);

-        strcpy(ruledup, rulestr);

+    if((rc = cli_initroots(engine, options)))

+        return rc;

-        if (has_short_string == 0) {

-#if 1

-            rc = load_oneldb(rulestr,

-                 engine->pua_cats && (options & CL_DB_PUA_MODE) && (options & (CL_DB_PUA_INCLUDE | CL_DB_PUA_EXCLUDE)),

-                 engine, options, rule->id, line++, &sigs, 0, ruledup, NULL);

-#endif

+    compiler.last_result = ERROR_SUCCESS;

+    STAILQ_INIT(&compiler.rules);

+    STAILQ_INIT(&compiler.current_rule_strings);

-        }

-        else {

-            cli_errmsg("cli_loadyara: has short strings, rule %s excluded\n", rulestr);

-        }

-#if 0

-        printf("totsize: %zu\treal size: %zu\n", totsize, strlen(ruledup));

-#endif

+    rc = yr_lex_parse_rules_file(fs, &compiler);

+    if (rc > 0) { /* rc = number of errors */

+        /* TODO - handle the various errors? */

+        cli_errmsg("cli_loadyara: failed to parse rules file\n");

+        return CL_EMALFDB;

+    }

-        free(rulestr);

-        free(ruledup);

-        free(rule->id);

-        free(rule);

-#ifdef YARA_PROTO

-        if (rc == CL_EMALFDB) {

-            yara_malform++;

-            cli_errmsg("+++++++++++++ load_oneldb returns CL_EMALFORMDB +++++++++++++\n");

-            rc = CL_SUCCESS;

-            continue;

-        }

-#endif

+    while (!STAILQ_EMPTY(&compiler.rules)) {

+        rule = STAILQ_FIRST(&compiler.rules);

+        STAILQ_REMOVE(&compiler.rules, rule, _yc_rule, link);

+

+        rules++;

+        sigs++;

+

+        /* TODO - PUA and engine->ignored */

+        rc = load_oneyara(rule, engine, options, &sigs);

         if (rc != CL_SUCCESS)

             break;

     }

-    return rc;

+    if(rc) {

+        cli_errmsg("cli_loadyara: problem parsing yara rule %s\n", rule->id);

+        return rc;

+    }

+

+    if(!rules) {

+        cli_errmsg("cli_loadyara: empty database file\n");

+        return CL_EMALFDB;

+    }

+

+    if(signo)

+        *signo += sigs;

+

+    cli_yaramsg("Successfully loaded %u of %u yara signatures from %s\n", sigs, rules, dbname);

+

+    return CL_SUCCESS;

 }

+

 #endif

 static int cli_loaddbdir(const char *dirname, struct cl_engine *engine, unsigned int *signo, unsigned int options);

@@ -3183,7 +3301,7 @@ int cli_load(const char *filename, struct cl_engine *engine, unsigned int *signo

 	ret = cli_loadopenioc(fs, dbname, engine, options);

 #ifndef _WIN32 /* FIXME: temp until clam yara for windows */

     } else if(cli_strbcasestr(dbname, ".yar") || cli_strbcasestr(dbname, ".yara")) {

-        ret = cli_loadyara(fs, dbname, engine, options, dbio);

+        ret = cli_loadyara(fs, engine, signo, options, dbio, dbname);

 #endif

     } else {

 	cli_dbgmsg("cli_load: unknown extension - assuming old database format\n");