/*

  *  Copyright (C) 2013-2019 Cisco Systems, Inc. and/or its affiliates. All rights reserved.

  *  Copyright (C) 2007-2013 Sourcefire, Inc.

  *  Copyright (C) 2002-2007 Tomasz Kojm <tkojm@clamav.net>

  *

  *  Authors: Tomasz Kojm

  *
  *  This program is free software; you can redistribute it and/or modify

  *  it under the terms of the GNU General Public License version 2 as
  *  published by the Free Software Foundation.

  *
  *  This program is distributed in the hope that it will be useful,
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  *  GNU General Public License for more details.
  *
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, write to the Free Software

  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  *  MA 02110-1301, USA.

  */
 

 #if HAVE_CONFIG_H
 #include "clamav-config.h"
 #endif
 

 #include <stdio.h>

 #include <stdint.h>

 #include <stdlib.h>
 #include <string.h>
 #include <ctype.h>

 #ifdef HAVE_UNISTD_H

 #include <unistd.h>

 #endif

 #include <dirent.h>
 #include <sys/types.h>
 #include <sys/stat.h>

 #ifdef HAVE_SYS_PARAM_H

 #include <sys/param.h>

 #endif

 #include <fcntl.h>

 #include <zlib.h>

 #include <errno.h>

 
 #include "clamav.h"

 #include "cvd.h"

 #ifdef HAVE_STRINGS_H

 #include <strings.h>
 #endif

 #include "matcher-ac.h"
 #include "matcher-bm.h"

 #include "matcher-pcre.h"

 #include "matcher-byte-comp.h"

 #include "matcher-hash.h"

 #include "matcher.h"

 #include "others.h"
 #include "str.h"

 #include "dconf.h"

 #include "filetypes.h"
 #include "filetypes_int.h"

 #include "readdb.h"

 #include "default.h"

 #include "dsig.h"

 #include "asn1.h"

 #include "phishcheck.h"

 #include "phish_whitelist.h"
 #include "phish_domaincheck_db.h"

 #include "regex_list.h"

 #include "hashtab.h"

 #include "mpool.h"

 #include "bytecode.h"

 #include "bytecode_api.h"

 #include "bytecode_priv.h"

 #include "cache.h"

 #include "openioc.h"

 #ifdef CL_THREAD_SAFE

 #include <pthread.h>

 static pthread_mutex_t cli_ref_mutex = PTHREAD_MUTEX_INITIALIZER;
 #endif

 
 #ifdef HAVE_YARA

 #include "yara_clam.h"

 #include "yara_compiler.h"

 #include "yara_grammar.h"
 #include "yara_lexer.h"

 #endif

 #define MAX_LDB_SUBSIGS 64
 

 char *cli_virname(const char *virname, unsigned int official)

 {

     char *newname, *pt;

     if (!virname)
         return NULL;

     if ((pt = strstr(virname, " (Clam)")))
         *pt = '\0';

     if (!virname[0]) {
         cli_errmsg("cli_virname: Empty virus name\n");
         return NULL;

     }
 

     if (official)

         return cli_strdup(virname);

     newname = (char *)cli_malloc(strlen(virname) + 11 + 1);
     if (!newname) {
         cli_errmsg("cli_virname: Can't allocate memory for newname\n");
         return NULL;

     }

     sprintf(newname, "%s.UNOFFICIAL", virname);
     return newname;

 }
 

 cl_error_t cli_sigopts_handler(struct cli_matcher *root, const char *virname, const char *hexsig, uint8_t sigopts, uint16_t rtype, uint16_t type, const char *offset, uint8_t target, const uint32_t *lsigid, unsigned int options)

 {

     char *hexcpy, *start, *end, *mid;

     unsigned int i;
     int ret = CL_SUCCESS;

     /*

      * cyclic loops with cli_parse_add are impossible now as cli_parse_add

      * no longer calls cli_sigopts_handler; leaving here for safety

      */

     if (sigopts & ACPATT_OPTION_ONCE) {

         cli_errmsg("cli_sigopts_handler: invalidly called multiple times!\n");

         return CL_EPARSE;
     }
 
     hexcpy = cli_strdup(hexsig);
     if (!hexcpy)
         return CL_EMEM;

     sigopts |= ACPATT_OPTION_ONCE;

     /* REGEX testing and sigopt handling */
     start = strchr(hexcpy, '/');

     end   = strrchr(hexcpy, '/');

 
     if (start != end) {
         /* FULLWORD regex sigopt handling */

         if (sigopts & ACPATT_OPTION_FULLWORD) {

             size_t ovrlen = strlen(hexcpy) + 21;
             char *hexovr  = cli_calloc(ovrlen, sizeof(char));

             if (!hexovr) {
                 free(hexcpy);

                 return CL_EMEM;

             }

             *start++ = '\0';

             *end++   = '\0';

             snprintf(hexovr, ovrlen, "%s/([\\W_]|\\A)%s([\\W_]|\\Z)/%s", hexcpy, start, end);

 
             free(hexcpy);
             hexcpy = hexovr;
         }

         /* NOCASE sigopt is passed onto the regex-opt handler */
         if (sigopts & ACPATT_OPTION_NOCASE) {

             size_t ovrlen = strlen(hexcpy) + 2;
             char *hexovr  = cli_calloc(ovrlen, sizeof(char));

             if (!hexovr) {
                 free(hexcpy);

                 return CL_EMEM;

             }

             snprintf(hexovr, ovrlen, "%si", hexcpy);

             free(hexcpy);
             hexcpy = hexovr;
         }
         /* WIDE sigopt is unsupported */
         if (sigopts & ACPATT_OPTION_WIDE) {
             cli_errmsg("cli_parse_add: wide modifier [w] is not supported for regex subsigs\n");

             free(hexcpy);

             return CL_EMALFDB;
         }

         ret = cli_parse_add(root, virname, hexcpy, sigopts, rtype, type, offset, target, lsigid, options);
         free(hexcpy);
         return ret;
     }

     /* BCOMP sigopt handling */
     start = strchr(hexcpy, '#');

     end   = strrchr(hexcpy, '#');
     mid   = strchr(hexcpy, '(');

     if (start != end && mid && (*(++mid) == '#' || !strncmp(mid, ">>", 2) || !strncmp(mid, "<<", 2) || !strncmp(mid, "0#", 2))) {

         /* TODO byte compare currently does not have support for sigopts, pass through */
         ret = cli_parse_add(root, virname, hexcpy, sigopts, rtype, type, offset, target, lsigid, options);
         free(hexcpy);
         return ret;
     }
 

     /* NORMAL HEXSIG sigopt handling */
     /* FULLWORD sigopt handling - only happens once */
     if (sigopts & ACPATT_OPTION_FULLWORD) {
         char *rechar;

         size_t ovrlen = strlen(hexcpy) + 7;
         char *hexovr  = cli_calloc(ovrlen, sizeof(char));

         if (!hexovr) {
             free(hexcpy);
             return CL_EMEM;
         }
 
         snprintf(hexovr, ovrlen, "(W)%s(W)", hexcpy);
 
         /* change the '[' and ']' to '{' and '}' since there are now two bytes */
         rechar = hexovr;
         while ((rechar = strchr(rechar, '['))) { //TEST TODO
             *rechar = '{';
 
             if (!(rechar = strchr(rechar, ']'))) {
                 cli_errmsg("cli_parse_add: unmatched '[' in signature %s\n", virname);

                 free(hexcpy);

                 free(hexovr);
                 return CL_EMALFDB;

             }

             *rechar = '}';

         }

 
         free(hexcpy);

         hexcpy = hexovr;
     }
 
     /* WIDE sigopt handling - only happens once (after fullword)
      * TODO - consider handling in cli_ac_addpatt? (two pattern possibility)
      */
     if (sigopts & ACPATT_OPTION_WIDE) {

         size_t ovrlen = 2 * strlen(hexcpy) + 1;
         char *hexovr  = cli_calloc(ovrlen, sizeof(char));

         if (!hexovr) {
             free(hexcpy);
             return CL_EMEM;
         }
 
         /* clamav-specific wildcards need to be handled here! */
         for (i = 0; i < strlen(hexcpy); ++i) {

             size_t len = strlen(hexovr);

 
             if (hexcpy[i] == '*' || hexcpy[i] == '|' || hexcpy[i] == ')') {
                 hexovr[len] = hexcpy[i];
             } else if (hexcpy[i] == '[') {
                 /* change the '[' and ']' to '{' and '}' since there are now two bytes */
                 hexovr[len++] = '{';
                 ++i;
                 while (i < strlen(hexcpy) && hexcpy[i] != ']')
                     hexovr[len++] = hexcpy[i++];
 
                 hexovr[len] = '}';
             } else if (hexcpy[i] == '{') {
                 while (i < strlen(hexcpy) && hexcpy[i] != '}')
                     hexovr[len++] = hexcpy[i++];
 
                 hexovr[len] = '}';
             } else if (hexcpy[i] == '!' || hexcpy[i] == '(') {
                 if (hexcpy[i] == '!')
                     hexovr[len++] = hexcpy[i++];
 
                 /* copies '(' */
                 hexovr[len] = hexcpy[i];
 

                 if (hexcpy[i + 1] == 'B' || hexcpy[i + 1] == 'L' || hexcpy[i + 1] == 'W') {
                     ++len;
                     ++i;

                     hexovr[len++] = hexcpy[i++];
                     if (hexcpy[i] != ')') {
                         free(hexcpy);
                         free(hexovr);
                         return CL_EMALFDB;
                     }
                     hexovr[len] = hexcpy[i];
                 }
             } else {
                 //snprintf(hexovr+len, ovrlen-len, "%02x%c%c", 0, hexcpy[i], hexcpy[i+1]);

                 snprintf(hexovr + len, ovrlen - len, "%c%c%02x", hexcpy[i], hexcpy[i + 1], 0);

                 ++i;
             }
         }
 
         /* NOCASE sigopt is handled in cli_ac_addsig */
         ret = cli_parse_add(root, virname, hexovr, sigopts, rtype, type, offset, target, lsigid, options);
         free(hexovr);
         if (ret != CL_SUCCESS || !(sigopts & ACPATT_OPTION_ASCII)) {
             free(hexcpy);
             return ret;
         } else {
             /* disable wide sigopt for ascii variant */
             sigopts &= ~ACPATT_OPTION_WIDE;
         }

     }
 

     /* ASCII sigopt; NOCASE sigopt is handled in cli_ac_addsig */
     ret = cli_parse_add(root, virname, hexcpy, sigopts, rtype, type, offset, target, lsigid, options);
     free(hexcpy);
     return ret;

 }
 

 #define PCRE_TOKENS 4

 cl_error_t cli_parse_add(struct cli_matcher *root, const char *virname, const char *hexsig, uint8_t sigopts, uint16_t rtype, uint16_t type, const char *offset, uint8_t target, const uint32_t *lsigid, unsigned int options)

 {

     struct cli_bm_patt *bm_new;

     char *pt, *hexcpy, *start = NULL, *mid = NULL, *end = NULL, *n, l, r;

     const char *wild;
     int ret, asterisk = 0, range;

     unsigned int i, j, hexlen, nest, parts = 0;

     int mindist = 0, maxdist = 0, error = 0;

     hexlen = strlen(hexsig);

     if (hexsig[0] == '$') {

         /* macro */

         unsigned int smin, smax, tid;

         struct cli_ac_patt *patt;
 

         if (hexsig[hexlen - 1] != '$') {

             cli_errmsg("cli_parseadd(): missing terminator $\n");
             return CL_EMALFDB;
         }
 
         if (!lsigid) {
             cli_errmsg("cli_parseadd(): macro signatures only valid inside logical signatures\n");
             return CL_EMALFDB;
         }
 

         if (sscanf(hexsig, "${%u-%u}%u$", &smin, &smax, &tid) != 3) {

             cli_errmsg("cli_parseadd(): invalid macro signature format\n");
             return CL_EMALFDB;
         }
 
         if (tid >= 32) {
             cli_errmsg("cli_parseadd(): only 32 macro groups are supported\n");
             return CL_EMALFDB;
         }
 

         patt = MPOOL_CALLOC(root->mempool, 1, sizeof(*patt));

         if (!patt)
             return CL_EMEM;
 
         /* this is not a pattern that will be matched by AC itself, rather it is a
          * pattern checked by the lsig code */
         patt->ch_mindist[0] = smin;
         patt->ch_maxdist[0] = smax;

         patt->sigid         = tid;
         patt->length[0]     = root->ac_mindepth;

 
         /* dummy */

         patt->pattern = MPOOL_CALLOC(root->mempool, patt->length[0], sizeof(*patt->pattern));

         if (!patt->pattern) {
             free(patt);
             return CL_EMEM;
         }
 
         if ((ret = cli_ac_addpatt(root, patt))) {

             MPOOL_FREE(root->mempool, patt->pattern);

             free(patt);
             return ret;
         }
 

         return CL_SUCCESS;

     }

     /* expected format => ^offset:trigger/regex/[cflags]$ */

     if (strchr(hexsig, '/')) {

         char *start, *end;

         const char *trigger, *pattern, *cflags;

         /* get copied */

         hexcpy = cli_strdup(hexsig);

         if (!hexcpy)

             return CL_EMEM;
 

         /* get delimiters-ed */
         start = strchr(hexcpy, '/');

         end   = strrchr(hexcpy, '/');

         /* get pcre-ed */

         if (start == end) {
             cli_errmsg("cli_parseadd(): PCRE subsig mismatched '/' delimiter\n");

             free(hexcpy);

             return CL_EMALFDB;
         }

 #if HAVE_PCRE

         /* get checked */
         if (hexsig[0] == '/') {
             cli_errmsg("cli_parseadd(): PCRE subsig must contain logical trigger\n");

             free(hexcpy);

             return CL_EMALFDB;
         }

         /* get NULL-ed */
         *start = '\0';

         *end   = '\0';

         /* get tokens-ed */
         trigger = hexcpy;

         pattern = start + 1;
         cflags  = end + 1;

         if (*cflags == '\0') /* get compat-ed */
             cflags = NULL;

         /* normal trigger, get added */
         ret = cli_pcre_addpatt(root, virname, trigger, pattern, cflags, offset, lsigid, options);
         free(hexcpy);
         return ret;

 #else

         free(hexcpy);
         cli_errmsg("cli_parseadd(): cannot parse PCRE subsig without PCRE support\n");
         return CL_EPARSE;

 #endif

     } else if ((wild = strchr(hexsig, '{'))) {
         if (sscanf(wild, "%c%u%c", &l, &range, &r) == 3 && l == '{' && r == '}' && range > 0 && range < 128) {

             hexcpy = cli_calloc(hexlen + 2 * range, sizeof(char));

             if (!hexcpy)

                 return CL_EMEM;

             strncpy(hexcpy, hexsig, wild - hexsig);

             for (i = 0; i < (unsigned int)range; i++)

                 strcat(hexcpy, "??");

             if (!(wild = strchr(wild, '}'))) {

                 cli_errmsg("cli_parse_add(): Problem adding signature: missing bracket\n");
                 free(hexcpy);
                 return CL_EMALFDB;
             }

             strcat(hexcpy, ++wild);

             ret = cli_parse_add(root, virname, hexcpy, sigopts, rtype, type, offset, target, lsigid, options);

             free(hexcpy);

             return ret;
         }

         root->ac_partsigs++;

         if (!(hexcpy = cli_strdup(hexsig)))

             return CL_EMEM;

         nest = 0;

         for (i = 0; i < hexlen; i++) {
             if (hexsig[i] == '(')

                 nest++;

             else if (hexsig[i] == ')')

                 nest--;

             else if (hexsig[i] == '{') {

                 if (nest) {
                     cli_errmsg("cli_parse_add(): Alternative match contains unsupported ranged wildcard\n");

                     free(hexcpy);

                     return CL_EMALFDB;
                 }
                 parts++;

             } else if (hexsig[i] == '*') {

                 if (nest) {
                     cli_errmsg("cli_parse_add(): Alternative match cannot contain unbounded wildcards\n");

                     free(hexcpy);

                     return CL_EMALFDB;
                 }

                 parts++;

             }
         }

         if (parts)

             parts++;
 
         start = pt = hexcpy;

         for (i = 1; i <= parts; i++) {
             if (i != parts) {
                 for (j = 0; j < strlen(start); j++) {
                     if (start[j] == '{') {

                         asterisk = 0;

                         pt       = start + j;

                         break;
                     }
 

                     if (start[j] == '*') {

                         asterisk = 1;

                         pt       = start + j;

                         break;
                     }
                 }

                 *pt++ = 0;
             }

             if ((ret = cli_ac_addsig(root, virname, start, sigopts, root->ac_partsigs, parts, i, rtype, type, mindist, maxdist, offset, lsigid, options))) {

                 cli_errmsg("cli_parse_add(): Problem adding signature (1).\n");
                 error = 1;
                 break;
             }

             if (i == parts)

                 break;

             mindist = maxdist = 0;

             if (asterisk) {

                 start = pt;
                 continue;
             }

             if (!(start = strchr(pt, '}'))) {

                 error = 1;
                 break;
             }

             *start++ = 0;

             if (!pt) {

                 error = 1;
                 break;
             }

             if (!strchr(pt, '-')) {
                 if (!cli_isnumber(pt) || (mindist = maxdist = atoi(pt)) < 0) {

                     error = 1;
                     break;
                 }
             } else {

                 if ((n = cli_strtok(pt, 0, "-"))) {
                     if (!cli_isnumber(n) || (mindist = atoi(n)) < 0) {

                         error = 1;
                         free(n);
                         break;
                     }
 
                     free(n);
                 }

                 if ((n = cli_strtok(pt, 1, "-"))) {
                     if (!cli_isnumber(n) || (maxdist = atoi(n)) < 0) {

                         error = 1;
                         free(n);
                         break;
                     }

                     free(n);
                 }

                 if ((n = cli_strtok(pt, 2, "-"))) { /* strict check */

                     error = 1;
                     free(n);
                     break;
                 }
             }
         }

         free(hexcpy);

         if (error) {

             cli_errmsg("cli_parseadd(): Problem adding signature (1b).\n");
             return CL_EMALFDB;
         }

     } else if (strchr(hexsig, '*')) {

         root->ac_partsigs++;

         nest = 0;

         for (i = 0; i < hexlen; i++) {
             if (hexsig[i] == '(')

                 nest++;

             else if (hexsig[i] == ')')

                 nest--;

             else if (hexsig[i] == '*') {

                 if (nest) {
                     cli_errmsg("cli_parse_add(): Alternative match cannot contain unbounded wildcards\n");
                     return CL_EMALFDB;
                 }

                 parts++;

             }
         }

         if (parts)

             parts++;

         for (i = 1; i <= parts; i++) {
             if ((pt = cli_strtok(hexsig, i - 1, "*")) == NULL) {

                 cli_errmsg("cli_parse_add():Can't extract part %d of partial signature.\n", i);

                 return CL_EMALFDB;
             }

             if ((ret = cli_ac_addsig(root, virname, pt, sigopts, root->ac_partsigs, parts, i, rtype, type, 0, 0, offset, lsigid, options))) {

                 cli_errmsg("cli_parse_add(): Problem adding signature (2).\n");
                 free(pt);
                 return ret;
             }
 
             free(pt);
         }

     } else if ((start = strchr(hexsig, '(')) && (mid = strchr(hexsig, '#')) && (end = strrchr(hexsig, '#')) && mid != end) {

 
         /* format seems to match byte_compare */

         if (CL_SUCCESS != (ret = cli_bcomp_addpatt(root, virname, hexsig, lsigid, options))) {

             cli_errmsg("cli_parse_add(): Problem adding signature (2b).\n");
             return ret;
         }
 

     } else if (root->ac_only || type || lsigid || sigopts || strpbrk(hexsig, "?([") || (root->bm_offmode && (!strcmp(offset, "*") || strchr(offset, ','))) || strstr(offset, "VI") || strchr(offset, '$')) {

         if (CL_SUCCESS != (ret = cli_ac_addsig(root, virname, hexsig, sigopts, 0, 0, 0, rtype, type, 0, 0, offset, lsigid, options))) {

             cli_errmsg("cli_parse_add(): Problem adding signature (3).\n");

             return ret;
         }

     } else {

         bm_new = (struct cli_bm_patt *)MPOOL_CALLOC(root->mempool, 1, sizeof(struct cli_bm_patt));

         if (!bm_new)

             return CL_EMEM;
 

         bm_new->pattern = (unsigned char *)CLI_MPOOL_HEX2STR(root->mempool, hexsig);

         if (!bm_new->pattern) {

             MPOOL_FREE(root->mempool, bm_new);

             return CL_EMALFDB;
         }

         bm_new->length = hexlen / 2;

         bm_new->virname = CLI_MPOOL_VIRNAME(root->mempool, virname, options & CL_DB_OFFICIAL);

         if (!bm_new->virname) {

             MPOOL_FREE(root->mempool, bm_new->pattern);
             MPOOL_FREE(root->mempool, bm_new);

             return CL_EMEM;
         }

         if (bm_new->length > root->maxpatlen)

             root->maxpatlen = bm_new->length;
 

         if (CL_SUCCESS != (ret = cli_bm_addpatt(root, bm_new, offset))) {

             cli_errmsg("cli_parse_add(): Problem adding signature (4).\n");

             MPOOL_FREE(root->mempool, bm_new->pattern);
             MPOOL_FREE(root->mempool, bm_new->virname);
             MPOOL_FREE(root->mempool, bm_new);

             return ret;
         }

     }
 

     return CL_SUCCESS;

 }

 cl_error_t cli_initroots(struct cl_engine *engine, unsigned int options)

 {

     int i, ret;
     struct cli_matcher *root;

     UNUSEDPARAM(options);

     for (i = 0; i < CLI_MTARGETS; i++) {
         if (!engine->root[i]) {
             cli_dbgmsg("Initializing engine->root[%d]\n", i);

             root = engine->root[i] = (struct cli_matcher *)MPOOL_CALLOC(engine->mempool, 1, sizeof(struct cli_matcher));

             if (!root) {
                 cli_errmsg("cli_initroots: Can't allocate memory for cli_matcher\n");
                 return CL_EMEM;
             }

 #ifdef USE_MPOOL

             root->mempool = engine->mempool;

 #endif

             root->type = i;
             if (cli_mtargets[i].ac_only || engine->ac_only)
                 root->ac_only = 1;
 
             cli_dbgmsg("Initializing AC pattern matcher of root[%d]\n", i);

             if (CL_SUCCESS != (ret = cli_ac_init(root, engine->ac_mindepth, engine->ac_maxdepth, engine->dconf->other & OTHER_CONF_PREFILTERING))) {

                 /* no need to free previously allocated memory here */
                 cli_errmsg("cli_initroots: Can't initialise AC pattern matcher\n");
                 return ret;
             }
 
             if (!root->ac_only) {
                 cli_dbgmsg("cli_initroots: Initializing BM tables of root[%d]\n", i);

                 if (CL_SUCCESS != (ret = cli_bm_init(root))) {

                     cli_errmsg("cli_initroots: Can't initialise BM pattern matcher\n");
                     return ret;
                 }
             }
         }

     }

     engine->root[1]->bm_offmode = 1; /* BM offset mode for PE files */

     return CL_SUCCESS;

 }
 

 char *cli_dbgets(char *buff, unsigned int size, FILE *fs, struct cli_dbio *dbio)

 {

     if (fs)
         return fgets(buff, size, fs);
 
     if (dbio->usebuf) {
         int bread;
         char *nl;
 
         while (1) {
             if (!dbio->bufpt) {
                 if (!dbio->size)
                     return NULL;
 
                 if (dbio->gzs) {
                     bread = gzread(dbio->gzs, dbio->readpt, dbio->readsize);
                     if (bread == -1) {
                         cli_errmsg("cli_dbgets: gzread() failed\n");
                         return NULL;
                     }
                 } else {
                     bread = fread(dbio->readpt, 1, dbio->readsize, dbio->fs);
                     if (!bread && ferror(dbio->fs)) {
                         cli_errmsg("cli_dbgets: fread() failed\n");
                         return NULL;
                     }
                 }
                 if (!bread)
                     return NULL;
                 dbio->readpt[bread] = 0;
                 dbio->bufpt         = dbio->buf;
                 dbio->size -= bread;
                 dbio->bread += bread;
                 if (dbio->hashctx)
                     cl_update_hash(dbio->hashctx, dbio->readpt, bread);
             }
             if (dbio->chkonly && dbio->bufpt) {
                 dbio->bufpt    = NULL;
                 dbio->readsize = dbio->size < dbio->bufsize ? dbio->size : dbio->bufsize - 1;
                 continue;
             }
             nl = strchr(dbio->bufpt, '\n');
             if (nl) {
                 if (nl - dbio->bufpt >= size) {
                     cli_errmsg("cli_dbgets: Line too long for provided buffer\n");
                     return NULL;
                 }
                 strncpy(buff, dbio->bufpt, nl - dbio->bufpt);
                 buff[nl - dbio->bufpt] = 0;
                 if (nl < dbio->buf + dbio->bufsize) {
                     dbio->bufpt = ++nl;
                 } else {
                     dbio->bufpt    = NULL;
                     dbio->readpt   = dbio->buf;
                     dbio->readsize = dbio->size < dbio->bufsize ? dbio->size : dbio->bufsize - 1;
                 }
                 return buff;
             } else {
                 unsigned int remain = dbio->buf + dbio->bufsize - 1 - dbio->bufpt;
 
                 if (dbio->bufpt == dbio->buf) {
                     cli_errmsg("cli_dbgets: Invalid data or internal buffer too small\n");
                     return NULL;
                 }
                 memmove(dbio->buf, dbio->bufpt, remain);
                 dbio->readpt   = dbio->buf + remain;
                 dbio->readsize = dbio->bufsize - remain;
                 dbio->readsize = dbio->size < dbio->bufsize - remain ? dbio->size : dbio->bufsize - remain - 1;
                 dbio->bufpt    = NULL;
             }
         }

     } else { /* use gzgets/fgets */

         char *pt;
         unsigned int bs;
 
         if (!dbio->size)
             return NULL;
 
         bs = dbio->size < size ? dbio->size + 1 : size;
         if (dbio->gzs)
             pt = gzgets(dbio->gzs, buff, bs);
         else
             pt = fgets(buff, bs, dbio->fs);
 
         if (!pt) {
             cli_errmsg("cli_dbgets: Preliminary end of data\n");
             return pt;
         }
         bs = strlen(buff);
         dbio->size -= bs;
         dbio->bread += bs;
         if (dbio->hashctx)
             cl_update_hash(dbio->hashctx, buff, bs);
         return pt;

     }
 }
 

 static char *cli_signorm(const char *signame)
 {
     char *new_signame = NULL;

     size_t pad        = 0;

     size_t nsz;

     if (!signame)
         return NULL;

     nsz = strlen(signame);

     if (nsz > 3 && signame[nsz - 1] == '}') {

         char *pt = strstr(signame, ".{");

         if (pt) /* strip the ".{ }" clause at the end of signame */

             nsz = pt - signame;
         else
             return NULL;
     } else if (nsz > 11) {

         if (!strncmp(signame + nsz - 11, ".UNOFFICIAL", 11))

             nsz -= 11;
         else
             return NULL;
     } else if (nsz > 2)
         return NULL;

     if (nsz < 3) {
         pad = 3 - nsz;
         nsz = 3;
     }

     new_signame = cli_calloc((nsz + 1), sizeof(char));

     if (!new_signame)

         return NULL;

     memcpy(new_signame, signame, nsz - pad);

     new_signame[nsz] = '\0';

     while (pad > 0)

         new_signame[nsz - pad--] = '\x20';

 
     return new_signame;

 }
 

 static int cli_chkign(const struct cli_matcher *ignored, const char *signame, const char *entry)

 {

     const char *md5_expected = NULL;

     char *norm_signame;

     unsigned char digest[16];

     int ret = 0;

     if (!ignored || !signame || !entry)

         return 0;

     norm_signame = cli_signorm(signame);
     if (norm_signame != NULL)

         signame = norm_signame;

     if (cli_bm_scanbuff((const unsigned char *)signame, strlen(signame), &md5_expected, NULL, ignored, 0, NULL, NULL, NULL) == CL_VIRUS)

         do {

             if (md5_expected) {

                 cl_hash_data("md5", entry, strlen(entry), digest, NULL);

                 if (memcmp(digest, (const unsigned char *)md5_expected, 16))

                     break;
             }

             cli_dbgmsg("Ignoring signature %s\n", signame);
             ret = 1;
         } while (0);

     if (norm_signame)

         free(norm_signame);

     return ret;

 }
 

 static int cli_chkpua(const char *signame, const char *pua_cats, unsigned int options)
 {

     char cat[32], *pt;
     const char *sig;
     int ret;

     if (strncmp(signame, "PUA.", 4)) {
         cli_dbgmsg("Skipping signature %s - no PUA prefix\n", signame);
         return 1;

     }
     sig = signame + 3;

     if (!(pt = strchr(sig + 1, '.'))) {
         cli_dbgmsg("Skipping signature %s - bad syntax\n", signame);
         return 1;

     }
 

     if ((unsigned int)(pt - sig + 2) > sizeof(cat)) {
         cli_dbgmsg("Skipping signature %s - too long category name\n", signame);
         return 1;

     }
 
     strncpy(cat, sig, pt - signame + 1);
     cat[pt - sig + 1] = 0;

     pt                = strstr(pua_cats, cat);

     if (options & CL_DB_PUA_INCLUDE)
         ret = pt ? 0 : 1;

     else

         ret = pt ? 1 : 0;

     if (ret)
         cli_dbgmsg("Skipping PUA signature %s - excluded category\n", signame);

 
     return ret;
 }
 

 static cl_error_t cli_loaddb(FILE *fs, struct cl_engine *engine, unsigned int *signo, unsigned int options, struct cli_dbio *dbio, const char *dbname)

 {

     char buffer[FILEBUFF], *buffer_cpy = NULL, *pt, *start;
     unsigned int line = 0, sigs = 0;
     int ret = 0;
     struct cli_matcher *root;

     UNUSEDPARAM(dbname);

     if (CL_SUCCESS != (ret = cli_initroots(engine, options)))

         return ret;

     root = engine->root[0];

     if (engine->ignored)
         if (!(buffer_cpy = cli_malloc(FILEBUFF))) {
             cli_errmsg("cli_loaddb: Can't allocate memory for buffer_cpy\n");
             return CL_EMEM;
         }

     while (cli_dbgets(buffer, FILEBUFF, fs, dbio)) {
         line++;
         if (buffer[0] == '#')
             continue;
         cli_chomp(buffer);
         if (engine->ignored)
             strcpy(buffer_cpy, buffer);

         pt = strchr(buffer, '=');
         if (!pt) {
             cli_errmsg("Malformed pattern line %d\n", line);
             ret = CL_EMALFDB;
             break;
         }

         start = buffer;
         *pt++ = 0;

         if (engine->ignored && cli_chkign(engine->ignored, start, buffer_cpy))
             continue;

         if (engine->cb_sigload && engine->cb_sigload("db", start, ~options & CL_DB_OFFICIAL, engine->cb_sigload_ctx)) {
             cli_dbgmsg("cli_loaddb: skipping %s due to callback\n", start);
             continue;
         }

         if (*pt == '=') continue;

         if (CL_SUCCESS != (ret = cli_parse_add(root, start, pt, 0, 0, 0, "*", 0, NULL, options))) {

             cli_dbgmsg("cli_loaddb: cli_parse_add failed on line %d\n", line);
             ret = CL_EMALFDB;
             break;
         }
         sigs++;

     }
 

     if (engine->ignored)
         free(buffer_cpy);

     if (!line) {
         cli_errmsg("Empty database file\n");
         return CL_EMALFDB;

     }
 

     if (ret) {
         cli_errmsg("Problem parsing database at line %d\n", line);
         return ret;

     }
 

     if (signo)
         *signo += sigs;

     return CL_SUCCESS;

 }
 

 #define ICO_TOKENS 4

 static cl_error_t cli_loadidb(FILE *fs, struct cl_engine *engine, unsigned int *signo, unsigned int options, struct cli_dbio *dbio)

 {

     const char *tokens[ICO_TOKENS + 1];
     char buffer[FILEBUFF], *buffer_cpy = NULL;
     uint8_t *hash;
     int ret           = CL_SUCCESS;
     unsigned int line = 0, sigs = 0, tokens_count, i, size, enginesize;
     struct icomtr *metric;
     struct icon_matcher *matcher;
 

     if (!(matcher = (struct icon_matcher *)MPOOL_CALLOC(engine->mempool, sizeof(*matcher), 1)))

         return CL_EMEM;
 
     if (engine->ignored)
         if (!(buffer_cpy = cli_malloc(FILEBUFF))) {
             cli_errmsg("cli_loadidb: Can't allocate memory for buffer_cpy\n");

             MPOOL_FREE(engine->mempool, matcher);

             return CL_EMEM;
         }
 
     while (cli_dbgets(buffer, FILEBUFF, fs, dbio)) {
         line++;
         if (buffer[0] == '#')
             continue;
 
         cli_chomp(buffer);
         if (engine->ignored)
             strcpy(buffer_cpy, buffer);
 
         tokens_count = cli_strtokenize(buffer, ':', ICO_TOKENS + 1, tokens);
         if (tokens_count != ICO_TOKENS) {
             cli_errmsg("cli_loadidb: Malformed hash at line %u (wrong token count)\n", line);
             ret = CL_EMALFDB;
             break;
         }
 
         if (strlen(tokens[3]) != 124) {
             cli_errmsg("cli_loadidb: Malformed hash at line %u (wrong length)\n", line);
             ret = CL_EMALFDB;
             break;
         }
 
         if (engine->ignored && cli_chkign(engine->ignored, tokens[0], buffer_cpy))
             continue;
 
         if (engine->cb_sigload && engine->cb_sigload("idb", tokens[0], ~options & CL_DB_OFFICIAL, engine->cb_sigload_ctx)) {
             cli_dbgmsg("cli_loadidb: skipping %s due to callback\n", tokens[0]);
             continue;
         }
 
         hash = (uint8_t *)tokens[3];
         if (cli_hexnibbles((char *)hash, 124)) {
             cli_errmsg("cli_loadidb: Malformed hash at line %u (bad chars)\n", line);
             ret = CL_EMALFDB;
             break;
         }
         size = (hash[0] << 4) + hash[1];
         if (size != 32 && size != 24 && size != 16) {
             cli_errmsg("cli_loadidb: Malformed hash at line %u (bad size)\n", line);
             ret = CL_EMALFDB;
             break;
         }
         enginesize = (size >> 3) - 2;
         hash += 2;
 

         metric = (struct icomtr *)MPOOL_REALLOC(engine->mempool, matcher->icons[enginesize], sizeof(struct icomtr) * (matcher->icon_counts[enginesize] + 1));

         if (!metric) {
             ret = CL_EMEM;
             break;
         }
 
         matcher->icons[enginesize] = metric;
         metric += matcher->icon_counts[enginesize];
         matcher->icon_counts[enginesize]++;
 
         for (i = 0; i < 3; i++) {
             if ((metric->color_avg[i] = (hash[0] << 8) | (hash[1] << 4) | hash[2]) > 4072)
                 break;
             if ((metric->color_x[i] = (hash[3] << 4) | hash[4]) > size - size / 8)
                 break;
             if ((metric->color_y[i] = (hash[5] << 4) | hash[6]) > size - size / 8)
                 break;
             hash += 7;
         }
         if (i != 3) {
             cli_errmsg("cli_loadidb: Malformed hash at line %u (bad color data)\n", line);
             ret = CL_EMALFDB;
             break;
         }
 
         for (i = 0; i < 3; i++) {
             if ((metric->gray_avg[i] = (hash[0] << 8) | (hash[1] << 4) | hash[2]) > 4072)
                 break;
             if ((metric->gray_x[i] = (hash[3] << 4) | hash[4]) > size - size / 8)
                 break;
             if ((metric->gray_y[i] = (hash[5] << 4) | hash[6]) > size - size / 8)
                 break;
             hash += 7;
         }
         if (i != 3) {
             cli_errmsg("cli_loadidb: Malformed hash at line %u (bad gray data)\n", line);
             ret = CL_EMALFDB;
             break;
         }
 
         for (i = 0; i < 3; i++) {
             metric->bright_avg[i] = (hash[0] << 4) | hash[1];
             if ((metric->bright_x[i] = (hash[2] << 4) | hash[3]) > size - size / 8)
                 break;
             if ((metric->bright_y[i] = (hash[4] << 4) | hash[5]) > size - size / 8)
                 break;
             hash += 6;
         }
         if (i != 3) {
             cli_errmsg("cli_loadidb: Malformed hash at line %u (bad bright data)\n", line);
             ret = CL_EMALFDB;
             break;
         }
 
         for (i = 0; i < 3; i++) {
             metric->dark_avg[i] = (hash[0] << 4) | hash[1];
             if ((metric->dark_x[i] = (hash[2] << 4) | hash[3]) > size - size / 8)
                 break;
             if ((metric->dark_y[i] = (hash[4] << 4) | hash[5]) > size - size / 8)
                 break;
             hash += 6;
         }
         if (i != 3) {
             cli_errmsg("cli_loadidb: Malformed hash at line %u (bad dark data)\n", line);
             ret = CL_EMALFDB;
             break;
         }
 
         for (i = 0; i < 3; i++) {
             metric->edge_avg[i] = (hash[0] << 4) | hash[1];
             if ((metric->edge_x[i] = (hash[2] << 4) | hash[3]) > size - size / 8)
                 break;
             if ((metric->edge_y[i] = (hash[4] << 4) | hash[5]) > size - size / 8)
                 break;
             hash += 6;
         }
         if (i != 3) {
             cli_errmsg("cli_loadidb: Malformed hash at line %u (bad edge data)\n", line);
             ret = CL_EMALFDB;
             break;
         }
 
         for (i = 0; i < 3; i++) {
             metric->noedge_avg[i] = (hash[0] << 4) | hash[1];
             if ((metric->noedge_x[i] = (hash[2] << 4) | hash[3]) > size - size / 8)
                 break;
             if ((metric->noedge_y[i] = (hash[4] << 4) | hash[5]) > size - size / 8)
                 break;
             hash += 6;
         }
         if (i != 3) {
             cli_errmsg("cli_loadidb: Malformed hash at line %u (bad noedge data)\n", line);
             ret = CL_EMALFDB;
             break;
         }
 
         metric->rsum   = (hash[0] << 4) | hash[1];
         metric->gsum   = (hash[2] << 4) | hash[3];
         metric->bsum   = (hash[4] << 4) | hash[5];
         metric->ccount = (hash[6] << 4) | hash[7];
         if (metric->rsum + metric->gsum + metric->bsum > 103 || metric->ccount > 100) {
             cli_errmsg("cli_loadidb: Malformed hash at line %u (bad spread data)\n", line);
             ret = CL_EMALFDB;
             break;
         }
 

         if (!(metric->name = CLI_MPOOL_STRDUP(engine->mempool, tokens[0]))) {

             ret = CL_EMEM;
             break;
         }
 
         for (i = 0; i < matcher->group_counts[0]; i++) {
             if (!strcmp(tokens[1], matcher->group_names[0][i]))
                 break;
         }
         if (i == matcher->group_counts[0]) {

             if (!(matcher->group_names[0] = MPOOL_REALLOC(engine->mempool, matcher->group_names[0], sizeof(char *) * (i + 1))) ||
                 !(matcher->group_names[0][i] = CLI_MPOOL_STRDUP(engine->mempool, tokens[1]))) {

                 ret = CL_EMEM;
                 break;
             }
             matcher->group_counts[0]++;
         }
         metric->group[0] = i;
 
         for (i = 0; i < matcher->group_counts[1]; i++) {
             if (!strcmp(tokens[2], matcher->group_names[1][i]))
                 break;
         }
         if (i == matcher->group_counts[1]) {

             if (!(matcher->group_names[1] = MPOOL_REALLOC(engine->mempool, matcher->group_names[1], sizeof(char *) * (i + 1))) ||
                 !(matcher->group_names[1][i] = CLI_MPOOL_STRDUP(engine->mempool, tokens[2]))) {

                 ret = CL_EMEM;
                 break;
             }
             matcher->group_counts[1]++;
         }
         metric->group[1] = i;
 
         if (matcher->group_counts[0] > 256 || matcher->group_counts[1] > 256) {
             cli_errmsg("cli_loadidb: too many icon groups!\n");
             ret = CL_EMALFDB;
             break;
         }
 
         sigs++;
     }
     if (engine->ignored)
         free(buffer_cpy);
 
     if (!line) {
         cli_errmsg("cli_loadidb: Empty database file\n");
         return CL_EMALFDB;
     }
 
     if (ret) {
         cli_errmsg("cli_loadidb: Problem parsing database at line %u\n", line);
         return ret;
     }
 
     if (signo)
         *signo += sigs;

     engine->iconcheck = matcher;

     return CL_SUCCESS;
 }

 static int cli_loadwdb(FILE *fs, struct cl_engine *engine, unsigned int options, struct cli_dbio *dbio)

 {

     int ret = 0;

     if (!(engine->dconf->phishing & PHISHING_CONF_ENGINE))
         return CL_SUCCESS;

     if (!engine->whitelist_matcher) {

         if (CL_SUCCESS != (ret = init_whitelist(engine))) {

             return ret;
         }

     }

     if (CL_SUCCESS != (ret = load_regex_matcher(engine, engine->whitelist_matcher, fs, NULL, options, 1, dbio, engine->dconf->other & OTHER_CONF_PREFILTERING))) {

         return ret;

     }

     return CL_SUCCESS;

 }
 

 static int cli_loadpdb(FILE *fs, struct cl_engine *engine, unsigned int *signo, unsigned int options, struct cli_dbio *dbio)

 {

     int ret = 0;

     if (!(engine->dconf->phishing & PHISHING_CONF_ENGINE))
         return CL_SUCCESS;

     if (!engine->domainlist_matcher) {

         if (CL_SUCCESS != (ret = init_domainlist(engine))) {

             return ret;
         }

     }

     if (CL_SUCCESS != (ret = load_regex_matcher(engine, engine->domainlist_matcher, fs, signo, options, 0, dbio, engine->dconf->other & OTHER_CONF_PREFILTERING))) {

         return ret;

     }
 
     return CL_SUCCESS;

 }

 #define NDB_TOKENS 6

 static int cli_loadndb(FILE *fs, struct cl_engine *engine, unsigned int *signo, unsigned short sdb, unsigned int options, struct cli_dbio *dbio, const char *dbname)

 {

     const char *tokens[NDB_TOKENS + 1];
     char buffer[FILEBUFF], *buffer_cpy = NULL;
     const char *sig, *virname, *offset, *pt;
     struct cli_matcher *root;
     int line = 0, sigs = 0, ret = 0, tokens_count;
     unsigned short target;
     unsigned int phish = options & CL_DB_PHISHING;

     UNUSEDPARAM(dbname);

     if (CL_SUCCESS != (ret = cli_initroots(engine, options)))

         return ret;

     if (engine->ignored)
         if (!(buffer_cpy = cli_malloc(FILEBUFF))) {
             cli_errmsg("cli_loadndb: Can't allocate memory for buffer_cpy\n");
             return CL_EMEM;
         }

     while (cli_dbgets(buffer, FILEBUFF, fs, dbio)) {
         line++;
         if (buffer[0] == '#')
             continue;

         if (!phish)
             if (!strncmp(buffer, "HTML.Phishing", 13) || !strncmp(buffer, "Email.Phishing", 14))
                 continue;

         cli_chomp(buffer);
         if (engine->ignored)
             strcpy(buffer_cpy, buffer);

         tokens_count = cli_strtokenize(buffer, ':', NDB_TOKENS + 1, tokens);
         if (tokens_count < 4 || tokens_count > 6) {
             ret = CL_EMALFDB;
             break;
         }

         virname = tokens[0];

         if (engine->pua_cats && (options & CL_DB_PUA_MODE) && (options & (CL_DB_PUA_INCLUDE | CL_DB_PUA_EXCLUDE)))
             if (cli_chkpua(virname, engine->pua_cats, options))
                 continue;

         if (engine->ignored && cli_chkign(engine->ignored, virname, buffer_cpy))
             continue;

         if (!sdb && engine->cb_sigload && engine->cb_sigload("ndb", virname, ~options & CL_DB_OFFICIAL, engine->cb_sigload_ctx)) {
             cli_dbgmsg("cli_loadndb: skipping %s due to callback\n", virname);
             continue;
         }

         if (tokens_count > 4) { /* min version */
             pt = tokens[4];

             if (!cli_isnumber(pt)) {
                 ret = CL_EMALFDB;
                 break;
             }

             if ((unsigned int)atoi(pt) > cl_retflevel()) {
                 cli_dbgmsg("Signature for %s not loaded (required f-level: %d)\n", virname, atoi(pt));
                 continue;
             }

             if (tokens_count == 6) { /* max version */
                 pt = tokens[5];
                 if (!cli_isnumber(pt)) {
                     ret = CL_EMALFDB;
                     break;
                 }

                 if ((unsigned int)atoi(pt) < cl_retflevel()) {
                     continue;
                 }
             }
         }

         if (!(pt = tokens[1]) || (strcmp(pt, "*") && !cli_isnumber(pt))) {
             ret = CL_EMALFDB;
             break;
         }
         target = (unsigned short)atoi(pt);

         if (target >= CLI_MTARGETS) {
             cli_dbgmsg("Not supported target type in signature for %s\n", virname);
             continue;
         }

         root = engine->root[target];

         offset = tokens[2];
         sig    = tokens[3];

         if (CL_SUCCESS != (ret = cli_parse_add(root, virname, sig, 0, 0, 0, offset, target, NULL, options))) {

             ret = CL_EMALFDB;
             break;
         }
         sigs++;

     }

     if (engine->ignored)
         free(buffer_cpy);

     if (!line) {
         cli_errmsg("Empty database file\n");
         return CL_EMALFDB;

     }

     if (ret) {
         cli_errmsg("Problem parsing database at line %d\n", line);
         return ret;

     }
 

     if (signo)
         *signo += sigs;

     if (sdb && sigs && !engine->sdb) {
         engine->sdb = 1;
         cli_dbgmsg("*** Self protection mechanism activated.\n");

     }
 

     return CL_SUCCESS;

 }
 

 struct lsig_attrib {
     const char *name;
     unsigned int type;
     void **pt;
 };
 
 /* TODO: rework this */
 static int lsigattribs(char *attribs, struct cli_lsig_tdb *tdb)
 {

 // clang-format off

 #define ATTRIB_TOKENS   10
 #define EXPR_TOKEN_MAX  16

     struct lsig_attrib attrtab[] = {
         { "Target",             CLI_TDB_UINT,       (void **) &tdb->target          },
         { "Engine",             CLI_TDB_RANGE,      (void **) &tdb->engine          },

         { "FileSize",           CLI_TDB_RANGE,      (void **) &tdb->filesize        },
         { "EntryPoint",         CLI_TDB_RANGE,      (void **) &tdb->ep              },
         { "NumberOfSections",   CLI_TDB_RANGE,      (void **) &tdb->nos             },

         { "IconGroup1",         CLI_TDB_STR,        (void **) &tdb->icongrp1        },
         { "IconGroup2",         CLI_TDB_STR,        (void **) &tdb->icongrp2        },

         { "Container",          CLI_TDB_FTYPE,      (void **) &tdb->container       },
         { "HandlerType",        CLI_TDB_FTYPE,      (void **) &tdb->handlertype     },
         { "Intermediates",      CLI_TDB_FTYPE_EXPR, (void **) &tdb->intermediates   },

 /*

         { "SectOff",            CLI_TDB_RANGE2,     (void **) &tdb->sectoff         },
         { "SectRVA",            CLI_TDB_RANGE2,     (void **) &tdb->sectrva         },
         { "SectVSZ",            CLI_TDB_RANGE2,     (void **) &tdb->sectvsz         },
         { "SectRAW",            CLI_TDB_RANGE2,     (void **) &tdb->sectraw         },
         { "SectRSZ",            CLI_TDB_RANGE2,     (void **) &tdb->sectrsz         },
         { "SectURVA",           CLI_TDB_RANGE2,     (void **) &tdb->secturva        },
         { "SectUVSZ",           CLI_TDB_RANGE2,     (void **) &tdb->sectuvsz        },
         { "SectURAW",           CLI_TDB_RANGE2,     (void **) &tdb->secturaw        },
         { "SectURSZ",           CLI_TDB_RANGE2,     (void **) &tdb->sectursz        },

 */

         { NULL,                 0,                  NULL,                           }

     };

     // clang-format on

     struct lsig_attrib *apt;
     char *tokens[ATTRIB_TOKENS], *pt, *pt2;
     unsigned int v1, v2, v3, i, j, tokens_count, have_newext = 0;
     uint32_t cnt, off[ATTRIB_TOKENS];

     tokens_count = cli_strtokenize(attribs, ',', ATTRIB_TOKENS, (const char **)tokens);

     for (i = 0; i < tokens_count; i++) {
         if (!(pt = strchr(tokens[i], ':'))) {

             cli_errmsg("lsigattribs: Incorrect format of attribute '%s'\n", tokens[i]);
             return -1;
         }
         *pt++ = 0;
 
         apt = NULL;

         for (j = 0; attrtab[j].name; j++) {
             if (!strcmp(attrtab[j].name, tokens[i])) {

                 apt = &attrtab[j];
                 break;
             }
         }

         if (!apt) {

             cli_dbgmsg("lsigattribs: Unknown attribute name '%s'\n", tokens[i]);
             return 1;
         }

         if (!strcmp(apt->name, "Engine")) {
             if (i) {

                 cli_errmsg("lsigattribs: For backward compatibility the Engine attribute must be on the first position\n");
                 return -1;
             }

         } else if (strcmp(apt->name, "Target")) {

             have_newext = 1;
         }

         switch (apt->type) {
             case CLI_TDB_UINT:
                 if (!cli_isnumber(pt)) {
                     cli_errmsg("lsigattribs: Invalid argument for %s\n", tokens[i]);
                     return -1;
                 }

                 off[i] = cnt = tdb->cnt[CLI_TDB_UINT]++;

                 tdb->val     = (uint32_t *)MPOOL_REALLOC2(tdb->mempool, tdb->val, tdb->cnt[CLI_TDB_UINT] * sizeof(uint32_t));

                 if (!tdb->val) {
                     tdb->cnt[CLI_TDB_UINT] = 0;
                     return -1;
                 }

                 tdb->val[cnt] = atoi(pt);
                 break;

             case CLI_TDB_FTYPE:
                 if ((v1 = cli_ftcode(pt)) == CL_TYPE_ERROR) {
                     cli_dbgmsg("lsigattribs: Unknown file type '%s' in %s\n", pt, tokens[i]);
                     return 1; /* skip */
                 }

                 off[i] = cnt = tdb->cnt[CLI_TDB_UINT]++;

                 tdb->val     = (uint32_t *)MPOOL_REALLOC2(tdb->mempool, tdb->val, tdb->cnt[CLI_TDB_UINT] * sizeof(uint32_t));

                 if (!tdb->val) {
                     tdb->cnt[CLI_TDB_UINT] = 0;
                     return -1;
                 }

                 tdb->val[cnt] = v1;
                 break;

             case CLI_TDB_FTYPE_EXPR: {

                 char *ftypes[EXPR_TOKEN_MAX];
                 unsigned int ftypes_count;
 

                 off[i] = cnt = tdb->cnt[CLI_TDB_UINT];

                 ftypes_count = cli_strtokenize(pt, '>', EXPR_TOKEN_MAX, (const char **)ftypes);
                 if (!ftypes_count) {

                     cli_dbgmsg("lsigattribs: No intermediate container tokens found.");
                     return 1;
                 }

                 tdb->cnt[CLI_TDB_UINT] += (ftypes_count + 1);

                 tdb->val = (uint32_t *)MPOOL_REALLOC2(tdb->mempool, tdb->val, tdb->cnt[CLI_TDB_UINT] * sizeof(uint32_t));

                 if (!tdb->val) {

                     tdb->cnt[CLI_TDB_UINT] = 0;
                     return -1;
                 }

                 tdb->val[cnt++] = ftypes_count;

                 for (j = 0; j < ftypes_count; j++) {
                     if ((v1 = cli_ftcode(ftypes[j])) == CL_TYPE_ERROR) {

                         cli_dbgmsg("lsigattribs: Unknown file type '%s' in %s\n", ftypes[j], tokens[i]);
                         return 1; /* skip */
                     }

                     tdb->val[cnt++] = v1;

                 }

             } break;

             case CLI_TDB_RANGE:
                 if (!(pt2 = strchr(pt, '-'))) {
                     cli_errmsg("lsigattribs: Incorrect parameters in '%s'\n", tokens[i]);
                     return -1;
                 }

                 *pt2++ = 0;
                 off[i] = cnt = tdb->cnt[CLI_TDB_RANGE];
                 tdb->cnt[CLI_TDB_RANGE] += 2;

                 tdb->range = (uint32_t *)MPOOL_REALLOC2(tdb->mempool, tdb->range, tdb->cnt[CLI_TDB_RANGE] * sizeof(uint32_t));

                 if (!tdb->range) {
                     tdb->cnt[CLI_TDB_RANGE] = 0;
                     return -1;
                 }

                 if (!cli_isnumber(pt) || !cli_isnumber(pt2)) {
                     cli_errmsg("lsigattribs: Invalid argument for %s\n", tokens[i]);
                     return -1;
                 }

                 tdb->range[cnt]     = atoi(pt);
                 tdb->range[cnt + 1] = atoi(pt2);
                 break;

             case CLI_TDB_RANGE2:
                 if (!strchr(pt, '-') || !strchr(pt, '.')) {
                     cli_errmsg("lsigattribs: Incorrect parameters in '%s'\n", tokens[i]);
                     return -1;
                 }

                 off[i] = cnt = tdb->cnt[CLI_TDB_RANGE];
                 tdb->cnt[CLI_TDB_RANGE] += 3;

                 tdb->range = (uint32_t *)MPOOL_REALLOC2(tdb->mempool, tdb->range, tdb->cnt[CLI_TDB_RANGE] * sizeof(uint32_t));

                 if (!tdb->range) {
                     tdb->cnt[CLI_TDB_RANGE] = 0;
                     return -1;
                 }

                 if (sscanf(pt, "%u.%u-%u", &v1, &v2, &v3) != 3) {
                     cli_errmsg("lsigattribs: Can't parse parameters in '%s'\n", tokens[i]);
                     return -1;
                 }

                 tdb->range[cnt]     = (uint32_t)v1;
                 tdb->range[cnt + 1] = (uint32_t)v2;
                 tdb->range[cnt + 2] = (uint32_t)v3;
                 break;

             case CLI_TDB_STR:
                 off[i] = cnt = tdb->cnt[CLI_TDB_STR];
                 tdb->cnt[CLI_TDB_STR] += strlen(pt) + 1;

                 tdb->str = (char *)MPOOL_REALLOC2(tdb->mempool, tdb->str, tdb->cnt[CLI_TDB_STR] * sizeof(char));

                 if (!tdb->str) {
                     cli_errmsg("lsigattribs: Can't allocate memory for tdb->str\n");
                     return -1;
                 }
                 memcpy(&tdb->str[cnt], pt, strlen(pt));
                 tdb->str[tdb->cnt[CLI_TDB_STR] - 1] = 0;
                 break;

             default:
                 /* All known TDB types handled above, skip unknown */
                 cli_dbgmsg("lsigattribs: Unknown attribute type '%u'\n", apt->type);
                 return 1; /* +1 = skip */

         }

     }
 

     if (!i) {

         cli_errmsg("lsigattribs: Empty TDB\n");
         return -1;

     }
 

     for (i = 0; i < tokens_count; i++) {
         for (j = 0; attrtab[j].name; j++) {
             if (!strcmp(attrtab[j].name, tokens[i])) {

                 apt = &attrtab[j];
                 break;
             }
         }

         if (!apt)

             continue;

         switch (apt->type) {
             case CLI_TDB_UINT:
             case CLI_TDB_FTYPE:
             case CLI_TDB_FTYPE_EXPR:
                 *apt->pt = (uint32_t *)&tdb->val[off[i]];
                 break;

             case CLI_TDB_RANGE:
             case CLI_TDB_RANGE2:
                 *apt->pt = (uint32_t *)&tdb->range[off[i]];
                 break;

             case CLI_TDB_STR:
                 *apt->pt = (char *)&tdb->str[off[i]];
                 break;

         }

     }
 

     if (have_newext && (!tdb->engine || tdb->engine[0] < 51)) {

         cli_errmsg("lsigattribs: For backward compatibility all signatures using new attributes must have the Engine attribute present and set to min_level of at least 51 (0.96)\n");
         return -1;

     }

     return 0;
 }
 

 #define FREE_TDB(x)                               \
     do {                                          \
         if (x.cnt[CLI_TDB_UINT])                  \

             MPOOL_FREE(x.mempool, x.val);         \

         if (x.cnt[CLI_TDB_RANGE])                 \

             MPOOL_FREE(x.mempool, x.range);       \

         if (x.cnt[CLI_TDB_STR])                   \

             MPOOL_FREE(x.mempool, x.str);         \

         if (x.macro_ptids)                        \

             MPOOL_FREE(x.mempool, x.macro_ptids); \

     } while (0);
 
 #define FREE_TDB_P(x)                               \
     do {                                            \
         if (x->cnt[CLI_TDB_UINT])                   \

             MPOOL_FREE(x->mempool, x->val);         \

         if (x->cnt[CLI_TDB_RANGE])                  \

             MPOOL_FREE(x->mempool, x->range);       \

         if (x->cnt[CLI_TDB_STR])                    \

             MPOOL_FREE(x->mempool, x->str);         \

         if (x->macro_ptids)                         \

             MPOOL_FREE(x->mempool, x->macro_ptids); \

     } while (0);

 static inline int init_tdb(struct cli_lsig_tdb *tdb, struct cl_engine *engine, char *target, const char *virname)

 {
     int ret;
 
 #ifdef USE_MPOOL
     tdb->mempool = engine->mempool;

 #else
     UNUSEDPARAM(engine);

 #endif

     if (CL_SUCCESS != (ret = lsigattribs(target, tdb))) {

         FREE_TDB_P(tdb);

         if (ret == 1) {

             cli_dbgmsg("init_tdb: Not supported attribute(s) in signature for %s, skipping\n", virname);
             return CL_BREAK;
         }
         return CL_EMALFDB;
     }
 

     if (tdb->engine) {
         if (tdb->engine[0] > cl_retflevel()) {

             FREE_TDB_P(tdb);
             cli_dbgmsg("init_tdb: Signature for %s not loaded (required f-level: %u)\n", virname, tdb->engine[0]);
             return CL_BREAK;

         } else if (tdb->engine[1] < cl_retflevel()) {

             FREE_TDB_P(tdb);
             return CL_BREAK;
         }
     }
 

     if (!tdb->target) {

         FREE_TDB_P(tdb);
         cli_errmsg("init_tdb: No target specified in TDB\n");
         return CL_EMALFDB;

     } else if (tdb->target[0] >= CLI_MTARGETS) {

         FREE_TDB_P(tdb);
         cli_dbgmsg("init_tdb: Not supported target type in signature for %s, skipping\n", virname);
         return CL_BREAK;
     }
 

     if ((tdb->icongrp1 || tdb->icongrp2) && tdb->target[0] != 1) {

         FREE_TDB_P(tdb);
         cli_errmsg("init_tdb: IconGroup is only supported in PE (target 1) signatures\n");
         return CL_EMALFDB;
     }
 

     if ((tdb->ep || tdb->nos) && tdb->target[0] != 1 && tdb->target[0] != 6 && tdb->target[0] != 9) {

         FREE_TDB_P(tdb);
         cli_errmsg("init_tdb: EntryPoint/NumberOfSections is only supported in PE/ELF/Mach-O signatures\n");
         return CL_EMALFDB;
     }
 
     return CL_SUCCESS;
 }
 

 /*     0         1        2      3        4        5    ... (max 66)

  * VirusName;Attributes;Logic;SubSig1[;SubSig2[;SubSig3 ... ]]

  * NOTE: Maximum of 64(see MAX_LDB_SUBSIGS) subsignatures (last would be token 66)

  */

 #define LDB_TOKENS 67

 #define SUB_TOKENS 4

 static int load_oneldb(char *buffer, int chkpua, struct cl_engine *engine, unsigned int options, const char *dbname, unsigned int line, unsigned int *sigs, unsigned bc_idx, const char *buffer_cpy, int *skip)

 {

     const char *sig, *virname, *offset, *logic, *sigopts;

     struct cli_ac_lsig **newtable, *lsig;

     char *tokens[LDB_TOKENS + 1], *subtokens[SUB_TOKENS + 1];

     int i, j, subsigs, tokens_count, subtokens_count;

     unsigned short target = 0;
     struct cli_matcher *root;
     struct cli_lsig_tdb tdb;
     uint32_t lsigid[2];

     uint8_t subsig_opts;

     int ret;

     UNUSEDPARAM(dbname);
 

     tokens_count = cli_ldbtokenize(buffer, ';', LDB_TOKENS + 1, (const char **)tokens, 2);
     if (tokens_count < 4) {

         cli_errmsg("Invalid or unsupported ldb signature format\n");

         return CL_EMALFDB;

     }

     virname = tokens[0];

     logic   = tokens[2];

     if (chkpua && cli_chkpua(virname, engine->pua_cats, options))

         return CL_SUCCESS;

     if (engine->ignored && cli_chkign(engine->ignored, virname, buffer_cpy ? buffer_cpy : virname)) {

         if (skip)

             *skip = 1;
         return CL_SUCCESS;

     }

     if (engine->cb_sigload && engine->cb_sigload("ldb", virname, ~options & CL_DB_OFFICIAL, engine->cb_sigload_ctx)) {

         cli_dbgmsg("cli_loadldb: skipping %s due to callback\n", virname);
         (*sigs)--;
         return CL_SUCCESS;

     }
 

     subsigs = cli_ac_chklsig(logic, logic + strlen(logic), NULL, NULL, NULL, 1);

     if (subsigs == -1) {

         cli_errmsg("Invalid or unsupported ldb logic\n");

         return CL_EMALFDB;

     }
     subsigs++;

     if (!line) {

         /* This is a logical signature from the bytecode, we need all
          * subsignatures, even if not referenced from the logical expression */

         if (subsigs > tokens_count - 3) {

             cli_errmsg("load_oneldb: Too many subsignatures: %u (max %u)\n",

                        subsigs, tokens_count - 3);

             return CL_EMALFDB;
         }

         subsigs = tokens_count - 3;
     } else if (subsigs != tokens_count - 3) {

         cli_errmsg("cli_loadldb: The number of subsignatures (== %u) doesn't match the IDs in the logical expression (== %u)\n", tokens_count - 3, subsigs);
         return CL_EMALFDB;

     }

 #if !HAVE_PCRE
     /* Regex Usage and Support Check */
     for (i = 0; i < subsigs; ++i) {
         char *slash = strchr(tokens[i + 3], '/');
         if (slash && strchr(slash + 1, '/')) {
             cli_warnmsg("cli_loadldb: logical signature for %s uses PCREs but support is disabled, skipping\n", virname);
             (*sigs)--;
             return CL_SUCCESS;
         }
     }
 #endif
 

     /* enforce MAX_LDB_SUBSIGS(currently 64) subsig cap */

     if (subsigs > MAX_LDB_SUBSIGS) {

         cli_errmsg("cli_loadldb: Broken logical expression or too many subsignatures\n");
         return CL_EMALFDB;

     }
 

     /* TDB */
     memset(&tdb, 0, sizeof(tdb));

     if (CL_SUCCESS != (ret = init_tdb(&tdb, engine, tokens[1], virname)) != CL_SUCCESS) {

         (*sigs)--;

         if (ret == CL_BREAK)
             return CL_SUCCESS;
         return ret;

     }
 

     root = engine->root[tdb.target[0]];

     lsig = (struct cli_ac_lsig *)MPOOL_CALLOC(engine->mempool, 1, sizeof(struct cli_ac_lsig));

     if (!lsig) {

         cli_errmsg("cli_loadldb: Can't allocate memory for lsig\n");
         FREE_TDB(tdb);
         return CL_EMEM;

     }

     lsig->type    = CLI_LSIG_NORMAL;

     lsig->u.logic = CLI_MPOOL_STRDUP(engine->mempool, logic);

     if (!lsig->u.logic) {

         cli_errmsg("cli_loadldb: Can't allocate memory for lsig->logic\n");
         FREE_TDB(tdb);

         MPOOL_FREE(engine->mempool, lsig);

         return CL_EMEM;

     }

     lsigid[0] = lsig->id = root->ac_lsigs;

     if (bc_idx)
         root->linked_bcs++;

     root->ac_lsigs++;

     newtable = (struct cli_ac_lsig **)MPOOL_REALLOC(engine->mempool, root->ac_lsigtable, root->ac_lsigs * sizeof(struct cli_ac_lsig *));

     if (!newtable) {

         if (bc_idx)
             root->linked_bcs--;

         root->ac_lsigs--;
         cli_errmsg("cli_loadldb: Can't realloc root->ac_lsigtable\n");
         FREE_TDB(tdb);

         MPOOL_FREE(engine->mempool, lsig);

         return CL_EMEM;

     }

     /* 0 marks no bc, we can't use a pointer to bc, since that is
      * realloced/moved during load */

     lsig->bc_idx                 = bc_idx;

     newtable[root->ac_lsigs - 1] = lsig;

     root->ac_lsigtable           = newtable;
     tdb.subsigs                  = subsigs;

     for (i = 0; i < subsigs; i++) {

         lsigid[1] = i;

         offset    = "*";

         sigopts     = NULL;

         subsig_opts = 0;
 

         subtokens_count = cli_ldbtokenize(tokens[3 + i], ':', SUB_TOKENS + 1, (const char **)subtokens, 0);
         if (!subtokens_count) {

             cli_errmsg("Invalid or unsupported ldb subsignature format\n");
             return CL_EMALFDB;
         }
 

         if ((subtokens_count % 2) == 0)

             offset = subtokens[0];

         if (subtokens_count == 3)

             sigopts = subtokens[2];

         else if (subtokens_count == 4)

             sigopts = subtokens[3];
 

         if (sigopts) { /* signature modifiers */
             for (j = 0; j < (int)strlen(sigopts); j++)
                 switch (sigopts[j]) {
                     case 'i':
                         subsig_opts |= ACPATT_OPTION_NOCASE;
                         break;
                     case 'f':
                         subsig_opts |= ACPATT_OPTION_FULLWORD;
                         break;
                     case 'w':
                         subsig_opts |= ACPATT_OPTION_WIDE;
                         break;
                     case 'a':
                         subsig_opts |= ACPATT_OPTION_ASCII;
                         break;
                     default:
                         cli_errmsg("cli_loadldb: Signature for %s uses invalid option: %02x\n", virname, sigopts[j]);
                         return CL_EMALFDB;

                 }

         }

         sig = (subtokens_count % 2) ? subtokens[0] : subtokens[1];
 

         if (subsig_opts)

             ret = cli_sigopts_handler(root, virname, sig, subsig_opts, 0, 0, offset, target, lsigid, options);

         else
             ret = cli_parse_add(root, virname, sig, 0, 0, 0, offset, target, lsigid, options);

         if (ret)

             return ret;
 

         if (sig[0] == '$' && i) {

             /* allow mapping from lsig back to pattern for macros */
             if (!tdb.macro_ptids)

                 tdb.macro_ptids = MPOOL_CALLOC(root->mempool, subsigs, sizeof(*tdb.macro_ptids));

             if (!tdb.macro_ptids)
                 return CL_EMEM;
 

             tdb.macro_ptids[i - 1] = root->ac_patterns - 1;

         }

     }

     memcpy(&lsig->tdb, &tdb, sizeof(tdb));

     return CL_SUCCESS;
 }

 static int cli_loadldb(FILE *fs, struct cl_engine *engine, unsigned int *signo, unsigned int options, struct cli_dbio *dbio, const char *dbname)
 {

     char buffer[CLI_DEFAULT_LSIG_BUFSIZE + 1], *buffer_cpy = NULL;
     unsigned int line = 0, sigs = 0;
     int ret;

     if (CL_SUCCESS != (ret = cli_initroots(engine, options)))

         return ret;

     if (engine->ignored) {
         if (!(buffer_cpy = cli_malloc(sizeof(buffer)))) {

             cli_errmsg("cli_loadldb: Can't allocate memory for buffer_cpy\n");
             return CL_EMEM;
         }

     }

     while (cli_dbgets(buffer, sizeof(buffer), fs, dbio)) {

         line++;

         if (buffer[0] == '#')

             continue;

         sigs++;
         cli_chomp(buffer);
 

         if (engine->ignored)

             strcpy(buffer_cpy, buffer);
 
         ret = load_oneldb(buffer,

                           engine->pua_cats && (options & CL_DB_PUA_MODE) && (options & (CL_DB_PUA_INCLUDE | CL_DB_PUA_EXCLUDE)),
                           engine, options, dbname, line, &sigs, 0, buffer_cpy, NULL);

         if (ret)
             break;

     }

     if (engine->ignored)

         free(buffer_cpy);

     if (!line) {

         cli_errmsg("Empty database file\n");
         return CL_EMALFDB;

     }
 

     if (ret) {

         cli_errmsg("Problem parsing database at line %u\n", line);
         return ret;

     }
 

     if (signo)

         *signo += sigs;

 
     return CL_SUCCESS;
 }
 

 static int cli_loadcbc(FILE *fs, struct cl_engine *engine, unsigned int *signo, unsigned int options, struct cli_dbio *dbio, const char *dbname)
 {

     char buf[4096];

     int rc, skip = 0;

     struct cli_all_bc *bcs = &engine->bcs;
     struct cli_bc *bc;

     unsigned sigs           = 0;

     unsigned security_trust = 0;

     unsigned i;

     /* TODO: virusname have a common prefix, and whitelist by that */

     if ((rc = cli_initroots(engine, options)))
         return rc;

     if (!(engine->dconf->bytecode & BYTECODE_ENGINE_MASK)) {
         return CL_SUCCESS;

     }

     if (engine->cb_sigload && engine->cb_sigload("cbc", dbname, ~options & CL_DB_OFFICIAL, engine->cb_sigload_ctx)) {
         cli_dbgmsg("cli_loadcbc: skipping %s due to callback\n", dbname);
         return CL_SUCCESS;

     }
 

     if (!(options & CL_DB_BYTECODE_UNSIGNED) && !(options & CL_DB_SIGNED)) {

         cli_warnmsg("Only loading signed bytecode, skipping load of unsigned bytecode!\n");
         cli_warnmsg("Turn on BytecodeUnsigned/--bytecode-unsigned to enable loading of unsigned bytecode\n");
         return CL_SUCCESS;

     }

     bcs->all_bcs = cli_realloc2(bcs->all_bcs, sizeof(*bcs->all_bcs) * (bcs->count + 1));

     if (!bcs->all_bcs) {

         cli_errmsg("cli_loadcbc: Can't allocate memory for bytecode entry\n");
         return CL_EMEM;

     }
     bcs->count++;

     bc = &bcs->all_bcs[bcs->count - 1];

 
     switch (engine->bytecode_security) {

         case CL_BYTECODE_TRUST_SIGNED:
             security_trust = !!(options & CL_DB_SIGNED);
             break;
         default:
             security_trust = 0;

     }
 

     rc = cli_bytecode_load(bc, fs, dbio, security_trust, options & CL_DB_BYTECODE_STATS);

     /* read remainder of DB, needed because cvd.c checks that we read the entire
      * file */

     while (cli_dbgets(buf, sizeof(buf), fs, dbio)) {
     }

     if (rc != CL_SUCCESS) {

         cli_bytecode_destroy(bc);
         cli_errmsg("Unable to load %s bytecode: %s\n", dbname, cl_strerror(rc));
         return rc;

     }

     if (bc->state == bc_skip) {

         cli_bytecode_destroy(bc);
         bcs->count--;
         return CL_SUCCESS;

     }

     bc->id = bcs->count; /* must set after _load, since load zeroes */

     if (engine->bytecode_mode == CL_BYTECODE_MODE_TEST)

         cli_infomsg(NULL, "bytecode %u -> %s\n", bc->id, dbname);

     if (bc->kind == BC_LOGICAL || bc->lsig) {

         unsigned oldsigs = sigs;

         if (!bc->lsig) {
             cli_errmsg("Bytecode %s has logical kind, but missing logical signature!\n", dbname);
             return CL_EMALFDB;
         }
         cli_dbgmsg("Bytecode %s(%u) has logical signature: %s\n", dbname, bc->id, bc->lsig);
         rc = load_oneldb(bc->lsig, 0, engine, options, dbname, 0, &sigs, bcs->count, NULL, &skip);
         if (rc != CL_SUCCESS) {
             cli_errmsg("Problem parsing logical signature %s for bytecode %s: %s\n",
                        bc->lsig, dbname, cl_strerror(rc));
             return rc;
         }
         if (skip) {
             cli_bytecode_destroy(bc);
             bcs->count--;
             return CL_SUCCESS;
         }

         if (sigs != oldsigs) {

             /* compiler ensures Engine field in lsig matches the one in bytecode,

            * so this should never happen. */

             cli_errmsg("Bytecode logical signature skipped, but bytecode itself not?");
             return CL_EMALFDB;

         }

     }

     sigs++;

     if (bc->kind != BC_LOGICAL) {

         if (bc->lsig) {
             /* runlsig will only flip a status bit, not report a match,

 	     * when the hooks are executed we only execute the hook if its
 	     * status bit is on */

             bc->hook_lsig_id = ++engine->hook_lsig_ids;
         }
         if (bc->kind >= _BC_START_HOOKS && bc->kind < _BC_LAST_HOOK) {
             unsigned hook       = bc->kind - _BC_START_HOOKS;
             unsigned cnt        = ++engine->hooks_cnt[hook];
             engine->hooks[hook] = cli_realloc2(engine->hooks[hook],
                                                sizeof(*engine->hooks[0]) * cnt);
             if (!engine->hooks[hook]) {
                 cli_errmsg("Out of memory allocating memory for hook %u", hook);
                 return CL_EMEM;
             }
             engine->hooks[hook][cnt - 1] = bcs->count - 1;
         } else
             switch (bc->kind) {
                 case BC_STARTUP:
                     for (i = 0; i < bcs->count - 1; i++)
                         if (bcs->all_bcs[i].kind == BC_STARTUP) {
                             struct cli_bc *bc0 = &bcs->all_bcs[i];
                             cli_errmsg("Can only load 1 BC_STARTUP bytecode, attempted to load 2nd!\n");
                             cli_warnmsg("Previous BC_STARTUP: %d %d by %s\n",
                                         bc0->id, (uint32_t)bc0->metadata.timestamp,
                                         bc0->metadata.sigmaker ? bc0->metadata.sigmaker : "N/A");
                             cli_warnmsg("Conflicting BC_STARTUP: %d %d by %s\n",
                                         bc->id, (uint32_t)bc->metadata.timestamp,
                                         bc->metadata.sigmaker ? bc->metadata.sigmaker : "N/A");
                             return CL_EMALFDB;
                         }
                     break;
                 default:
                     cli_errmsg("Bytecode: unhandled bytecode kind %u\n", bc->kind);
                     return CL_EMALFDB;
             }

     }

     if (signo)

         *signo += sigs;

     return CL_SUCCESS;
 }
 

 /*     0       1      2     3        4            5          6      7
  * MagicType:Offset:HexSig:Name:RequiredType:DetectedType[:MinFL[:MaxFL]]
  */

 #define FTM_TOKENS 8

 static int cli_loadftm(FILE *fs, struct cl_engine *engine, unsigned int options, unsigned int internal, struct cli_dbio *dbio)

 {

     const char *tokens[FTM_TOKENS + 1], *pt;
     char buffer[FILEBUFF];
     unsigned int line = 0, sigs = 0, tokens_count;
     struct cli_ftype *new;
     cli_file_t rtype, type;
     int ret;
     int magictype;
 

     if (CL_SUCCESS != (ret = cli_initroots(engine, options)))

         return ret;
 
     while (1) {
         if (internal) {
             options |= CL_DB_OFFICIAL;
             if (!ftypes_int[line])
                 break;
             strncpy(buffer, ftypes_int[line], sizeof(buffer));
             buffer[sizeof(buffer) - 1] = '\0';
         } else {
             if (!cli_dbgets(buffer, FILEBUFF, fs, dbio))
                 break;
             if (buffer[0] == '#')
                 continue;
             cli_chomp(buffer);
         }
         line++;
         tokens_count = cli_strtokenize(buffer, ':', FTM_TOKENS + 1, tokens);
 
         if (tokens_count < 6 || tokens_count > 8) {
             ret = CL_EMALFDB;
             break;
         }
 
         if (tokens_count > 6) { /* min version */
             pt = tokens[6];
             if (!cli_isnumber(pt)) {
                 ret = CL_EMALFDB;
                 break;
             }
             if ((unsigned int)atoi(pt) > cl_retflevel()) {
                 cli_dbgmsg("cli_loadftm: File type signature for %s not loaded (required f-level: %u)\n", tokens[3], atoi(pt));
                 continue;
             }
             if (tokens_count == 8) { /* max version */
                 pt = tokens[7];
                 if (!cli_isnumber(pt)) {
                     ret = CL_EMALFDB;
                     break;
                 }
                 if ((unsigned int)atoi(pt) < cl_retflevel())
                     continue;
             }
         }
 
         rtype = cli_ftcode(tokens[4]);
         type  = cli_ftcode(tokens[5]);
         if (rtype == CL_TYPE_ERROR || type == CL_TYPE_ERROR) {
             ret = CL_EMALFDB;
             break;
         }
 
         if (!cli_isnumber(tokens[0])) {
             cli_errmsg("cli_loadftm: Invalid value for the first field\n");
             ret = CL_EMALFDB;
             break;
         }

         magictype = atoi(tokens[0]);

         if (magictype == 1) { /* A-C */

             if (CL_SUCCESS != (ret = cli_parse_add(engine->root[0], tokens[3], tokens[2], 0, rtype, type, tokens[1], 0, NULL, options)))

                 break;
 
         } else if ((magictype == 0) || (magictype == 4)) { /* memcmp() */
             if (!cli_isnumber(tokens[1])) {
                 cli_errmsg("cli_loadftm: Invalid offset\n");
                 ret = CL_EMALFDB;
                 break;
             }

             new = (struct cli_ftype *)MPOOL_MALLOC(engine->mempool, sizeof(struct cli_ftype));

             if (!new) {
                 ret = CL_EMEM;
                 break;
             }
             new->type   = type;
             new->offset = atoi(tokens[1]);

             new->magic  = (unsigned char *)CLI_MPOOL_HEX2STR(engine->mempool, tokens[2]);

             if (!new->magic) {
                 cli_errmsg("cli_loadftm: Can't decode the hex string\n");
                 ret = CL_EMALFDB;

                 MPOOL_FREE(engine->mempool, new);

                 break;
             }
             new->length = (uint16_t)strlen(tokens[2]) / 2;

             new->tname  = CLI_MPOOL_STRDUP(engine->mempool, tokens[3]);

             if (!new->tname) {

                 MPOOL_FREE(engine->mempool, new->magic);
                 MPOOL_FREE(engine->mempool, new);

                 ret = CL_EMEM;
                 break;
             }

             /* files => ftypes, partitions => ptypes */

             if (magictype == 4) {
                 new->next      = engine->ptypes;
                 engine->ptypes = new;
             } else {
                 new->next      = engine->ftypes;
                 engine->ftypes = new;

             }

         } else {
             cli_dbgmsg("cli_loadftm: Unsupported mode %u\n", atoi(tokens[0]));
             continue;
         }
         sigs++;

     }
 

     if (ret) {
         cli_errmsg("Problem parsing %s filetype database at line %u\n", internal ? "built-in" : "external", line);
         return ret;

     }
 

     if (!sigs) {
         cli_errmsg("Empty %s filetype database\n", internal ? "built-in" : "external");
         return CL_EMALFDB;

     }
 

     cli_dbgmsg("Loaded %u filetype definitions\n", sigs);

     return CL_SUCCESS;
 }
 

 #define INFO_NSTR "11088894983048545473659556106627194923928941791795047620591658697413581043322715912172496806525381055880964520618400224333320534660299233983755341740679502866829909679955734391392668378361221524205396631090105151641270857277080310734320951653700508941717419168723942507890702904702707587451621691050754307850383399865346487203798464178537392211402786481359824461197231102895415093770394216666324484593935762408468516826633192140826667923494822045805347809932848454845886971706424360558667862775876072059437703365380209101697738577515476935085469455279994113145977994084618328482151013142393373316337519977244732747977"
 #define INFO_ESTR "100002049"
 #define INFO_TOKENS 3

 static int cli_loadinfo(FILE *fs, struct cl_engine *engine, unsigned int options, struct cli_dbio *dbio)
 {

     const char *tokens[INFO_TOKENS + 1];
     char buffer[FILEBUFF];
     unsigned int line = 0, tokens_count, len;
     char hash[32];

     struct cli_dbinfo *last = NULL, *new;

     int ret = CL_SUCCESS, dsig = 0;

     void *ctx;

     if (!dbio) {
         cli_errmsg("cli_loadinfo: .info files can only be loaded from within database container files\n");
         return CL_EMALFDB;

     }

     ctx = cl_hash_init("sha256");

     if (!(ctx))
         return CL_EMALFDB;
 

     while (cli_dbgets(buffer, FILEBUFF, fs, dbio)) {
         line++;
         if (!(options & CL_DB_UNSIGNED) && !strncmp(buffer, "DSIG:", 5)) {
             dsig = 1;
             cl_finish_hash(ctx, hash);
             if (cli_versig2((unsigned char *)hash, buffer + 5, INFO_NSTR, INFO_ESTR) != CL_SUCCESS) {
                 cli_errmsg("cli_loadinfo: Incorrect digital signature\n");
                 ret = CL_EMALFDB;
             }
             break;
         }
         len = strlen(buffer);
         if (!len) {
             buffer[len]     = '\n';

             buffer[len + 1] = 0;

         } else {
             if (dbio->usebuf && buffer[len - 1] != '\n' && len + 1 < FILEBUFF) {
                 /* cli_dbgets in buffered mode strips \n */
                 buffer[len]     = '\n';
                 buffer[len + 1] = 0;
             }
         }
         cl_update_hash(ctx, buffer, strlen(buffer));
         cli_chomp(buffer);
         if (!strncmp("ClamAV-VDB:", buffer, 11)) {
             if (engine->dbinfo) { /* shouldn't be initialized at this point */
                 cli_errmsg("cli_loadinfo: engine->dbinfo already initialized\n");
                 ret = CL_EMALFDB;
                 break;
             }

             last = engine->dbinfo = (struct cli_dbinfo *)MPOOL_CALLOC(engine->mempool, 1, sizeof(struct cli_bm_patt));

             if (!engine->dbinfo) {
                 ret = CL_EMEM;
                 break;
             }
             engine->dbinfo->cvd = cl_cvdparse(buffer);
             if (!engine->dbinfo->cvd) {
                 cli_errmsg("cli_loadinfo: Can't parse header entry\n");
                 ret = CL_EMALFDB;
                 break;
             }
             continue;
         }
 
         if (!last) {
             cli_errmsg("cli_loadinfo: Incorrect file format\n");
             ret = CL_EMALFDB;
             break;
         }
         tokens_count = cli_strtokenize(buffer, ':', INFO_TOKENS + 1, tokens);
         if (tokens_count != INFO_TOKENS) {
             ret = CL_EMALFDB;
             break;
         }

         new = (struct cli_dbinfo *)MPOOL_CALLOC(engine->mempool, 1, sizeof(struct cli_dbinfo));

         if (!new) {
             ret = CL_EMEM;
             break;
         }

         new->name = CLI_MPOOL_STRDUP(engine->mempool, tokens[0]);

         if (!new->name) {

             MPOOL_FREE(engine->mempool, new);

             ret = CL_EMEM;
             break;
         }
 
         if (!cli_isnumber(tokens[1])) {
             cli_errmsg("cli_loadinfo: Invalid value in the size field\n");

             MPOOL_FREE(engine->mempool, new->name);
             MPOOL_FREE(engine->mempool, new);

             ret = CL_EMALFDB;
             break;
         }
         new->size = atoi(tokens[1]);
 

         if (strlen(tokens[2]) != 64 || !(new->hash = CLI_MPOOL_HEX2STR(engine->mempool, tokens[2]))) {

             cli_errmsg("cli_loadinfo: Malformed SHA256 string at line %u\n", line);

             MPOOL_FREE(engine->mempool, new->name);
             MPOOL_FREE(engine->mempool, new);

             ret = CL_EMALFDB;
             break;

         }

         last->next = new;
         last       = new;

     }

 
     if (!(options & CL_DB_UNSIGNED) && !dsig) {
         cli_errmsg("cli_loadinfo: Digital signature not found\n");
         return CL_EMALFDB;
     }
 
     if (ret) {
         cli_errmsg("cli_loadinfo: Problem parsing database at line %u\n", line);
         return ret;

     }
 
     return CL_SUCCESS;
 }
 

 #define IGN_MAX_TOKENS 3

 static int cli_loadign(FILE *fs, struct cl_engine *engine, unsigned int options, struct cli_dbio *dbio)

 {

     const char *tokens[IGN_MAX_TOKENS + 1], *signame, *hash = NULL;
     char buffer[FILEBUFF];
     unsigned int line = 0, tokens_count, len;
     struct cli_bm_patt *new;
     int ret = CL_SUCCESS;

     UNUSEDPARAM(options);
 

     if (!engine->ignored) {

         engine->ignored = (struct cli_matcher *)MPOOL_CALLOC(engine->mempool, 1, sizeof(struct cli_matcher));

         if (!engine->ignored)
             return CL_EMEM;

 #ifdef USE_MPOOL

         engine->ignored->mempool = engine->mempool;

 #endif

         if (CL_SUCCESS != (ret = cli_bm_init(engine->ignored))) {

             cli_errmsg("cli_loadign: Can't initialise AC pattern matcher\n");
             return ret;
         }
     }
 
     while (cli_dbgets(buffer, FILEBUFF, fs, dbio)) {
         line++;
         if (buffer[0] == '#')
             continue;
         cli_chomp(buffer);
 
         tokens_count = cli_strtokenize(buffer, ':', IGN_MAX_TOKENS + 1, tokens);
         if (tokens_count > IGN_MAX_TOKENS) {
             ret = CL_EMALFDB;
             break;
         }
 
         if (tokens_count == 1) {
             signame = buffer;
         } else if (tokens_count == 2) {
             signame = tokens[0];
             hash    = tokens[1];
         } else { /* old mode */
             signame = tokens[2];
         }
         if (!(len = strlen(signame))) {
             cli_errmsg("cli_loadign: No signature name provided\n");
             ret = CL_EMALFDB;
             break;
         }

         if (len < 3) {
             int pad = 3 - len;

             /* patch-up for Boyer-Moore minimum length of 3: pad with spaces */

             if (signame != buffer) {

                 strncpy(buffer, signame, len);

                 signame = buffer;
             }
             buffer[3] = '\0';
             while (pad > 0)

                 buffer[3 - pad--] = '\x20';

             len = 3;
         }

         new = (struct cli_bm_patt *)MPOOL_CALLOC(engine->mempool, 1, sizeof(struct cli_bm_patt));

         if (!new) {
             ret = CL_EMEM;
             break;
         }

         new->pattern = (unsigned char *)CLI_MPOOL_STRDUP(engine->mempool, signame);

         if (!new->pattern) {

             MPOOL_FREE(engine->mempool, new);

             ret = CL_EMEM;
             break;
         }
         if (hash) {

             if (strlen(hash) != 32 || !(new->virname = CLI_MPOOL_HEX2STR(engine->mempool, hash))) {

                 cli_errmsg("cli_loadign: Malformed MD5 string at line %u\n", line);

                 MPOOL_FREE(engine->mempool, new->pattern);
                 MPOOL_FREE(engine->mempool, new);

                 ret = CL_EMALFDB;
                 break;
             }
         }
         new->length = len;
         new->boundary |= BM_BOUNDARY_EOL;
 

         if (CL_SUCCESS != (ret = cli_bm_addpatt(engine->ignored, new, "0"))) {

             if (hash)

                 MPOOL_FREE(engine->mempool, new->virname);
             MPOOL_FREE(engine->mempool, new->pattern);
             MPOOL_FREE(engine->mempool, new);

             break;
         }
     }
 
     if (ret) {
         cli_errmsg("cli_loadign: Problem parsing database at line %u\n", line);
         return ret;

     }
 
     return CL_SUCCESS;
 }
 

 #define MD5_HDB 0
 #define MD5_MDB 1
 #define MD5_FP 2
 #define MD5_IMP 3

 #define MD5_TOKENS 5

 static int cli_loadhash(FILE *fs, struct cl_engine *engine, unsigned int *signo, unsigned int mode, unsigned int options, struct cli_dbio *dbio, const char *dbname)
 {

     const char *tokens[MD5_TOKENS + 1];
     char buffer[FILEBUFF], *buffer_cpy = NULL;
     const char *pt, *virname;

     int ret                 = CL_SUCCESS;

     unsigned int size_field = 1, md5_field = 0, line = 0, sigs = 0, tokens_count;

     unsigned int req_fl = 0;

     struct cli_matcher *db;
     unsigned long size;

     if (mode == MD5_MDB) {
         size_field = 0;
         md5_field  = 1;
         db         = engine->hm_mdb;
     } else if (mode == MD5_HDB)
         db = engine->hm_hdb;
     else if (mode == MD5_IMP)
         db = engine->hm_imp;

     else

         db = engine->hm_fp;

     if (!db) {

         if (!(db = MPOOL_CALLOC(engine->mempool, 1, sizeof(*db))))

             return CL_EMEM;

 #ifdef USE_MPOOL

         db->mempool = engine->mempool;

 #endif

         if (mode == MD5_HDB)
             engine->hm_hdb = db;
         else if (mode == MD5_MDB)
             engine->hm_mdb = db;
         else if (mode == MD5_IMP)
             engine->hm_imp = db;
         else
             engine->hm_fp = db;
     }
 
     if (engine->ignored)
         if (!(buffer_cpy = cli_malloc(FILEBUFF))) {
             cli_errmsg("cli_loadhash: Can't allocate memory for buffer_cpy\n");
             return CL_EMEM;
         }
 
     while (cli_dbgets(buffer, FILEBUFF, fs, dbio)) {
         line++;
         if (buffer[0] == '#')
             continue;
         cli_chomp(buffer);
         if (engine->ignored)
             strcpy(buffer_cpy, buffer);
 
         tokens_count = cli_strtokenize(buffer, ':', MD5_TOKENS + 1, tokens);
         if (tokens_count < 3) {
             ret = CL_EMALFDB;
             break;
         }
         if (tokens_count > MD5_TOKENS - 2) {
             req_fl = atoi(tokens[MD5_TOKENS - 2]);
 
             if (tokens_count > MD5_TOKENS) {
                 ret = CL_EMALFDB;
                 break;
             }
 
             if (cl_retflevel() < req_fl)
                 continue;
             if (tokens_count == MD5_TOKENS) {
                 int max_fl = atoi(tokens[MD5_TOKENS - 1]);
                 if (cl_retflevel() > (unsigned int)max_fl)
                     continue;
             }
         }
 

         if (strcmp(tokens[size_field], "*")) {

             size = strtoul(tokens[size_field], (char **)&pt, 10);
             if (*pt || !size || size >= 0xffffffff) {
                 cli_errmsg("cli_loadhash: Invalid value for the size field\n");
                 ret = CL_EMALFDB;
                 break;
             }
         } else {
             size = 0;

             // The wildcard feature was added in FLEVEL 73, so for backwards
             // compatibility with older clients, ensure that a minimum FLEVEL
             // is specified.  This check doesn't apply to .imp rules, though,
             // since this rule category wasn't introduced until FLEVEL 90, and
             // has always supported wildcard usage in rules.
             if (mode != MD5_IMP && ((tokens_count < MD5_TOKENS - 1) || (req_fl < 73))) {

                 cli_errmsg("cli_loadhash: Minimum FLEVEL field must be at least 73 for wildcard size hash signatures."
                            " For reference, running FLEVEL is %d\n",
                            cl_retflevel());
                 ret = CL_EMALFDB;
                 break;
             }
         }
 
         pt = tokens[2]; /* virname */
         if (engine->pua_cats && (options & CL_DB_PUA_MODE) && (options & (CL_DB_PUA_INCLUDE | CL_DB_PUA_EXCLUDE)))
             if (cli_chkpua(pt, engine->pua_cats, options))
                 continue;
 
         if (engine->ignored && cli_chkign(engine->ignored, pt, buffer_cpy))
             continue;
 
         if (engine->cb_sigload) {
             const char *dot = strchr(dbname, '.');
             if (!dot)
                 dot = dbname;
             else
                 dot++;
             if (engine->cb_sigload(dot, pt, ~options & CL_DB_OFFICIAL, engine->cb_sigload_ctx)) {
                 cli_dbgmsg("cli_loadhash: skipping %s (%s) due to callback\n", pt, dot);
                 continue;
             }
         }
 

         virname = CLI_MPOOL_VIRNAME(engine->mempool, pt, options & CL_DB_OFFICIAL);

         if (!virname) {
             ret = CL_EMALFDB;
             break;
         }
 

         if (CL_SUCCESS != (ret = hm_addhash_str(db, tokens[md5_field], size, virname))) {

             cli_errmsg("cli_loadhash: Malformed hash string at line %u\n", line);

             MPOOL_FREE(engine->mempool, (void *)virname);

             break;
         }
 
         sigs++;
     }
     if (engine->ignored)
         free(buffer_cpy);
 
     if (!line) {
         cli_errmsg("cli_loadhash: Empty database file\n");
         return CL_EMALFDB;
     }
 
     if (ret) {
         cli_errmsg("cli_loadhash: Problem parsing database at line %u\n", line);
         return ret;
     }
 
     if (signo)
         *signo += sigs;

 
     return CL_SUCCESS;
 }
 

 #define MD_TOKENS 9

 static int cli_loadmd(FILE *fs, struct cl_engine *engine, unsigned int *signo, int type, unsigned int options, struct cli_dbio *dbio, const char *dbname)

 {

     const char *tokens[MD_TOKENS + 1];
     char buffer[FILEBUFF], *buffer_cpy = NULL;
     unsigned int line = 0, sigs = 0, tokens_count;
     int ret = CL_SUCCESS;
     struct cli_cdb *new;

     UNUSEDPARAM(dbname);

     if (engine->ignored)
         if (!(buffer_cpy = cli_malloc(FILEBUFF))) {
             cli_errmsg("cli_loadmd: Can't allocate memory for buffer_cpy\n");
             return CL_EMEM;
         }
 
     while (cli_dbgets(buffer, FILEBUFF, fs, dbio)) {
         line++;
         if (buffer[0] == '#')
             continue;
 
         cli_chomp(buffer);
         if (engine->ignored)
             strcpy(buffer_cpy, buffer);
 
         tokens_count = cli_strtokenize(buffer, ':', MD_TOKENS + 1, tokens);
         if (tokens_count != MD_TOKENS) {
             ret = CL_EMALFDB;
             break;
         }
 
         if (strcmp(tokens[1], "*") && !cli_isnumber(tokens[1])) {
             cli_errmsg("cli_loadmd: Invalid value for the 'encrypted' field\n");
             ret = CL_EMALFDB;
             break;
         }
         if (strcmp(tokens[3], "*") && !cli_isnumber(tokens[3])) {
             cli_errmsg("cli_loadmd: Invalid value for the 'original size' field\n");
             ret = CL_EMALFDB;
             break;
         }
         if (strcmp(tokens[4], "*") && !cli_isnumber(tokens[4])) {
             cli_errmsg("cli_loadmd: Invalid value for the 'compressed size' field\n");
             ret = CL_EMALFDB;
             break;
         }
         if (strcmp(tokens[6], "*") && !cli_isnumber(tokens[6])) {
             cli_errmsg("cli_loadmd: Invalid value for the 'compression method' field\n");
             ret = CL_EMALFDB;
             break;
         }
         if (strcmp(tokens[7], "*") && !cli_isnumber(tokens[7])) {
             cli_errmsg("cli_loadmd: Invalid value for the 'file number' field\n");
             ret = CL_EMALFDB;
             break;
         }
         if (strcmp(tokens[8], "*") && !cli_isnumber(tokens[8])) {
             cli_errmsg("cli_loadmd: Invalid value for the 'max depth' field\n");
             ret = CL_EMALFDB;
             break;
         }
 

         new = (struct cli_cdb *)MPOOL_CALLOC(engine->mempool, 1, sizeof(struct cli_cdb));

         if (!new) {
             ret = CL_EMEM;
             break;
         }
 

         new->virname = CLI_MPOOL_VIRNAME(engine->mempool, tokens[0], options & CL_DB_OFFICIAL);

         if (!new->virname) {

             MPOOL_FREE(engine->mempool, new);

             ret = CL_EMEM;
             break;
         }
         new->ctype = (type == 1) ? CL_TYPE_ZIP : CL_TYPE_RAR;
 
         if (engine->ignored && cli_chkign(engine->ignored, new->virname, buffer /*_cpy*/)) {

             MPOOL_FREE(engine->mempool, new->virname);
             MPOOL_FREE(engine->mempool, new);

             continue;
         }
 
         if (engine->cb_sigload && engine->cb_sigload("md", new->virname, ~options & CL_DB_OFFICIAL, engine->cb_sigload_ctx)) {
             cli_dbgmsg("cli_loadmd: skipping %s due to callback\n", new->virname);

             MPOOL_FREE(engine->mempool, new->virname);
             MPOOL_FREE(engine->mempool, new);

             continue;
         }
 
         new->encrypted = strcmp(tokens[1], "*") ? atoi(tokens[1]) : 2;
 
         if (strcmp(tokens[2], "*") && cli_regcomp(&new->name, tokens[2], REG_EXTENDED | REG_NOSUB)) {
             cli_errmsg("cli_loadmd: Can't compile regular expression %s in signature for %s\n", tokens[2], tokens[0]);

             MPOOL_FREE(engine->mempool, new->virname);
             MPOOL_FREE(engine->mempool, new);

             ret = CL_EMEM;
             break;
         }
         new->csize[0] = new->csize[1] = CLI_OFF_ANY;
 
         if (!strcmp(tokens[3], "*"))
             new->fsizer[0] = new->fsizer[1] = CLI_OFF_ANY;
         else
             new->fsizer[0] = new->fsizer[1] = atoi(tokens[3]);
 
         if (!strcmp(tokens[4], "*"))
             new->fsizec[0] = new->fsizec[1] = CLI_OFF_ANY;
         else
             new->fsizec[0] = new->fsizec[1] = atoi(tokens[4]);
 
         if (strcmp(tokens[5], "*")) {
             new->res1 = cli_hex2num(tokens[5]);
             if (new->res1 == -1) {

                 MPOOL_FREE(engine->mempool, new->virname);
                 MPOOL_FREE(engine->mempool, new);

                 if (new->name.re_magic)
                     cli_regfree(&new->name);
                 ret = CL_EMALFDB;
                 break;
             }
         }
 
         /* tokens[6] - not used */
 
         new->filepos[0] = new->filepos[1] = strcmp(tokens[7], "*") ? (unsigned int)atoi(tokens[7]) : (unsigned int)CLI_OFF_ANY;
 
         /* tokens[8] - not used */
 
         new->next   = engine->cdb;
         engine->cdb = new;
         sigs++;
     }
     if (engine->ignored)
         free(buffer_cpy);
 
     if (!line) {
         cli_errmsg("Empty database file\n");
         return CL_EMALFDB;
     }
 
     if (ret) {
         cli_errmsg("Problem parsing database at line %d\n", line);
         return ret;
     }
 
     if (signo)
         *signo += sigs;

     return CL_SUCCESS;

 }
 

 /*    0		 1		2		3	         4	       5	      6	      7	      8   9    10     11
  * VirusName:ContainerType:ContainerSize:FileNameREGEX:FileSizeInContainer:FileSizeReal:IsEncrypted:FilePos:Res1:Res2[:MinFL[:MaxFL]]

  */

 
 #define CDB_TOKENS 12

 static int cli_loadcdb(FILE *fs, struct cl_engine *engine, unsigned int *signo, unsigned int options, struct cli_dbio *dbio)
 {

     const char *tokens[CDB_TOKENS + 1];
     char buffer[FILEBUFF], *buffer_cpy = NULL;
     unsigned int line = 0, sigs = 0, tokens_count, n0, n1;
     int ret = CL_SUCCESS;
     struct cli_cdb *new;
 
     if (engine->ignored)
         if (!(buffer_cpy = cli_malloc(FILEBUFF))) {
             cli_errmsg("cli_loadcdb: Can't allocate memory for buffer_cpy\n");
             return CL_EMEM;
         }
 
     while (cli_dbgets(buffer, FILEBUFF, fs, dbio)) {
         line++;
         if (buffer[0] == '#')
             continue;
 
         cli_chomp(buffer);
         if (engine->ignored)
             strcpy(buffer_cpy, buffer);
 
         tokens_count = cli_strtokenize(buffer, ':', CDB_TOKENS + 1, tokens);
         if (tokens_count > CDB_TOKENS || tokens_count < CDB_TOKENS - 2) {
             ret = CL_EMALFDB;
             break;
         }
 
         if (tokens_count > 10) { /* min version */
             if (!cli_isnumber(tokens[10])) {
                 ret = CL_EMALFDB;
                 break;
             }
             if ((unsigned int)atoi(tokens[10]) > cl_retflevel()) {
                 cli_dbgmsg("cli_loadcdb: Container signature for %s not loaded (required f-level: %u)\n", tokens[0], atoi(tokens[10]));
                 continue;
             }
             if (tokens_count == CDB_TOKENS) { /* max version */
                 if (!cli_isnumber(tokens[11])) {
                     ret = CL_EMALFDB;
                     break;
                 }
                 if ((unsigned int)atoi(tokens[11]) < cl_retflevel())
                     continue;
             }
         }
 

         new = (struct cli_cdb *)MPOOL_CALLOC(engine->mempool, 1, sizeof(struct cli_cdb));

         if (!new) {
             ret = CL_EMEM;
             break;
         }
 

         new->virname = CLI_MPOOL_VIRNAME(engine->mempool, tokens[0], options & CL_DB_OFFICIAL);

         if (!new->virname) {

             MPOOL_FREE(engine->mempool, new);

             ret = CL_EMEM;
             break;
         }
 
         if (engine->ignored && cli_chkign(engine->ignored, new->virname, buffer /*_cpy*/)) {

             MPOOL_FREE(engine->mempool, new->virname);
             MPOOL_FREE(engine->mempool, new);

             continue;
         }
 
         if (engine->cb_sigload && engine->cb_sigload("cdb", new->virname, ~options & CL_DB_OFFICIAL, engine->cb_sigload_ctx)) {
             cli_dbgmsg("cli_loadcdb: skipping %s due to callback\n", new->virname);

             MPOOL_FREE(engine->mempool, new->virname);
             MPOOL_FREE(engine->mempool, new);

             continue;
         }
 
         if (!strcmp(tokens[1], "*")) {
             new->ctype = CL_TYPE_ANY;
         } else if ((new->ctype = cli_ftcode(tokens[1])) == CL_TYPE_ERROR) {
             cli_errmsg("cli_loadcdb: Unknown container type %s in signature for %s, skipping\n", tokens[1], tokens[0]);

             ret = CL_EMALFDB;

             MPOOL_FREE(engine->mempool, new->virname);
             MPOOL_FREE(engine->mempool, new);

             break;
         }
 
         if (strcmp(tokens[3], "*") && cli_regcomp(&new->name, tokens[3], REG_EXTENDED | REG_NOSUB)) {
             cli_errmsg("cli_loadcdb: Can't compile regular expression %s in signature for %s\n", tokens[3], tokens[0]);

             MPOOL_FREE(engine->mempool, new->virname);
             MPOOL_FREE(engine->mempool, new);

             ret = CL_EMEM;
             break;
         }
 
 #define CDBRANGE(token_str, dest)                                             \
     if (strcmp(token_str, "*")) {                                             \
         if (strchr(token_str, '-')) {                                         \
             if (sscanf(token_str, "%u-%u", &n0, &n1) != 2) {                  \
                 ret = CL_EMALFDB;                                             \
             } else {                                                          \
                 dest[0] = n0;                                                 \
                 dest[1] = n1;                                                 \
             }                                                                 \
         } else {                                                              \
             if (!cli_isnumber(token_str))                                     \
                 ret = CL_EMALFDB;                                             \
             else                                                              \
                 dest[0] = dest[1] = (unsigned int)atoi(token_str);            \
         }                                                                     \
         if (ret != CL_SUCCESS) {                                              \
             cli_errmsg("cli_loadcdb: Invalid value %s in signature for %s\n", \
                        token_str, tokens[0]);                                 \
             if (new->name.re_magic)                                           \
                 cli_regfree(&new->name);                                      \

             MPOOL_FREE(engine->mempool, new->virname);                        \
             MPOOL_FREE(engine->mempool, new);                                 \

             ret = CL_EMEM;                                                    \
             break;                                                            \
         }                                                                     \
     } else {                                                                  \
         dest[0] = dest[1] = CLI_OFF_ANY;                                      \
     }
 
         CDBRANGE(tokens[2], new->csize);
         CDBRANGE(tokens[4], new->fsizec);
         CDBRANGE(tokens[5], new->fsizer);
         CDBRANGE(tokens[7], new->filepos);
 
         if (!strcmp(tokens[6], "*")) {
             new->encrypted = 2;
         } else {
             if (strcmp(tokens[6], "0") && strcmp(tokens[6], "1")) {
                 cli_errmsg("cli_loadcdb: Invalid encryption flag value in signature for %s\n", tokens[0]);
                 if (new->name.re_magic)
                     cli_regfree(&new->name);

                 MPOOL_FREE(engine->mempool, new->virname);
                 MPOOL_FREE(engine->mempool, new);

                 ret = CL_EMEM;
                 break;
             }
             new->encrypted = *tokens[6] - 0x30;
         }
 
         if (strcmp(tokens[9], "*")) {

             new->res2 = CLI_MPOOL_STRDUP(engine->mempool, tokens[9]);

             if (!new->res2) {
                 cli_errmsg("cli_loadcdb: Can't allocate memory for res2 in signature for %s\n", tokens[0]);
                 if (new->name.re_magic)
                     cli_regfree(&new->name);

                 MPOOL_FREE(engine->mempool, new->virname);
                 MPOOL_FREE(engine->mempool, new);

                 ret = CL_EMEM;
                 break;
             }
         }
 
         new->next   = engine->cdb;
         engine->cdb = new;
         sigs++;
     }
     if (engine->ignored)
         free(buffer_cpy);
 
     if (!line) {
         cli_errmsg("Empty database file\n");
         return CL_EMALFDB;
     }
 
     if (ret) {
         cli_errmsg("Problem parsing database at line %u\n", line);
         return ret;
     }
 
     if (signo)
         *signo += sigs;

 
     return CL_SUCCESS;
 }
 

 /*

  * name;trusted;subject;serial;pubkey;exp;codesign;timesign;certsign;notbefore;comment[;minFL[;maxFL]]

  * Name and comment are ignored. They're just for the end user.

  * Exponent is ignored for now and hardcoded to \x01\x00\x01.

  */

 #define CRT_TOKENS 13

 static int cli_loadcrt(FILE *fs, struct cl_engine *engine, struct cli_dbio *dbio)
 {

     char buffer[FILEBUFF];

     char *tokens[CRT_TOKENS + 1];
     size_t line = 0, tokens_count;

     cli_crt ca;

     int ret       = CL_SUCCESS;
     char *subject = NULL, *pubkey = NULL, *serial = NULL;

     const uint8_t exp[] = "\x01\x00\x01";
 

     if (!(engine->dconf->pe & PE_CONF_CERTS)) {
         cli_dbgmsg("cli_loadcrt: Ignoring .crb sigs due to DCONF configuration\n");
         return ret;
     }
 
     if (engine->engine_options & ENGINE_OPTIONS_DISABLE_PE_CERTS) {
         cli_dbgmsg("cli_loadcrt: Ignoring .crb sigs due to engine options\n");
         return ret;
     }
 

     cli_crt_init(&ca);

     memset(ca.issuer, 0xca, sizeof(ca.issuer));

 
     while (cli_dbgets(buffer, FILEBUFF, fs, dbio)) {
         line++;
 
         if (buffer[0] == '#')
             continue;
 
         cli_chomp(buffer);
         if (!strlen(buffer))
             continue;
 
         tokens_count = cli_strtokenize(buffer, ';', CRT_TOKENS + 1, (const char **)tokens);
         if (tokens_count > CRT_TOKENS || tokens_count < CRT_TOKENS - 2) {

             cli_errmsg("cli_loadcrt: line %u: Invalid number of tokens: %u\n", (unsigned int)line, (unsigned int)tokens_count);

             ret = CL_EMALFDB;
             goto end;
         }
 

         if (tokens_count > CRT_TOKENS - 2) {

             if (!cli_isnumber(tokens[CRT_TOKENS - 1])) {

                 cli_errmsg("cli_loadcrt: line %u: Invalid minimum feature level\n", (unsigned int)line);

                 ret = CL_EMALFDB;
                 goto end;
             }

             if ((unsigned int)atoi(tokens[CRT_TOKENS - 1]) > cl_retflevel()) {

                 cli_dbgmsg("cli_loadcrt: Cert %s not loaded (required f-level: %u)\n", tokens[0], cl_retflevel());
                 continue;
             }
 
             if (tokens_count == CRT_TOKENS) {
                 if (!cli_isnumber(tokens[CRT_TOKENS])) {

                     cli_errmsg("cli_loadcrt: line %u: Invalid maximum feature level\n", (unsigned int)line);

                     ret = CL_EMALFDB;
                     goto end;
                 }
 

                 if ((unsigned int)atoi(tokens[CRT_TOKENS]) < cl_retflevel()) {

                     cli_dbgmsg("cli_ladcrt: Cert %s not loaded (maximum f-level: %s)\n", tokens[0], tokens[CRT_TOKENS]);
                     continue;
                 }
             }
         }
 

         switch (tokens[1][0]) {
             case '1':
                 ca.isBlacklisted = 0;
                 break;
             case '0':
                 ca.isBlacklisted = 1;
                 break;
             default:

                 cli_errmsg("cli_loadcrt: line %u: Invalid trust specification. Expected 0 or 1\n", (unsigned int)line);

                 ret = CL_EMALFDB;
                 goto end;
         }
 
         subject = cli_hex2str(tokens[2]);

         if (strlen(tokens[3])) {
             serial = cli_hex2str(tokens[3]);
             if (!serial) {

                 cli_errmsg("cli_loadcrt: line %u: Cannot convert serial to binary string\n", (unsigned int)line);

                 ret = CL_EMALFDB;
                 goto end;
             }
             memcpy(ca.serial, serial, sizeof(ca.serial));
             free(serial);
         } else {

             memset(ca.serial, 0xca, sizeof(ca.serial));

         }
         pubkey = cli_hex2str(tokens[4]);
         cli_dbgmsg("cli_loadcrt: subject: %s\n", tokens[2]);
         cli_dbgmsg("cli_loadcrt: public key: %s\n", tokens[4]);

 
         if (!subject) {

             cli_errmsg("cli_loadcrt: line %u: Cannot convert subject to binary string\n", (unsigned int)line);

             ret = CL_EMALFDB;
             goto end;
         }
         if (!pubkey) {

             cli_errmsg("cli_loadcrt: line %u: Cannot convert public key to binary string\n", (unsigned int)line);

             ret = CL_EMALFDB;
             goto end;
         }
 
         memcpy(ca.subject, subject, sizeof(ca.subject));

         if (mp_read_unsigned_bin(&(ca.n), (const unsigned char *)pubkey, strlen(tokens[4]) / 2) || mp_read_unsigned_bin(&(ca.e), exp, sizeof(exp) - 1)) {

             cli_errmsg("cli_loadcrt: line %u: Cannot convert exponent to binary data\n", (unsigned int)line);

             ret = CL_EMALFDB;
             goto end;
         }
 

         switch (tokens[6][0]) {

             case '1':
                 ca.codeSign = 1;
                 break;
             case '0':
                 ca.codeSign = 0;
                 break;
             default:

                 cli_errmsg("cli_loadcrt: line %u: Invalid code sign specification. Expected 0 or 1\n", (unsigned int)line);

                 ret = CL_EMALFDB;
                 goto end;
         }
 

         switch (tokens[7][0]) {

             case '1':
                 ca.timeSign = 1;
                 break;
             case '0':
                 ca.timeSign = 0;
                 break;
             default:

                 cli_errmsg("cli_loadcrt: line %u: Invalid time sign specification. Expected 0 or 1\n", (unsigned int)line);

                 ret = CL_EMALFDB;
                 goto end;
         }
 

         switch (tokens[8][0]) {
             case '1':
                 ca.certSign = 1;
                 break;
             case '0':
                 ca.certSign = 0;
                 break;
             default:

                 cli_errmsg("cli_loadcrt: line %u: Invalid cert sign specification. Expected 0 or 1\n", (unsigned int)line);

                 ret = CL_EMALFDB;
                 goto end;
         }
 

         if (strlen(tokens[0]))
             ca.name = tokens[0];
         else
             ca.name = NULL;
 

         if (strlen(tokens[9]))
             ca.not_before = atoi(tokens[8]);

         ca.not_after = (-1U) >> 1;

         ca.hashtype = CLI_HASHTYPE_ANY;

         crtmgr_add(&(engine->cmgr), &ca);

         free(subject);
         free(pubkey);

         subject = pubkey = NULL;

     }
 
 end:

     if (subject)
         free(subject);
     if (pubkey)
         free(pubkey);
 

     cli_dbgmsg("Number of certs: %d\n", engine->cmgr.items);
     cli_crt_clear(&ca);
     return ret;
 }
 

 static int cli_loadmscat(FILE *fs, const char *dbname, struct cl_engine *engine, unsigned int options, struct cli_dbio *dbio)
 {

     fmap_t *map;
 

     UNUSEDPARAM(options);
     UNUSEDPARAM(dbio);
 

     /* If loading in signatures stored in .cat files is disabled, then skip.
      * If Authenticoded signature parsing in general is disabled, then also
      * skip. */
     if (!(engine->dconf->pe & PE_CONF_CATALOG) || !(engine->dconf->pe & PE_CONF_CERTS)) {
         cli_dbgmsg("cli_loadmscat: Ignoring .cat sigs due to DCONF configuration\n");
         return 0;
     }
 
     if (engine->engine_options & ENGINE_OPTIONS_DISABLE_PE_CERTS) {
         cli_dbgmsg("cli_loadmscat: Ignoring .cat sigs due to engine options\n");
         return 0;
     }
 

     if (!(map = fmap(fileno(fs), 0, 0))) {
         cli_dbgmsg("Can't map cat: %s\n", dbname);
         return 0;

     }

     if (asn1_load_mscat(map, engine))
         cli_dbgmsg("Failed to load certificates from cat: %s\n", dbname);

     funmap(map);

     return 0;
 }
 

 static int cli_loadopenioc(FILE *fs, const char *dbname, struct cl_engine *engine, unsigned int options)

 {
     int rc;

     rc = openioc_parse(dbname, fileno(fs), engine, options);

     if (rc != CL_SUCCESS)
         return CL_EMALFDB;

     return rc;

 }
 

 #ifdef HAVE_YARA

 #define YARA_DEBUG 1

 #if (YARA_DEBUG == 2)

 #define cli_yaramsg(...) cli_errmsg(__VA_ARGS__)

 #elif (YARA_DEBUG == 1)

 #define cli_yaramsg(...) cli_dbgmsg(__VA_ARGS__)
 #else

 #define cli_yaramsg(...)

 #endif
 

 static char *parse_yara_hex_string(YR_STRING *string, int *ret);

 static char *parse_yara_hex_string(YR_STRING *string, int *ret)

 {

     char *res, *str, *ovr;

     size_t slen, reslen = 0, i, j;

     if (!(string) || !(string->string)) {
         if (ret) *ret = CL_ENULLARG;

         return NULL;

     }

     if (!STRING_IS_HEX(string)) {
         if (ret) *ret = CL_EARG;

         return NULL;

     }

     str = (char *)(string->string);

     if ((slen = string->length) == 0) {

         if (ret) *ret = CL_EARG;

         return NULL;

     }

     str = strchr(str, '{') + 1;

     for (i = 0; i < slen - 1; i++) {

         switch (str[i]) {

             case ' ':
             case '\t':
             case '\r':
             case '\n':
             case '}': /* end of hex string */
                 break;
             default:
                 reslen++;
                 break;

         }
     }
 

     reslen++;
     res = cli_calloc(reslen, 1);

     if (!(res)) {
         if (ret) *ret = CL_EMEM;

         return NULL;

     }

     for (i = 0, j = 0; i < slen - 1 && j < reslen; i++) {

         switch (str[i]) {

             case ' ':
             case '\t':
             case '\r':
             case '\n':
             case '}':
                 break;
             case '[':
                 /* unbounded range check */
                 if ((i + 2 < slen - 1) && (str[i + 1] == '-') && (str[i + 2] == ']')) {
                     res[j++] = '*';
                     i += 2;
                 } else {
                     res[j++] = '{';
                 }
                 break;
             case ']':
                 res[j++] = '}';
                 break;
             default:
                 res[j++] = str[i];
                 break;

         }

     }
 

 /* FIXME: removing this code because anchored bytes are not sufficiently

    general for the purposes of yara rule to ClamAV sig conversions.
    1. ClamAV imposes a maximum value for the upper range limit of 32:
       #define AC_CH_MAXDIST 32
       Values larger cause an error in matcher-ac.c
    2. If the upper range values is not present, ClamAV sets the missing
       range value to be equal to the lower range value. This changes the
       semantic of yara jumps.
 */
 #ifdef YARA_ANCHOR_SUPPORT

     /* backward anchor overwrite, 2 (hex chars in one byte) */
     if ((ovr = strchr(res, '{')) && ((ovr - res) == 2)) {
         *ovr = '[';
         if ((ovr = strchr(ovr, '}')))
             *ovr = ']';
         else {

             free(res);

             if (ret) *ret = CL_EMALFDB;
             return NULL;
         }
     }
     /* forward anchor overwrite, 2 (hex chars in one byte) +1 (NULL char) */

     if ((ovr = strrchr(res, '}')) && ((res + j - ovr) == 3)) {

         *ovr = ']';
         if ((ovr = strrchr(res, '{')))
             *ovr = '[';
         else {

             free(res);

             if (ret) *ret = CL_EMALFDB;
             return NULL;
         }
     }

 #else
     if (((ovr = strchr(res, '{')) && ((ovr - res) == 2)) ||

         ((ovr = strrchr(res, '}')) && ((res + j - ovr) == 3))) {

         cli_errmsg("parse_yara_hex_string: Single byte subpatterns unsupported in ClamAV\n");
         free(res);
         if (ret != NULL)
             *ret = CL_EMALFDB;
         return NULL;
     }
 #endif

     if (ret)
         *ret = CL_SUCCESS;

     return res;

 }
 

 struct cli_ytable_entry {
     char *offset;
     char *hexstr;

     uint8_t sigopts;

 };

 struct cli_ytable {
     struct cli_ytable_entry **table;

     int32_t tbl_cnt;

 };

 static int32_t ytable_lookup(const char *hexsig)

 {

     (void)hexsig;

     /* TODO - WRITE ME! */
     return -1;
 }
 

 static cl_error_t ytable_add_attrib(struct cli_ytable *ytable, const char *hexsig, const char *value, int type)

 {
     int32_t lookup;
 
     if (!ytable || !value)
         return CL_ENULLARG;
 
     if (!hexsig)

         lookup = ytable->tbl_cnt - 1; /* assuming to attach to current string */

     else
         lookup = ytable_lookup(hexsig);
 
     if (lookup < 0) {

         cli_yaramsg("ytable_add_attrib: hexsig cannot be found\n");

         return CL_EARG;
     }
 

     if (type) {

         /* add to sigopts */
         switch (*value) {

             case 'i':
                 ytable->table[lookup]->sigopts |= ACPATT_OPTION_NOCASE;
                 break;
             case 'f':
                 ytable->table[lookup]->sigopts |= ACPATT_OPTION_FULLWORD;
                 break;
             case 'w':
                 ytable->table[lookup]->sigopts |= ACPATT_OPTION_WIDE;
                 break;
             case 'a':
                 ytable->table[lookup]->sigopts |= ACPATT_OPTION_ASCII;
                 break;
             default:
                 cli_yaramsg("ytable_add_attrib: invalid sigopt %02x\n", *value);
                 return CL_EARG;

         }

     } else {

         /* overwrite the previous offset */
         if (ytable->table[lookup]->offset)
             free(ytable->table[lookup]->offset);

         ytable->table[lookup]->offset = cli_strdup(value);
 
         if (!ytable->table[lookup]->offset) {
             cli_yaramsg("ytable_add_attrib: ran out of memory for offset\n");
             return CL_EMEM;
         }

     }
 
     return CL_SUCCESS;
 }
 

 /* function is dumb - TODO - rewrite using hashtable */
 static int ytable_add_string(struct cli_ytable *ytable, const char *hexsig)
 {
     struct cli_ytable_entry *new;
     struct cli_ytable_entry **newtable;
     int ret;
 
     if (!ytable || !hexsig)
         return CL_ENULLARG;
 
     new = cli_calloc(1, sizeof(struct cli_ytable_entry));
     if (!new) {
         cli_yaramsg("ytable_add_string: out of memory for new ytable entry\n");
         return CL_EMEM;
     }
 
     new->hexstr = cli_strdup(hexsig);
     if (!new->hexstr) {
         cli_yaramsg("ytable_add_string: out of memory for hexsig copy\n");
         free(new);
         return CL_EMEM;
     }
 
     ytable->tbl_cnt++;
     newtable = cli_realloc(ytable->table, ytable->tbl_cnt * sizeof(struct cli_ytable_entry *));
     if (!newtable) {
         cli_yaramsg("ytable_add_string: failed to reallocate new ytable table\n");
         free(new->hexstr);
         free(new);
         ytable->tbl_cnt--;
         return CL_EMEM;
     }
 

     newtable[ytable->tbl_cnt - 1] = new;
     ytable->table                 = newtable;

     if (CL_SUCCESS != (ret = ytable_add_attrib(ytable, NULL, "*", 0)) != CL_SUCCESS) {

         cli_yaramsg("ytable_add_string: failed to add default offset\n");
         free(new->hexstr);
         free(new);
         ytable->tbl_cnt--;
         return ret;
     }
 
     return CL_SUCCESS;
 }
 
 static void ytable_delete(struct cli_ytable *ytable)

 {

     int32_t i;

     if (!ytable)
         return;

     if (ytable->table) {

         for (i = 0; i < ytable->tbl_cnt; ++i) {
             free(ytable->table[i]->offset);
             free(ytable->table[i]->hexstr);

             free(ytable->table[i]);

         }

         free(ytable->table);

     }
 }
 

 /* should only operate on HEX STRINGS */

 static int yara_hexstr_verify(YR_STRING *string, const char *hexstr, uint32_t *lsigid, struct cl_engine *engine, unsigned int options)

 {
     int ret = CL_SUCCESS;
 
     /* Quick Check 1: NULL String */
     if (!hexstr || !string) {
         cli_warnmsg("load_oneyara[verify]: string is empty\n");
         return CL_ENULLARG;
     }
 
     /* Quick Check 2: String Too Short */

     if (strlen(hexstr) / 2 < CLI_DEFAULT_AC_MINDEPTH) {

         cli_warnmsg("load_oneyara[verify]: string is too short: %s\n", string->identifier);

         return CL_EMALFDB;
     }
 

     /* Long Check: Attempt to load hexstr */

     if (CL_SUCCESS != (ret = cli_sigopts_handler(engine->test_root, "test-hex", hexstr, 0, 0, 0, "*", 0, lsigid, options)) != CL_SUCCESS) {

         if (ret == CL_EMALFDB) {
             cli_warnmsg("load_oneyara[verify]: recovered from database loading error\n");
             /* TODO: if necessary, reset testing matcher if error occurs */
             cli_warnmsg("load_oneyara[verify]: string failed test insertion: %s\n", string->identifier);

         }

         return ret;

     }
 

     return CL_SUCCESS;

 }
 

 static unsigned int yara_total, yara_loaded, yara_malform, yara_empty, yara_complex;

 #define YARATARGET0 "Target:0"
 #define YARATARGET1 "Target:1"
 #define EPSTR "EP+0:"

 /* yara has no apparent cap on the number of strings; TODO - should we have one? */
 /* function base off load_oneldb */

 static int load_oneyara(YR_RULE *rule, int chkpua, struct cl_engine *engine, unsigned int options, unsigned int *sigs)

 {

     YR_STRING *string;

     struct cli_ytable ytable;

     size_t i;
     int str_error = 0, ret = CL_SUCCESS;

     struct cli_lsig_tdb tdb;
     uint32_t lsigid[2];
     struct cli_matcher *root;

     struct cli_ac_lsig **newtable, *lsig, *tsig = NULL;

     unsigned short target = 0;

     char *logic = NULL, *target_str = NULL;

     char *newident = NULL;

     /* size_t lsize; */       // only used in commented out code
     /* char *exp_op = "|"; */ // only used in commented out code

     cli_yaramsg("load_oneyara: attempting to load %s\n", rule->identifier);

     if (!rule) {
         cli_errmsg("load_oneyara: empty rule passed as argument\n");
         return CL_ENULLARG;
     }

     /* PUA and IGN checks */
     if (chkpua && cli_chkpua(rule->identifier, engine->pua_cats, options))

         return CL_SUCCESS;
 

     if (engine->ignored && cli_chkign(engine->ignored, rule->identifier, rule->identifier)) {

         return CL_SUCCESS;
     }

     newident = cli_malloc(strlen(rule->identifier) + 5 + 1);

     if (!newident) {

         cli_errmsg("cli_loadyara(): newident == NULL\n");
         return CL_EMEM;

     }
 

     snprintf(newident, strlen(rule->identifier) + 5 + 1, "YARA.%s", rule->identifier);

     if (engine->cb_sigload && engine->cb_sigload("yara", newident, ~options & CL_DB_OFFICIAL, engine->cb_sigload_ctx)) {

         cli_dbgmsg("cli_loadyara: skipping %s due to callback\n", newident);
         free(newident);

         (*sigs)--;
         return CL_SUCCESS;
     }

 
     memset(&ytable, 0, sizeof(ytable));

 
     /*** rule specific checks ***/

 #ifdef YARA_FINISHED
     if (RULE_IS_PRIVATE(rule)) {
         cli_warnmsg("load_oneyara: private modifier for yara rule is unsupported\n");
         cli_yaramsg("RULE_IS_PRIVATE                yes\n");
     }
     if (RULE_IS_GLOBAL(rule)) {
         cli_warnmsg("load_oneyara: global modifier for yara rule is unsupported\n");
         cli_yaramsg("RULE_IS_GLOBAL                 yes\n");
     }
     if ((rule->g_flags) & RULE_GFLAGS_REQUIRE_FILE) {
         cli_warnmsg("load_oneyara: RULE_GFLAGS_REQUIRE_FILE for yara rule is unsupported\n");
         cli_yaramsg("RULE_GFLAGS_REQUIRE_FILE       yes\n");
     }
 
     if (RULE_IS_NULL(rule) || ((rule->g_flags) & RULE_GFLAGS_REQUIRE_EXECUTABLE)) {

         cli_warnmsg("load_oneyara: skipping %s due to unsupported rule gflags\n", newident);

 
         cli_yaramsg("RULE_IS_NULL                   %s\n", RULE_IS_NULL(rule) ? "yes" : "no");
         cli_yaramsg("RULE_GFLAGS_REQUIRE_EXECUTABLE %s\n", ((rule->g_flags) & RULE_GFLAGS_REQUIRE_EXECUTABLE) ? "yes" : "no");
 

         free(newident);

         (*sigs)--;
         return CL_SUCCESS;
     }

 #else
     /*
     cli_warnmsg("load_oneyara: yara support is incomplete, rule flags are ignored\n");
 
     if (RULE_IS_PRIVATE(rule))
         cli_yaramsg("RULE_IS_PRIVATE                yes\n");
     if (RULE_IS_GLOBAL(rule))
         cli_yaramsg("RULE_IS_GLOBAL                 yes\n");
     if (RULE_IS_NULL(rule))
         cli_yaramsg("RULE_IS_NULL                   yes\n");
     if ((rule->g_flags) & RULE_GFLAGS_REQUIRE_FILE)
         cli_yaramsg("RULE_GFLAGS_REQUIRE_FILE       yes\n");
     if ((rule->g_flags) & RULE_GFLAGS_REQUIRE_EXECUTABLE)
         cli_yaramsg("RULE_GFLAGS_REQUIRE_EXECUTABLE yes\n");
     */

 #endif

     if (engine->cb_sigload && engine->cb_sigload("yara", newident, ~options & CL_DB_OFFICIAL, engine->cb_sigload_ctx)) {

         cli_dbgmsg("load_oneyara: skipping %s due to callback\n", newident);

         (*sigs)--;

         free(newident);

         return CL_SUCCESS;
     }
 
     /*** verification step - can clamav load it?       ***/
     /*** initial population pass for the strings table ***/

     STAILQ_FOREACH(string, &rule->strings, link)
     {

         char *substr = NULL;

 
         /* string type handler */
         if (STRING_IS_NULL(string)) {

             cli_warnmsg("load_oneyara: skipping NULL string %s\n", newident);

             //str_error++; /* kill the insertion? */
             continue;

 #ifdef YARA_FINISHED
         } else if (STRING_IS_LITERAL(string)) {
             /* TODO - handle literal strings, short-circuits other string type handling */

             cli_yaramsg("load_oneyara: literal string: [%.*s] => [%s]\n", string->length, string->string, substr);

 #else
         } else if (STRING_IS_LITERAL(string)) {
             cli_errmsg("load_oneyara: literal strings are unsupported, reorganize existing code\n");

 #endif

         } else if (STRING_IS_HEX(string)) {

             substr = parse_yara_hex_string(string, &ret);
             if (ret != CL_SUCCESS) {
                 cli_errmsg("load_oneyara: error in parsing yara hex string\n");
                 str_error++;
                 break;
             }

             /* handle lack of hexstr support here in order to suppress */

             /* initialize testing matcher */

             if (!engine->test_root) {

                 engine->test_root = (struct cli_matcher *)MPOOL_CALLOC(engine->mempool, 1, sizeof(struct cli_matcher));

                 if (!engine->test_root) {
                     cli_errmsg("load_oneyara[verify]: cannot allocate memory for test cli_matcher\n");

                     free(substr);

                     return CL_EMEM;
                 }
 #ifdef USE_MPOOL
                 engine->test_root->mempool = engine->mempool;
 #endif

                 if (CL_SUCCESS != (ret = cli_ac_init(engine->test_root, engine->ac_mindepth, engine->ac_maxdepth, engine->dconf->other & OTHER_CONF_PREFILTERING))) {

                     cli_errmsg("load_oneyara: cannot initialize test ac root\n");

                     free(substr);

                     return ret;
                 }
             }
 
             /* generate a test lsig if one does not exist */
             if (!tsig) {
                 /*** populating lsig ***/

                 tsig = (struct cli_ac_lsig *)MPOOL_CALLOC(engine->mempool, 1, sizeof(struct cli_ac_lsig));

                 if (!tsig) {

                     cli_errmsg("load_oneyara: cannot allocate memory for test lsig\n");

                     free(substr);

                     return CL_EMEM;
                 }
 
                 root = engine->test_root;
 
                 tsig->type = CLI_YARA_NORMAL;
                 lsigid[0] = tsig->id = root->ac_lsigs;
 
                 root->ac_lsigs++;

                 newtable = (struct cli_ac_lsig **)MPOOL_REALLOC(engine->mempool, root->ac_lsigtable, root->ac_lsigs * sizeof(struct cli_ac_lsig *));

                 if (!newtable) {

                     root->ac_lsigs--;
                     cli_errmsg("load_oneyara: cannot allocate test root->ac_lsigtable\n");

                     MPOOL_FREE(engine->mempool, tsig);

                     free(substr);

                     return CL_EMEM;
                 }
 
                 newtable[root->ac_lsigs - 1] = tsig;

                 root->ac_lsigtable           = newtable;

             }
 
             /* attempt to insert hexsig */
             lsigid[1] = 0;

             ret       = yara_hexstr_verify(string, substr, lsigid, engine, options);

             if (ret != CL_SUCCESS) {

                 str_error++;

                 free(substr);

                 break;

             }

             cli_yaramsg("load_oneyara: hex string: [%.*s] => [%s]\n", string->length, string->string, substr);

 
             ytable_add_string(&ytable, substr);
             free(substr);

         } else if (STRING_IS_REGEXP(string)) {

             /* TODO - rewrite to NOT use PCRE_BYPASS */

 #if HAVE_PCRE

             size_t length = strlen(PCRE_BYPASS) + string->length + 3;

 
             substr = cli_calloc(length, sizeof(char));
             if (!substr) {
                 cli_errmsg("load_oneyara: cannot allocate memory for converted regex string\n");
                 str_error++;
                 ret = CL_EMEM;
                 break;
             }
 

             snprintf(substr, length, "%s/%.*s/", PCRE_BYPASS, string->length, string->string);

             cli_yaramsg("load_oneyara: regex string: [%.*s] => [%s]\n", string->length, string->string, substr);

 
             ytable_add_string(&ytable, substr);
             free(substr);

 #else

             cli_warnmsg("cli_loadyara: %s uses PCREs but support is disabled\n", newident);

             str_error++;
             ret = CL_SUCCESS;
             break;
 #endif

         } else {

             /* TODO - extract the string length to handle NULL hex-escaped characters
              * For now, we'll just use the strlen we get which crudely finds the length
              */

             size_t length  = string->length;
             size_t totsize = 2 * length + 1;

             if (length < CLI_DEFAULT_AC_MINDEPTH) {

                 cli_warnmsg("load_oneyara: string is too short %s\n", newident);

                 str_error++;
                 continue;
             }
 
             substr = cli_calloc(totsize, sizeof(char));
             if (!substr) {
                 cli_errmsg("load_oneyara: cannot allocate memory for converted generic string\n");
                 str_error++;
                 ret = CL_EMEM;
                 break;
             }
 

             for (i = 0; i < length; ++i) {

                 size_t len = strlen(substr);

                 snprintf(substr + len, totsize - len, "%02x", string->string[i]);

             }
 

             cli_yaramsg("load_oneyara: generic string: [%.*s] => [%s]\n", string->length, string->string, substr);

 
             ytable_add_string(&ytable, substr);
             free(substr);

         }
 

         /* modifier handler */
         if (STRING_IS_NO_CASE(string)) {

             cli_yaramsg("STRING_IS_NO_CASE         %s\n", STRING_IS_SINGLE_MATCH(string) ? "yes" : "no");

             if (CL_SUCCESS != (ret = ytable_add_attrib(&ytable, NULL, "i", 1)) != CL_SUCCESS) {

                 cli_warnmsg("load_oneyara: failed to add 'nocase' sigopt\n");

                 str_error++;
                 break;
             }

         }
         if (STRING_IS_ASCII(string)) {

             cli_yaramsg("STRING_IS_ASCII           %s\n", STRING_IS_SINGLE_MATCH(string) ? "yes" : "no");

             if (CL_SUCCESS != (ret = ytable_add_attrib(&ytable, NULL, "a", 1)) != CL_SUCCESS) {

                 cli_warnmsg("load_oneyara: failed to add 'ascii' sigopt\n");

                 str_error++;
                 break;
             }

         }
         if (STRING_IS_WIDE(string)) {

             cli_yaramsg("STRING_IS_WIDE            %s\n", STRING_IS_SINGLE_MATCH(string) ? "yes" : "no");

             /* handle lack of 'wide' support for regex here in order to suppress */
             if (STRING_IS_REGEXP(string)) {
                 cli_warnmsg("load_oneyara[verify]: wide modifier [w] is not supported for regex subsigs\n");
                 str_error++;
                 break;
             }

             if (CL_SUCCESS != (ret = ytable_add_attrib(&ytable, NULL, "w", 1)) != CL_SUCCESS) {

                 cli_warnmsg("load_oneyara: failed to add 'wide' sigopt\n");

                 str_error++;
                 break;
             }

         }
         if (STRING_IS_FULL_WORD(string)) {

             cli_yaramsg("STRING_IS_FULL_WORD       %s\n", STRING_IS_SINGLE_MATCH(string) ? "yes" : "no");

             if (CL_SUCCESS != (ret = ytable_add_attrib(&ytable, NULL, "f", 1)) != CL_SUCCESS) {

                 cli_warnmsg("load_oneyara: failed to add 'fullword' sigopt\n");

                 str_error++;
                 break;
             }

         }
 

 #ifdef YARA_FINISHED

         /* special modifier handler */