/*
* A fast filter for static patterns.
*
* Copyright (C) 2013-2019 Cisco Systems, Inc. and/or its affiliates. All rights reserved.
* Copyright (C) 2008-2013 Sourcefire, Inc.
*
* Authors: Török Edvin
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#if HAVE_CONFIG_H
#include "clamav-config.h"
#endif
#include "clamav.h"
#include "filtering.h"
#include "matcher-ac.h"
#include <string.h>
#include <assert.h>
#include "perflogging.h"
/* ----- shift-or filtering -------------- */
/*
* Description of algorithm:
*
* Multiple patterns are added to the filter.
* The filter retains an approximation of these patterns, which can lead to
* false positive matches, but not false negative matches.
*
* For each position in the filter we retain what qgrams can match at that
* position, for example (if we'd use characters as qgrams):
* pattern1: atu
* pattern2: bzf
* pattern3: xat
*
* filter accepts:
* [abx][tza][uft]
*
* But it also accepts (false positives):
* azu, azf, azt, ...
*
* It doesn't however accept:
* aaa, atz, ...
*
* This is implemented by having a bit-level state-machine with MAXSOPATLEN (=32) states,
* each active bit meaning that a state is active.
*
* The states are activated sequentially, eachtransition decision is made
* considering if we can accept the character at position X.
* Since we can start a match at any position, position 0 is
* reactivated each time.
* When the last position is activated, the filter reports a match.
* If we can't accept the character at position X, the state remains inactive,
* and further states aren't activated (unless we activate this state in the
* future).
*
* Essentially this is an automaton like this:
*
* /\ (a|b|x) (t|z|a) (u|f|t)
* [S1] ---------> [S2] -------> [S3] ---------> [S4] -> match
* \_______________/ |
* \_____________________________/
*
*
* But we are tracking multiple active states at each time (or run N automatons
* in parallel if you like, N = number of states).
*
* We can have S3 and S2 active, meaning that if the next character is
* acceptable, it transitions to S1,S3 and S4 being active, otherwise it
* transitions to S1 being active.
*
* Active states can either be represented as a binary 1 or 0, and using
* bit-shifting and masking.
* If we choose 1, we must use &, and after shifting always reactivate bit 0.
* If we choose 0, we must use |, and after shifting we don't need to do
* anything (since by shifting a 0 is implicitly introduced).
*
* This file implements the latter (shift-or) method.
*
* The discussion above considered pattern to be of same length (or truncated to
* be so). In reality patterns are of variable length, and we often have short
* pattern.
*
* Thus another bitmap was introduced, meaning that if (end[Q] == set), then
* a pattern can end at this position.
* Also we would fill the pattern's position filters quite quickly with only 256
* choices for a position, so the algorithm uses overlapping qgrams of length 2:
* 'abcd' is 3 qgrams: 'ab','bc','cd'
*
* The algorithm is very sensitive to the end[Q] filter, since it can have false
* positives due to short patterns!
* For optimal performance we need:
* - patterns as long as possible
* - probability for end[Q] to match low (avoid 0000, and other common case
* - choose the most "diverse" subset from a long pattern
*
* diverse = referring to what we are scanning, so that the filter rarely
* matches, so this actually means that we *want* to avoid adding more
* characters to the filter, if we have 2 patterns:
* abxfg, and dalabxpo, it may be preferable to shift the 2nd one so that we
* don't add new character at the beginning.
*
* With NDB signatures there are more challenges to overcome:
* e8??0000000aa
*
* will make the filter accept:
* e8<all-256-values-here>, <all-256-values>00, ... 000000aa
*
* We should delay the pattern end as long as possible, especially if it is 0000
* The problem is that now the filter accepts 0000 on position 3, regardless
* of what we have on position 1 (even if we have something else than e8), so
* we have to be very careful not to allow 0000 on first position too,
* otherwise the filter will happily accept 000000000000.
*
* To optimize cache usage there are 2 end filters, one character (fits L1), and one qgram
* based (fits L2), both must match for the filter to consider it a match.
*
*
*/
/*#define DETAILED_DEBUG*/
#ifdef DETAILED_DEBUG
#define detailed_dbg cli_dbgmsg
#else
#define detailed_dbg(...)
#endif
#define BITMAP_CONTAINS(bmap, val) ((bmap)[(val) >> 5] & (1 << ((val) & 0x1f)))
#define BITMAP_INSERT(bmap, val) ((bmap)[(val) >> 5] |= (1 << ((val) & 0x1f)))
void filter_init(struct filter *m)
{
memset(m->B, ~0, sizeof(m->B));
memset(m->end, ~0, sizeof(m->end));
}
/* because we use uint32_t */
#define MAXSOPATLEN 8
static inline int filter_isset(const struct filter *m, unsigned pos, uint16_t val)
{
return !(m->B[val] & (1<<pos));
}
static inline void filter_set_atpos(struct filter *m, unsigned pos, uint16_t val)
{
if (!filter_isset(m, pos, val)) {
cli_perf_log_count(FILTER_LOAD, pos);
m->B[val] &= ~(1<<pos);
}
}
static inline int filter_end_isset(const struct filter *m, unsigned pos, uint16_t a)
{
return !(m->end[a] & (1<<pos));
}
static inline void filter_set_end(struct filter *m, unsigned pos, uint16_t a)
{
if (!filter_end_isset(m, pos, a)) {
cli_perf_log_count(FILTER_END_LOAD, pos);
m->end[a] &= ~(1 << pos);
}
}
#define MAX_CHOICES 8
/* just an arbitrary limit, if patterns are longer, we cut
* the filter can only use MAXSOPATLEN (32) characters,
* this longer buffer is needed so that we can choose the "best" subpattern from
* it */
#define MAXPATLEN 255
/* merge another pattern into the filter
* add('abc'); add('bcd'); will match [ab][bc][cd] */
int filter_add_static(struct filter *m, const unsigned char *pattern, unsigned long len, const char *name)
{
uint16_t q = 0;
uint8_t j, maxlen;
uint32_t best = 0xffffffff;
uint8_t best_pos = 0;
UNUSEDPARAM(name);
cli_perf_log_count(TRIE_ORIG_LEN, len > 8 ? 8 : len);
/* TODO: choose best among MAXCHOICES */
/* cut length */
if(len > MAXPATLEN) {
len = MAXPATLEN;
}
if(len < 2)
return -1;
/* we want subsigs to be as long as possible */
if (len > 4) {
maxlen = len - 4;
if (maxlen == 1) maxlen = 2;
} else
maxlen = 2;
for(j=0;(best < 100 && j<MAX_CHOICES) || (j < maxlen) ;j++) {
uint32_t num = MAXSOPATLEN;
uint8_t k;
if (j+2 > len)
break;
for(k=j;k<len-1 && (k-j < MAXSOPATLEN);k++) {
q = cli_readint16( &pattern[k] );
/* we want to favor subsigs that add as little as
* possible to the filter */
num += filter_isset(m, k-j, q) ? 0 : MAXSOPATLEN - (k-j);
if ((k == j || k == j+1) && (q == 0x0000 || q == 0xffff))
num += k==j ? 10000 : 1000;/* bad */
}
/* it is very important to keep the end set small */
num += 10*(filter_end_isset(m, k-j-1, q) ? 0 : 1);
/* it is very important to have signatures as long as possible
* */
num += 5*(MAXSOPATLEN - (k-j));
/* if we are lower length than threshold penalize */
if (k-j+1 < 4)
num += 200;
/* favour longer patterns */
num -= (2*MAXSOPATLEN - (k + 1+j))*(k-j)/2;
if (num < best) {
best = num;
best_pos = j;
}
}
assert(best_pos < len-1);
if (pattern[best_pos] == 0 && pattern[best_pos+1] == 0) {
detailed_dbg("filter (warning): subsignature begins with zero (static): %s\n", name);
}
pattern += best_pos;
len -= best_pos;
/* cut length */
if(len > MAXSOPATLEN) {
len = MAXSOPATLEN;
}
/* Shift-Or like preprocessing */
for(j=0;j < len-1;j++) {
/* use overlapping little-endian 2-grams. We need them overlapping because matching can start at any position */
q = cli_readint16( &pattern[j] );
filter_set_atpos(m, j, q);
}
/* we use variable length patterns, use last character to mark pattern end,
* can lead to false positives.*/
/* mark that at state j, the q-gram q can end the pattern */
if(j) {
j--;
filter_set_end(m, j, q);
}
return j+2;
}
struct char_spec {
/* if non-null i-th character = alt[start + step*i]; start+step*i < end;
*/
struct cli_ac_special *alt;
uint8_t start;
uint8_t end;
uint8_t step;
uint8_t negative;
};
static inline unsigned char spec_ith_char(const struct char_spec *spec, unsigned i)
{
const struct cli_ac_special *alt = spec->alt;
if (alt) {
assert (alt->type == 1);
assert (i < alt->num);
return (alt->alt).byte[i];
}
return i;
}
#ifndef MIN
#define MIN(a,b) ((a) < (b) ? (a) : (b))
#endif
#define SPEC_FOREACH(spec0, k0, spec1, k1) do {\
unsigned char c0 = spec_ith_char(spec0, k0);\
unsigned char c1 = spec_ith_char(spec1, k1);\
unsigned c0end, c1end, cc0,cc1;\
c0end = spec0->negative ? 255 : c0;\
c1end = spec1->negative ? 255 : c1;\
cc0 = spec0->negative ? 0 : c0;\
cc1 = spec1->negative ? 0 : c1;\
for (;cc0 <= c0end;cc0++) {\
for (;cc1 <= c1end; cc1++) {\
uint16_t a = cc0 | (cc1<<8);\
if (spec0->negative && cc0 == c0)\
continue;\
if (spec1->negative && cc1 == c1)\
continue;
#define SPEC_END_FOR }}} while(0)
enum badness {
reject,
/* try to avoid if possible */
avoid_first,
avoid_anywhere, /* includes avoid_first! */
/* not that bad, but still not best */
dontlike,
acceptable,
like
};
static inline void get_score(enum badness badness, unsigned i, const struct filter *m, const struct char_spec *spec0, const struct char_spec *spec1, int32_t *score, int32_t *score_end)
{
int32_t base;
unsigned k0, k1, num_introduced = 0, num_end_introduced = 0;
switch (badness) {
case reject:
/* not reached */
assert(0);
base = -0x7fffff;
break;
case avoid_first:
if (!i)
base = -0x700000;
else
base = 0;
break;
case avoid_anywhere:
if (!i)
base = -0x720000;
else
base = -0x1000;
break;
case dontlike:
base = 0;
break;
case acceptable:
base = 0x200;
break;
case like:
/* a bit better only */
base = 0x201;
break;
}
if (base < 0) {
*score = base;
*score_end = base;
return;
}
/* at most 256 iterations here, otherwise base would be negative */
for(k0=spec0->start;k0 <= spec0->end;k0 += spec0->step) {
for(k1=spec1->start;k1 <= spec1->end;k1 += spec1->step) {
SPEC_FOREACH(spec0, k0, spec1, k1) {
num_introduced += filter_isset(m, i, a);
num_end_introduced += filter_end_isset(m, i, a);
} SPEC_END_FOR;
}
}
*score = base - num_introduced;
*score_end = base - num_end_introduced;
if (badness == avoid_first && i) {
/* what is bad to begin with, is bad at end too */
*score_end -= 0x1000;
}
}
struct choice {
enum badness base;
unsigned begin;
unsigned len;
};
static inline void add_choice(struct choice *choices, unsigned *cnt, unsigned i, unsigned ie, enum badness badness)
{
struct choice *choice;
int i_neg = -1;
assert(ie < MAXPATLEN);
if (ie < i+1)
return;
if (*cnt >= MAX_CHOICES)
return;
if (badness > avoid_first && *cnt >= (MAX_CHOICES >> 1)) {
unsigned j;
/* replace very bad picks if we're full */
for (j=0;j<*cnt;j++) {
if (choices[j].base < badness) {
if (i_neg == -1 || choices[j].base < choices[i_neg].base) {
i_neg = j;
}
}
}
}
if (i_neg != -1) {
choice = &choices[i_neg];
} else {
choice = &choices[(*cnt)++];
}
choice->begin = i;
choice->len = ie - i + 1;
choice->base = badness;
}
static inline int32_t spec_iter(const struct char_spec *spec)
{
unsigned count;
assert(spec->step);
count = (spec->step + spec->end - spec->start)/spec->step;
if (spec->negative) /* all chars except itself are added */
count *= 254;
return count;
}
int filter_add_acpatt(struct filter *m, const struct cli_ac_patt *pat)
{
unsigned i, j = 0, stop = 0, l=0;
uint16_t k0, k1;
struct char_spec chars[MAXPATLEN];
enum badness char_badness[MAXPATLEN];
unsigned char patc[MAXPATLEN];
unsigned altcnt = 0;
int32_t best_score = -0x7fffffff;
unsigned best_score_i = 0;
unsigned best_score_len = 0;
struct char_spec *spec0 = NULL, *spec1 = NULL;
struct choice choices[MAX_CHOICES];
unsigned choices_cnt = 0;
unsigned prefix_len = pat->prefix_length[0];
unsigned speci;
j = MIN(prefix_len + pat->length[0], MAXPATLEN);
for(i=0;i<j;i++) {
const uint16_t p = i < prefix_len ? pat->prefix[i] : pat->pattern[i - prefix_len];
if ((p&CLI_MATCH_METADATA) != CLI_MATCH_CHAR)
break;
patc[i] = (uint8_t)p;
}
if (i == j) {
/* all static, use add_static it has better heuristics for this
* case */
return filter_add_static(m, patc, j, pat->virname);
}
cli_perf_log_count(TRIE_ORIG_LEN, j > 8 ? 8 : j);
i = 0;
if (!prefix_len) {
while ((pat->pattern[i] & CLI_MATCH_METADATA) == CLI_MATCH_SPECIAL) {
/* we support only ALT_CHAR, skip the rest */
if (pat->special_table[altcnt]->type == 1)
break;
altcnt++;
i++;
}
}
/* transform AC characters into our representation */
for (speci=0;i<j && !stop; speci++,i++) {
struct char_spec *spec = &chars[speci];
const uint16_t p = i < prefix_len ? pat->prefix[i] : pat->pattern[i - prefix_len];
spec->alt = NULL;
spec->negative = 0;
switch (p & CLI_MATCH_METADATA) {
case CLI_MATCH_CHAR:
spec->start = spec->end = (uint8_t)p;
spec->step = 1;
break;
case CLI_MATCH_NOCASE:
if ((uint8_t)p >= 'a' && (uint8_t)p <= 'z') {
spec->start = (uint8_t)p - ('a' - 'A');
spec->end = (uint8_t)p;
spec->step = ('a' - 'A');
}
else if ((uint8_t)p >= 'A' && (uint8_t)p <= 'Z') {
spec->start = (uint8_t)p;
spec->end = (uint8_t)p + ('a' - 'A');
spec->step = ('a' - 'A');
}
else {
spec->start = spec->end = (uint8_t)p;
spec->step = 1;
}
break;
case CLI_MATCH_IGNORE:
spec->start = 0x00;
spec->end = 0xff;
spec->step = 1;
break;
case CLI_MATCH_SPECIAL:
assert(pat->special_table);
/* assert(altcnt < pat->alt); */
assert(pat->special_table[altcnt]);
spec->negative = pat->special_table[altcnt]->negative;
switch (pat->special_table[altcnt++]->type) {
case 1: /* ALT_CHAR */
spec->start = 0;
spec->end = pat->special_table[altcnt-1]->num - 1;
spec->step = 1;
spec->alt = pat->special_table[altcnt-1];
break;
default:
stop = 1;
break; /* TODO: should something be done here?
* */
}
break;
case CLI_MATCH_NIBBLE_HIGH:
spec->start = (p & 0xf0);
spec->end = spec->start | 0x0f;
spec->step = 1;
break;
case CLI_MATCH_NIBBLE_LOW:
spec->start = (p & 0xf);
spec->end = 0xf0 | spec->start;
spec->step = 0x10;
break;
default:
cli_errmsg("filtering: unknown wildcard character: %d\n", p);
return -1;
}
}
if (stop) --speci;
j = speci;
if (j < 2) {
if (stop)
cli_warnmsg("Don't know how to create filter for: %s\n",pat->virname);
else
cli_warnmsg("Subpattern too short: %s\n", pat->virname);
return -1;
}
for(i=0;i<j-1;i++) {
int32_t num_iter;
/* new qgrams added to the filter */
spec0 = &chars[i];
spec1 = &chars[i+1];
num_iter = spec_iter(spec0) * spec_iter(spec1);
if (num_iter >= 0x100) {
if (num_iter == 0x10000)
char_badness[i] = reject;
else
char_badness[i] = avoid_anywhere;
} else {
int8_t binary = 0;
enum badness scor = acceptable;
for(k0=spec0->start;k0 <= spec0->end;k0 += spec0->step) {
for(k1=spec1->start;k1 <= spec1->end;k1 += spec1->step) {
unsigned char c0 = spec_ith_char(spec0, k0);
unsigned char c1 = spec_ith_char(spec1, k1);
if (spec0->negative || spec1->negative) {
scor = avoid_anywhere;
break;
}
if ((!c0 && !c1) || (c0 == 0xff && c1 == 0xff)) {
scor = avoid_first;
break;
}
if (c0 == c1) {
scor = dontlike;
break;
}
if ((c0 < 32 || c0 > 127) && (c1 < 32 || c1 >127))
binary = 1;
}
}
if (scor == acceptable && binary) {
/* slightly favor binary */
scor = like;
}
char_badness[i] = scor;
}
}
/* try to choose best subpattern */
/* calculating the score for all possible i start pos
* and all possible length is too slow, so choose best among N choices
* only */
for (i=0;i<j-1 && choices_cnt < MAX_CHOICES;i++) {
enum badness base0 = like, base1 = like;
unsigned kend = MIN(j-1, (i + MAXSOPATLEN)&~1), k;
int ki = -0xff;
/* add 2 scores: pattern with max length, one where we stop at
* first negative, and one we stop at last positive, but never
* include reject */
assert(kend-1 < j-1);
if (char_badness[i] == reject)
continue;
if ((char_badness[i] == avoid_anywhere || char_badness[i] == avoid_first)
&& choices_cnt > 0)
/* if we have another choice don't choose this */
continue;
while ((kend > i+3) && char_badness[kend-1] == reject) kend--;
for (k=i;k<kend;k++) {
enum badness badness = char_badness[k];
if (badness < acceptable) {
if (badness == reject) {
/* this is a never pick */
kend = k;
break;
}
if (badness == avoid_first && k != i)
badness = dontlike;
if (k == i && badness == avoid_anywhere)
badness = avoid_first;
if (ki == -0xff)
ki = k;
}
base0 = MIN(base0, badness);
if (ki == -0xff)
base1 = MIN(base1, badness);
}
add_choice(choices, &choices_cnt, i, kend, base0);
if (ki > (int)i) {
/* ki|ki+1|??| */
/* try subpattern from after the wildcard */
i = ki;
}
/* if score is positive, it replaces a negative choice */
}
for(l=0;l<choices_cnt;l++) {
int32_t score;
unsigned kend;
unsigned k;
i = choices[l].begin;
kend = i + choices[l].len;
score = 0;
for(k = i; k < kend-1; k++) {
unsigned p = k - i;
int32_t iscore, score_end;
assert(k < j);
get_score(char_badness[k], p, m, &chars[k], &chars[k+1],
&iscore, &score_end);
/* give more importance to the score of the characters
* at the beginning */
/* TODO: tune magic number here */
if (p < 6) {
iscore *= (6-p);
score_end *= (6-p);
}
score += iscore;
if (score + score_end > best_score) {
/* we may have negative scores, so truncating
* the pattern could actually get us a higher
* score */
best_score = score + score_end;
best_score_len = p + 2;
best_score_i = i;
assert(i + best_score_len <= j);
}
}
}
if (best_score <= -0x7fffffff) {
cli_warnmsg("filter rejecting %s due to very bad score: %ld\n", pat->virname, (long)best_score);
return -1;
}
if (choices_cnt == 0) {
cli_warnmsg("filter rejecting %s because there are no viable choices", pat->virname);
return -1;
}
assert(best_score_len >= 2);
detailed_dbg("filter %s score: %ld, %u (+ %u)\n", pat->virname, (long)best_score, best_score_i, best_score_len);
/* Shift-Or like preprocessing */
assert(1 < best_score_len);
for (i=0;i < best_score_len-1;i++) {
spec0 = &chars[best_score_i + i];
spec1 = &chars[best_score_i + i + 1];
/* use overlapping little-endian 2-grams, overlapping because match can start
* at any position (including odd) */
for(k0=spec0->start;k0 <= spec0->end;k0 += spec0->step) {
for(k1=spec1->start;k1 <= spec1->end;k1 += spec1->step) {
SPEC_FOREACH(spec0, k0, spec1, k1) {
if (!cc0 && !cc1 && !i) {
detailed_dbg("filter (warning): subsignature begins with zero: %s\n",pat->virname);
}
filter_set_atpos(m, i, a);
} SPEC_END_FOR;
}
}
}
j = best_score_len - 2;
if (spec0 && spec1) {
for (k0=spec0->start;k0 <= spec0->end;k0 += spec0->step) {
for (k1=spec1->start;k1 <= spec1->end;k1 += spec1->step) {
SPEC_FOREACH(spec0, k0, spec1, k1) {
if (!cc0 && !cc1) {
detailed_dbg("filter (warning): subsignature ends with zero: %s\n",pat->virname);
}
filter_set_end(m, j, a);
} SPEC_END_FOR;
}
}
}
return j+2;
}
/* state 11110011 means that we may have a match of length min 4, max 5 */
__hot__ int filter_search_ext(const struct filter *m, const unsigned char *data, unsigned long len, struct filter_match_info *inf)
{
size_t j;
uint8_t state = ~0;
const uint8_t *B = m->B;
const uint8_t *End = m->end;
if (len < 2) return -1;
/* look for first match */
for (j=0; j < len-1;j++) {
uint8_t match_state_end;
const uint16_t q0 = cli_readint16( &data[j] );
state = (state << 1) | B[q0];
match_state_end = state | End[q0];
if (match_state_end != 0xff) {
inf->first_match = j;
return 0;
}
}
/* no match, inf is invalid */
return -1;
}
/* this is like a FSM, with multiple active states at the same time.
* each bit in "state" means an active state, when a char is encountered
* we determine what states can remain active.
* The FSM transition rules are expressed as bit-masks */
long filter_search(const struct filter *m, const unsigned char *data, unsigned long len)
{
size_t j;
uint8_t state = ~0;
const uint8_t *B = m->B;
const uint8_t *End = m->end;
/* we use 2-grams, must be higher than 1 */
if(len < 2) return -1;
/* Shift-Or like search algorithm */
for(j=0;j < len-1; j++) {
const uint16_t q0 = cli_readint16( &data[j] );
uint8_t match_end;
state = (state << 1) | B[q0];
/* state marks with a 0 bit all active states
* End[q0] marks with a 0 bit all states where the q-gram 'q' can end a pattern
* if we got two 0's at matching positions, it means we encountered a pattern's end */
match_end = state | End[q0];
if(match_end != 0xff) {
/* if state is reachable, and this character can finish a pattern, assume match */
/* to reduce false positives check if qgram can finish the pattern */
/* return position of probable match */
/* find first 0 starting from MSB, the position of that bit as counted from LSB, is the length of the
* longest pattern that could match */
return j >= MAXSOPATLEN ? j - MAXSOPATLEN : 0;
}
}
/* no match */
return -1;
}