Browse code

Support for macros in logical subsignatures (bb #164).

In the LDB there is (one or more) special subsignature ${min-max}MACROID$,
which means:
must match any signature from group MACROID (for current filetype),
and the match must occur at a distance of min-max from the start(!) of the
previous logical subsignature match.
It also has the sideeffect of making the previous subsignature considered a
match only if both that and the macro matches. The offset of first match for
the previous logical subsig will be the offset where the {min-max} distance is
satisfied.

The macro logical subsignature will have a count of 0 (if it didn't match
together with the previous subsig), or a count of 1 if it did.

The matches can occur anywhere (even in
different ac scan buffers), since I don't call cli_ac_scanbuff I just use the
offset of first match (which we have for the bytecode anyway).

There can be at most 32 macro groups, signatures are added to a macro group by
using $MACROID$ as offset.

For example pdb entries could be converted to PDB:3:$0:<hexsig of domainname>
if we assign macro id 0 to PDB (and we can assign 31 more macro ids to
whatever).

Example:
test.ldb:
TestMacro;Target:0;0&1;616161;${3-4}12$

test.ndb:
D:0:$12:6262
D:0:$12:6363
D:0:$11:6262

test.dat:
aaaaxccdd

test-nomatch.dat:
aaaaxxxccdd

Török Edvin authored on 2010/02/08 20:45:03
Showing 5 changed files
... ...
@@ -952,6 +952,8 @@ int cli_ac_initdata(struct cli_ac_data *data, uint32_t partsigs, uint32_t lsigs,
952 952
 		data->lsigsuboff[i][j] = CLI_OFF_NONE;
953 953
 	}
954 954
     }
955
+    for (i=0;i<32;i++)
956
+	data->macro_lastmatch[i] = CLI_OFF_NONE;
955 957
 
956 958
     return CL_SUCCESS;
957 959
 }
... ...
@@ -1052,6 +1054,62 @@ inline static int ac_addtype(struct cli_matched_type **list, cli_file_t type, of
1052 1052
     return CL_SUCCESS;
1053 1053
 }
1054 1054
 
1055
+static inline void lsig_sub_matched(const struct cli_matcher *root, struct cli_ac_data *mdata, uint32_t lsigid1, uint32_t lsigid2, uint32_t realoff)
1056
+{
1057
+    if(mdata->lsigsuboff[lsigid1][lsigid2] == CLI_OFF_NONE)
1058
+	mdata->lsigsuboff[lsigid1][lsigid2] = realoff;
1059
+    else if (mdata->lsigcnt[lsigid1][lsigid2] == 1) {
1060
+	/* Check that the previous match had a macro match following it at the 
1061
+	 * correct distance. This check is only done after the 1st match.*/
1062
+	const struct cli_lsig_tdb *tdb = &root->ac_lsigtable[lsigid1]->tdb;
1063
+	const struct cli_ac_patt *macropt;
1064
+	uint32_t id, last_macro_match, smin, smax, last_macroprev_match;
1065
+	if (!tdb->macro_ptids)
1066
+	    return;
1067
+	id = tdb->macro_ptids[lsigid2];
1068
+	if (!id)
1069
+	    return;
1070
+	macropt = root->ac_pattable[id];
1071
+	smin = macropt->ch_mindist[0];
1072
+	smax = macropt->ch_maxdist[0];
1073
+	/* start of last macro match */
1074
+	last_macro_match = mdata->macro_lastmatch[macropt->sigid];
1075
+	/* start of previous lsig subsig match */
1076
+	last_macroprev_match = mdata->lsigsuboff[lsigid1][lsigid2];
1077
+	if (last_macro_match != CLI_OFF_NONE)
1078
+	    cli_dbgmsg("Checking macro match: %u + (%u - %u) == %u\n",
1079
+		       last_macroprev_match, smin, smax, last_macro_match);
1080
+	if (last_macro_match == CLI_OFF_NONE ||
1081
+	    last_macroprev_match + smin > last_macro_match ||
1082
+	    last_macroprev_match + smax < last_macro_match) {
1083
+	    cli_dbgmsg("Canceled false lsig macro match\n");
1084
+	    /* Previous match was false, cancel it and make this match the first
1085
+	     * one.*/
1086
+	    mdata->lsigcnt[lsigid1][lsigid2] = 0;
1087
+	    mdata->lsigsuboff[lsigid1][lsigid2] = realoff;
1088
+	} else {
1089
+	    /* mark the macro sig itself matched */
1090
+	    mdata->lsigcnt[lsigid1][lsigid2+1]++;
1091
+	    mdata->lsigsuboff[lsigid1][lsigid2+1] = last_macro_match;
1092
+	}
1093
+    }
1094
+    if (realoff != CLI_OFF_NONE) {
1095
+	mdata->lsigcnt[lsigid1][lsigid2]++;
1096
+    }
1097
+}
1098
+
1099
+void cli_ac_chkmacro(struct cli_matcher *root, struct cli_ac_data *data, unsigned lsigid1)
1100
+{
1101
+    const struct cli_lsig_tdb *tdb = &root->ac_lsigtable[lsigid1]->tdb;
1102
+    unsigned i;
1103
+    /* Loop through all subsigs, and if they are tied to macros check that the
1104
+     * macro matched at a correct distance */
1105
+    for (i=0;i<tdb->subsigs;i++) {
1106
+	lsig_sub_matched(root, data, lsigid1, i, CLI_OFF_NONE);
1107
+    }
1108
+}
1109
+
1110
+
1055 1111
 int cli_ac_scanbuff(const unsigned char *buffer, uint32_t length, const char **virname, void **customdata, struct cli_ac_result **res, const struct cli_matcher *root, struct cli_ac_data *mdata, uint32_t offset, cli_file_t ftype, struct cli_matched_type **ftoffset, unsigned int mode, const cli_ctx *ctx)
1056 1112
 {
1057 1113
 	struct cli_ac_node *current;
... ...
@@ -1084,7 +1142,7 @@ int cli_ac_scanbuff(const unsigned char *buffer, uint32_t length, const char **v
1084 1084
 	    patt = current->list;
1085 1085
 	    while(patt) {
1086 1086
 		bp = i + 1 - patt->depth;
1087
-		if(patt->offdata[0] != CLI_OFF_VERSION && !patt->next_same && (patt->offset_min != CLI_OFF_ANY) && (!patt->sigid || patt->partno == 1)) {
1087
+		if(patt->offdata[0] != CLI_OFF_VERSION && patt->offdata[0] != CLI_OFF_MACRO && !patt->next_same && (patt->offset_min != CLI_OFF_ANY) && (!patt->sigid || patt->partno == 1)) {
1088 1088
 		    if(patt->offset_min == CLI_OFF_NONE) {
1089 1089
 			patt = patt->next;
1090 1090
 			continue;
... ...
@@ -1116,6 +1174,10 @@ int cli_ac_scanbuff(const unsigned char *buffer, uint32_t length, const char **v
1116 1116
 				continue;
1117 1117
 			    }
1118 1118
 			    cli_dbgmsg("cli_ac_scanbuff: VI match for offset %x\n", realoff);
1119
+			} else if (patt->offdata[0] == CLI_OFF_MACRO) {
1120
+			    mdata->macro_lastmatch[patt->offdata[1]] = realoff;
1121
+			    pt = pt->next_same;
1122
+			    continue;
1119 1123
 			} else if(pt->offset_min != CLI_OFF_ANY && (!pt->sigid || pt->partno == 1)) {
1120 1124
 			    if(pt->offset_min == CLI_OFF_NONE) {
1121 1125
 				pt = pt->next_same;
... ...
@@ -1210,9 +1272,7 @@ int cli_ac_scanbuff(const unsigned char *buffer, uint32_t length, const char **v
1210 1210
 
1211 1211
 				} else { /* !pt->type */
1212 1212
 				    if(pt->lsigid[0]) {
1213
-					mdata->lsigcnt[pt->lsigid[1]][pt->lsigid[2]]++;
1214
-					if(mdata->lsigsuboff[pt->lsigid[1]][pt->lsigid[2]] == CLI_OFF_NONE)
1215
-					    mdata->lsigsuboff[pt->lsigid[1]][pt->lsigid[2]] = realoff;
1213
+					lsig_sub_matched(root, mdata, pt->lsigid[1], pt->lsigid[2], realoff);
1216 1214
 					pt = pt->next_same;
1217 1215
 					continue;
1218 1216
 				    }
... ...
@@ -1255,9 +1315,7 @@ int cli_ac_scanbuff(const unsigned char *buffer, uint32_t length, const char **v
1255 1255
 				}
1256 1256
 			    } else {
1257 1257
 				if(pt->lsigid[0]) {
1258
-				    mdata->lsigcnt[pt->lsigid[1]][pt->lsigid[2]]++;
1259
-				    if(mdata->lsigsuboff[pt->lsigid[1]][pt->lsigid[2]] == CLI_OFF_NONE)
1260
-					mdata->lsigsuboff[pt->lsigid[1]][pt->lsigid[2]] = realoff;
1258
+				    lsig_sub_matched(root, mdata, pt->lsigid[1], pt->lsigid[2], realoff);
1261 1259
 				    pt = pt->next_same;
1262 1260
 				    continue;
1263 1261
 				}
... ...
@@ -1701,7 +1759,7 @@ int cli_ac_addsig(struct cli_matcher *root, const char *virname, const char *hex
1701 1701
 	return ret;
1702 1702
     }
1703 1703
 
1704
-    if(new->offdata[0] != CLI_OFF_ANY && new->offdata[0] != CLI_OFF_ABSOLUTE) {
1704
+    if(new->offdata[0] != CLI_OFF_ANY && new->offdata[0] != CLI_OFF_ABSOLUTE && new->offdata[0] != CLI_OFF_MACRO) {
1705 1705
 	root->ac_reloff = (struct cli_ac_patt **) mpool_realloc2(root->mempool, root->ac_reloff, (root->ac_reloff_num + 1) * sizeof(struct cli_ac_patt *));
1706 1706
 	if(!root->ac_reloff) {
1707 1707
 	    cli_errmsg("cli_ac_addsig: Can't allocate memory for root->ac_reloff\n");
... ...
@@ -39,6 +39,7 @@ struct cli_ac_data {
39 39
     uint32_t **lsigcnt;
40 40
     uint32_t **lsigsuboff;
41 41
     uint32_t *offset;
42
+    uint32_t macro_lastmatch[32];
42 43
     /** Hashset for versioninfo matching */
43 44
     struct cli_hashset vinfo;
44 45
 };
... ...
@@ -88,6 +89,7 @@ struct cli_ac_result {
88 88
 
89 89
 int cli_ac_addpatt(struct cli_matcher *root, struct cli_ac_patt *pattern);
90 90
 int cli_ac_initdata(struct cli_ac_data *data, uint32_t partsigs, uint32_t lsigs, uint32_t reloffsigs, uint8_t tracklen);
91
+void cli_ac_chkmacro(struct cli_matcher *root, struct cli_ac_data *data, unsigned lsigid1);
91 92
 int cli_ac_chklsig(const char *expr, const char *end, uint32_t *lsigcnt, unsigned int *cnt, uint64_t *ids, unsigned int parse_only);
92 93
 void cli_ac_freedata(struct cli_ac_data *data);
93 94
 int cli_ac_scanbuff(const unsigned char *buffer, uint32_t length, const char **virname, void **customdata, struct cli_ac_result **res, const struct cli_matcher *root, struct cli_ac_data *mdata, uint32_t offset, cli_file_t ftype, struct cli_matched_type **ftoffset, unsigned int mode, const cli_ctx *ctx);
... ...
@@ -184,6 +184,17 @@ int cli_caloff(const char *offstr, struct cli_target_info *info, fmap_t *map, un
184 184
 	} else if(!strncmp(offcpy, "VI", 2)) {
185 185
 	    /* versioninfo */
186 186
 	    offdata[0] = CLI_OFF_VERSION;
187
+	} else if (strchr(offcpy, '$')) {
188
+	    if (sscanf(offcpy, "$%u$", &n) != 1) {
189
+		cli_errmsg("cli_caloff: Invalid macro($) in offset: %s\n", offcpy);
190
+		return CL_EMALFDB;
191
+	    }
192
+	    if (n >= 32) {
193
+		cli_errmsg("cli_caloff: at most 32 macro groups supported\n");
194
+		return CL_EMALFDB;
195
+	    }
196
+	    offdata[0] = CLI_OFF_MACRO;
197
+	    offdata[1] = n;
187 198
 	} else {
188 199
 	    offdata[0] = CLI_OFF_ABSOLUTE;
189 200
 	    if(!cli_isnumber(offcpy)) {
... ...
@@ -194,7 +205,8 @@ int cli_caloff(const char *offstr, struct cli_target_info *info, fmap_t *map, un
194 194
 	    *offset_max = *offset_min + offdata[2];
195 195
 	}
196 196
 
197
-	if(offdata[0] != CLI_OFF_ANY && offdata[0] != CLI_OFF_ABSOLUTE && offdata[0] != CLI_OFF_EOF_MINUS) {
197
+	if(offdata[0] != CLI_OFF_ANY && offdata[0] != CLI_OFF_ABSOLUTE &&
198
+	   offdata[0] != CLI_OFF_EOF_MINUS && offdata[0] != CLI_OFF_MACRO) {
198 199
 	    if(target != 1 && target != 6 && target != 9) {
199 200
 		cli_errmsg("cli_caloff: Invalid offset type for target %u\n", target);
200 201
 		return CL_EMALFDB;
... ...
@@ -472,6 +484,7 @@ int cli_fmap_scandesc(cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli
472 472
     for(i = 0; i < xroot->ac_lsigs; i++) { \
473 473
 	evalcnt = 0; \
474 474
 	evalids = 0; \
475
+	cli_ac_chkmacro(xroot, &xdata, i);\
475 476
 	if(cli_ac_chklsig(xroot->ac_lsigtable[i]->logic, xroot->ac_lsigtable[i]->logic + strlen(xroot->ac_lsigtable[i]->logic), xdata.lsigcnt[i], &evalcnt, &evalids, 0) == 1) { \
476 477
 	    if(xroot->ac_lsigtable[i]->tdb.container && xroot->ac_lsigtable[i]->tdb.container[0] != ctx->container_type) \
477 478
 		continue; \
... ...
@@ -61,6 +61,8 @@ struct cli_lsig_tdb {
61 61
 		   *secturva, *sectuvsz, *secturaw, *sectursz;
62 62
     */
63 63
     const char *icongrp1, *icongrp2;
64
+    uint32_t *macro_ptids;
65
+    uint32_t subsigs;
64 66
 #ifdef USE_MPOOL
65 67
     mpool_t *mempool;
66 68
 #endif
... ...
@@ -155,6 +157,7 @@ struct cli_target_info {
155 155
 #define CLI_OFF_SL_PLUS     5
156 156
 #define CLI_OFF_SX_PLUS     6
157 157
 #define CLI_OFF_VERSION     7
158
+#define CLI_OFF_MACRO       8
158 159
 
159 160
 int cli_scanbuff(const unsigned char *buffer, uint32_t length, uint32_t offset, cli_ctx *ctx, cli_file_t ftype, struct cli_ac_data **acdata);
160 161
 
... ...
@@ -118,6 +118,44 @@ int cli_parse_add(struct cli_matcher *root, const char *virname, const char *hex
118 118
 
119 119
 
120 120
     hexlen = strlen(hexsig);
121
+    if (hexsig[0] == '$') {
122
+	/* macro */
123
+	unsigned smin, smax, tid;
124
+	struct cli_ac_patt *pt;
125
+	if (hexsig[hexlen-1] != '$') {
126
+	    cli_errmsg("cli_parseadd(): missing terminator $\n");
127
+	    return CL_EMALFDB;
128
+	}
129
+	if (!lsigid) {
130
+	    cli_errmsg("cli_parseadd(): macro signatures only valid inside logical signatures\n");
131
+	    return CL_EMALFDB;
132
+	}
133
+	if (sscanf(hexsig,"${%u-%u}%u$",
134
+		   &smin, &smax, &tid)  != 3) {
135
+	    cli_errmsg("cli_parseadd(): invalid macro signature format\n");
136
+	    return CL_EMALFDB;
137
+	}
138
+	if (tid >= 32) {
139
+	    cli_errmsg("cli_parseadd(): only 32 macro groups are supported\n");
140
+	    return CL_EMALFDB;
141
+	}
142
+	pt = mpool_calloc(root->mempool, 1, sizeof(*pt));
143
+	if (!pt)
144
+	    return CL_EMEM;
145
+	/* this is not a pattern that will be matched by AC itself, rather it is a
146
+	 * pattern checked by the lsig code */
147
+	pt->ch_mindist[0] = smin;
148
+	pt->ch_maxdist[0] = smax;
149
+	pt->sigid = tid;
150
+	pt->length = root->ac_mindepth;
151
+	/* dummy */
152
+	pt->pattern = mpool_calloc(root->mempool, pt->length, sizeof(*pt->pattern));
153
+	if ((ret = cli_ac_addpatt(root, pt))) {
154
+	    free(pt);
155
+	    return ret;
156
+	}
157
+	return CL_SUCCESS;
158
+    }
121 159
     if(strchr(hexsig, '{')) {
122 160
 
123 161
 	root->ac_partsigs++;
... ...
@@ -239,7 +277,7 @@ int cli_parse_add(struct cli_matcher *root, const char *virname, const char *hex
239 239
 	    free(pt);
240 240
 	}
241 241
 
242
-    } else if(root->ac_only || type || lsigid || strpbrk(hexsig, "?([") || (root->bm_offmode && (!strcmp(offset, "*") || strchr(offset, ','))) || strstr(offset, "VI")) {
242
+    } else if(root->ac_only || type || lsigid || strpbrk(hexsig, "?([") || (root->bm_offmode && (!strcmp(offset, "*") || strchr(offset, ','))) || strstr(offset, "VI") || strchr(offset, '$')) {
243 243
 	if((ret = cli_ac_addsig(root, virname, hexsig, 0, 0, 0, rtype, type, 0, 0, offset, lsigid, options))) {
244 244
 	    cli_errmsg("cli_parse_add(): Problem adding signature (3).\n");
245 245
 	    return ret;
... ...
@@ -1112,6 +1150,8 @@ static int lsigattribs(char *attribs, struct cli_lsig_tdb *tdb)
1112 1112
     mpool_free(x.mempool, x.range);	\
1113 1113
   if(x.cnt[CLI_TDB_STR])		\
1114 1114
     mpool_free(x.mempool, x.str);		\
1115
+  if(x.macro_ptids)\
1116
+    mpool_free(x.mempool, x.macro_ptids);\
1115 1117
   } while(0);
1116 1118
 
1117 1119
 #define LDB_TOKENS 67
... ...
@@ -1219,7 +1259,6 @@ static int load_oneldb(char *buffer, int chkpua, int chkign, struct cl_engine *e
1219 1219
     }
1220 1220
 
1221 1221
     lsigid[0] = lsig->id = root->ac_lsigs;
1222
-    memcpy(&lsig->tdb, &tdb, sizeof(tdb));
1223 1222
 
1224 1223
     root->ac_lsigs++;
1225 1224
     newtable = (struct cli_ac_lsig **) mpool_realloc(engine->mempool, root->ac_lsigtable, root->ac_lsigs * sizeof(struct cli_ac_lsig *));
... ...
@@ -1235,6 +1274,7 @@ static int load_oneldb(char *buffer, int chkpua, int chkign, struct cl_engine *e
1235 1235
     lsig->bc_idx = bc_idx;
1236 1236
     newtable[root->ac_lsigs - 1] = lsig;
1237 1237
     root->ac_lsigtable = newtable;
1238
+    tdb.subsigs = subsigs;
1238 1239
 
1239 1240
     for(i = 0; i < subsigs; i++) {
1240 1241
 	lsigid[1] = i;
... ...
@@ -1251,6 +1291,14 @@ static int load_oneldb(char *buffer, int chkpua, int chkign, struct cl_engine *e
1251 1251
 
1252 1252
 	if((ret = cli_parse_add(root, virname, sig, 0, 0, offset, target, lsigid, options)))
1253 1253
 	    return ret;
1254
+	if(sig[0] == '$' && i) {
1255
+	    /* allow mapping from lsig back to pattern for macros */
1256
+	    if (!tdb.macro_ptids)
1257
+		tdb.macro_ptids = mpool_calloc(root->mempool, subsigs, sizeof(*tdb.macro_ptids));
1258
+	    if (!tdb.macro_ptids)
1259
+		return CL_EMEM;
1260
+	    tdb.macro_ptids[i-1] = root->ac_patterns-1;
1261
+	}
1254 1262
 
1255 1263
 	if(tdb.engine) {
1256 1264
 	    if(tdb.engine[0] > cl_retflevel()) {
... ...
@@ -1265,6 +1313,7 @@ static int load_oneldb(char *buffer, int chkpua, int chkign, struct cl_engine *e
1265 1265
 	    }
1266 1266
 	}
1267 1267
     }
1268
+    memcpy(&lsig->tdb, &tdb, sizeof(tdb));
1268 1269
     return CL_SUCCESS;
1269 1270
 }
1270 1271