@@ -103,6 +103,9 @@ CLAMAV_PRIVATE {

     cli_str2hex;

     cli_hashfile;

     cli_hashstream;

+    text_normalize_init;

+    text_normalize_reset;

+    text_normalize_map;

     html_normalise_map;

     cli_utf16toascii;

@@ -29,6 +29,7 @@

 #include <ctype.h>

 #include "clamav.h"

 #include "textnorm.h"

+#include "bignum_fast.h"

 int text_normalize_init(struct text_norm_state *state, unsigned char *out, size_t out_len)

 {

@@ -55,6 +56,7 @@ enum normalize_action {

 	NORMALIZE_ADD_32

 };

+

 /* use shorter names in the table */

 #define IGN NORMALIZE_SKIP

 #define WSP NORMALIZE_AS_WHITESPACE

@@ -121,3 +123,38 @@ size_t text_normalize_buffer(struct text_norm_state *state, const unsigned char

 	return i;

 }

+/* Normalizes the text in @fmap and stores the result in @state's buffer.

+ * Returns number of characters written to buffer. */

+size_t text_normalize_map(struct text_norm_state *state, fmap_t *map, size_t offset)

+{

+	const unsigned char *map_loc;

+	unsigned int map_pgsz;

+	uint64_t map_len;

+	size_t buff_len;

+	size_t acc;

+	size_t acc_total;

+	size_t acc_len;

+

+	map_len = map->len;

+	map_pgsz = map->pgsz;

+	buff_len = state->out_len;

+

+	acc_total = 0;

+	acc = 0;

+

+	while (1) {

+		/* Break out if we've reached the end of the map or our buffer. */

+		if(!(acc_len = MIN_3(map_pgsz, map_len - offset, buff_len - acc_total))) break;

+

+		/* If map_loc is NULL, then there's nothing left to do but recover. */

+		if(!(map_loc = fmap_need_off_once(map, offset, acc_len))) break;

+		offset += acc_len;

+

+		/* If we didn't normalize anything, no need to update values, just break out. */

+		if(!(acc = text_normalize_buffer(state, map_loc, acc_len))) break;

+		acc_total += acc;

+	}

+

+	return acc_total;

+}

+

@@ -20,6 +20,11 @@

  *  MA 02110-1301, USA.

  */

+#ifndef __TEXTNORM_H

+#define __TEXTNORM_H

+

+#include "fmap.h"

+

 struct text_norm_state {

 	unsigned char *out;

 	size_t out_len;

@@ -27,6 +32,14 @@ struct text_norm_state {

 	int space_written;

 };

+#define ASCII_FILE_BUFF_LENGTH 131072

+#define MAX_ASCII_FILE_SIZE 20000000

+

+#define MIN_3(x,y,z) ((x)<(y) ? ((x)<(z)?(x):(z)) : ((y)<(z)?(y):(z)))

+

 int text_normalize_init(struct text_norm_state *state, unsigned char *out, size_t out_len);

 void text_normalize_reset(struct text_norm_state* state);

 size_t text_normalize_buffer(struct text_norm_state *state, const unsigned char *buf, const size_t buf_len);

+size_t text_normalize_map(struct text_norm_state *state, fmap_t *map, size_t offset);

+

+#endif

@@ -110,6 +110,7 @@ const struct clam_option __clam_options[] = {

     { NULL, "mdb", 0, CLOPT_TYPE_BOOL, MATCH_BOOL, 0, NULL, 0, OPT_SIGTOOL, "", "" },

     { NULL, "print-certs", 0, CLOPT_TYPE_STRING, NULL, -1, NULL, 0, OPT_SIGTOOL, "", "" },

     { NULL, "html-normalise", 0, CLOPT_TYPE_STRING, NULL, -1, NULL, 0, OPT_SIGTOOL, "", "" },

+    { NULL, "ascii-normalise", 0, CLOPT_TYPE_STRING, NULL, -1, NULL, 0, OPT_SIGTOOL, "", "" },

     { NULL, "utf16-decode", 0, CLOPT_TYPE_STRING, NULL, -1, NULL, 0, OPT_SIGTOOL, "", "" },

     { NULL, "build", 'b', CLOPT_TYPE_STRING, NULL, -1, NULL, 0, OPT_SIGTOOL, "", "" },

     { NULL, "max-bad-sigs", 0, CLOPT_TYPE_NUMBER, MATCH_NUMBER, 3000, NULL, 0, OPT_SIGTOOL, "Maximum number of mismatched signatures when building a CVD. Zero disables this limit.", "3000" },

@@ -43,6 +43,8 @@

 #include <netinet/in.h>

 #include <arpa/inet.h>

 #include <sys/wait.h>

+#else

+#include "w32_stat.h"

 #endif

 #include <dirent.h>

 #include <ctype.h>

@@ -66,6 +68,7 @@

 #include "libclamav/str.h"

 #include "libclamav/ole2_extract.h"

 #include "libclamav/htmlnorm.h"

+#include "libclamav/textnorm.h"

 #include "libclamav/default.h"

 #include "libclamav/fmap.h"

 #include "libclamav/readdb.h"

@@ -223,6 +226,79 @@ static int htmlnorm(const struct optstruct *opts)

     return 0;

 }

+static int asciinorm(const struct optstruct *opts)

+{

+    const char *fname;

+    unsigned char *norm_buff;

+    struct text_norm_state state;

+    size_t map_off;

+    fmap_t *map; 

+    int fd, ofd;

+

+    fname = optget(opts, "ascii-normalise")->strarg;

+    fd = open(fname, O_RDONLY);

+

+    if (fd == -1) {

+	mprintf("!asciinorm: Can't open file %s\n", fname);

+	return -1;

+    }

+

+    if(!(norm_buff = malloc(ASCII_FILE_BUFF_LENGTH))) {

+	mprintf("!asciinorm: Can't allocate memory\n");

+	close(fd);

+	return -1;

+    }

+

+    if (!(map = fmap(fd, 0, 0))) {

+	mprintf("!fmap: Could not map fd %d\n", fd);

+	close(fd);

+	free(norm_buff);

+	return -1;

+    }

+

+    if (map->len > MAX_ASCII_FILE_SIZE) {

+	mprintf("!asciinorm: File size of %zu too large\n", map->len);

+	close(fd);

+	free(norm_buff);

+	funmap(map);

+	return -1;

+    }

+

+    ofd = open("./normalised_text", O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);

+    if (ofd == -1) {

+	mprintf("!asciinorm: Can't open file ./normalised_text\n");

+	close(fd);

+	free(norm_buff);

+	funmap(map);

+	return -1;

+    }

+

+    text_normalize_init(&state, norm_buff, ASCII_FILE_BUFF_LENGTH);

+

+    map_off = 0;

+    while(map_off != map->len) {

+	    size_t written;

+	    if (!(written = text_normalize_map(&state, map, map_off))) break;

+	    map_off += written;

+ 

+	    if (write(ofd, norm_buff, state.out_pos) == -1) {

+		    mprintf("!asciinorm: Can't write to file ./normalised_text\n");

+		    close(fd);

+		    close(ofd);

+		    free(norm_buff);

+		    funmap(map);

+		    return -1;

+	    }

+	    text_normalize_reset(&state);

+    }

+

+    close(fd);

+    close(ofd);

+    free(norm_buff);

+    funmap(map);

+    return 0;

+}

+

 static int utf16decode(const struct optstruct *opts)

 {

 	const char *fname;

@@ -3012,6 +3088,7 @@ static void help(void)

     mprintf("                                           or SHA256 sigs for FILES\n");

     mprintf("    --mdb [FILES]                          generate .mdb sigs\n");

     mprintf("    --html-normalise=FILE                  create normalised parts of HTML file\n");

+    mprintf("    --ascii-normalise=FILE                 create normalised text file from ascii source\n");

     mprintf("    --utf16-decode=FILE                    decode UTF16 encoded files\n");

     mprintf("    --info=FILE            -i FILE         print database information\n");

     mprintf("    --build=NAME [cvd] -b NAME             build a CVD file\n");

@@ -3106,6 +3183,8 @@ int main(int argc, char **argv)

 	ret = hashsig(opts, 1, 1);

     else if(optget(opts, "html-normalise")->enabled)

 	ret = htmlnorm(opts);

+    else if(optget(opts, "ascii-normalise")->enabled)

+	ret = asciinorm(opts);

     else if(optget(opts, "utf16-decode")->enabled)

 	ret = utf16decode(opts);

     else if(optget(opts, "build")->enabled)

@@ -170,6 +170,9 @@ EXPORTS cli_pcre_freeoff @44373 NONAME

 EXPORTS cli_pcre_recaloff @44374 NONAME

 EXPORTS cli_pcre_perf_events_destroy @44375 NONAME

 EXPORTS cli_pcre_perf_print @44376 NONAME

+EXPORTS text_normalize_init @44377 NONAME

+EXPORTS text_normalize_reset @44378 NONAME

+EXPORTS text_normalize_map @44379 NONAME

 ; compatibility layer, tommath, zlib

 EXPORTS w32_srand @44269 NONAME