new file mode 100644

@@ -0,0 +1,33 @@

+libclamav/textdet.c includes modified code from file-4.23/src/ascmagic.c.

+The original LEGAL.NOTICE file for file-4.23 is reproduced below.

+

+--------------------------------------------------------------------------

+$File: LEGAL.NOTICE,v 1.15 2006/05/03 18:48:33 christos Exp $

+Copyright (c) Ian F. Darwin 1986, 1987, 1989, 1990, 1991, 1992, 1994, 1995.

+Software written by Ian F. Darwin and others;

+maintained 1994- Christos Zoulas.

+

+This software is not subject to any export provision of the United States

+Department of Commerce, and may be exported to any country or planet.

+

+Redistribution and use in source and binary forms, with or without

+modification, are permitted provided that the following conditions

+are met:

+1. Redistributions of source code must retain the above copyright

+   notice immediately at the beginning of the file, without modification,

+   this list of conditions, and the following disclaimer.

+2. Redistributions in binary form must reproduce the above copyright

+   notice, this list of conditions and the following disclaimer in the

+   documentation and/or other materials provided with the distribution.

+ 

+THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND

+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

+ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR

+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS

+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)

+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY

+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF

+SUCH DAMAGE.

@@ -1,3 +1,8 @@

+Mon Jan  7 14:50:24 CET 2008 (tk)

+---------------------------------

+  * libclamav/textdet.c: text detection code based on file-4.23

+  * libclamav/filetypes.c: re-enable text detection (ASCII, UTF8, UTF16)

+

 Sun Jan  6 19:35:28 EET 2008 (edwin)

 ------------------------------------

  * build system: improve iconv() detection, by actually trying to link a

@@ -17,7 +17,7 @@

 #  MA 02110-1301, USA.

 SUBDIRS = libclamunrar libclamunrar_iface libclamav clamscan clamd clamdscan freshclam sigtool clamconf database docs etc clamav-milter

-EXTRA_DIST = FAQ contrib test examples BUGS shared libclamav.pc.in UPGRADE COPYING.bzip2 COPYING.lzma COPYING.unrar COPYING.LGPL

+EXTRA_DIST = FAQ contrib test examples BUGS shared libclamav.pc.in UPGRADE COPYING.bzip2 COPYING.lzma COPYING.unrar COPYING.LGPL COPYING.file

 bin_SCRIPTS=clamav-config

@@ -236,7 +236,7 @@ target_vendor = @target_vendor@

 top_builddir = @top_builddir@

 top_srcdir = @top_srcdir@

 SUBDIRS = libclamunrar libclamunrar_iface libclamav clamscan clamd clamdscan freshclam sigtool clamconf database docs etc clamav-milter

-EXTRA_DIST = FAQ contrib test examples BUGS shared libclamav.pc.in UPGRADE COPYING.bzip2 COPYING.lzma COPYING.unrar COPYING.LGPL

+EXTRA_DIST = FAQ contrib test examples BUGS shared libclamav.pc.in UPGRADE COPYING.bzip2 COPYING.lzma COPYING.unrar COPYING.LGPL COPYING.file

 bin_SCRIPTS = clamav-config

 pkgconfigdir = $(libdir)/pkgconfig

 pkgconfig_DATA = libclamav.pc

@@ -56,6 +56,8 @@ libclamav_la_SOURCES = \

 	str.h \

 	scanners.c \

 	scanners.h \

+	textdet.c \

+	textdet.h \

 	filetypes.c \

 	filetypes.h \

 	filetypes_int.h \

@@ -78,17 +78,17 @@ LTLIBRARIES = $(lib_LTLIBRARIES) $(noinst_LTLIBRARIES)

 @ENABLE_UNRAR_TRUE@	$(top_builddir)/libclamunrar_iface/libclamunrar_iface.la

 am_libclamav_la_OBJECTS = matcher-ac.lo matcher-bm.lo matcher.lo \

 	md5.lo others.lo readdb.lo cvd.lo dsig.lo str.lo scanners.lo \

-	filetypes.lo rtf.lo blob.lo mbox.lo message.lo snprintf.lo \

-	table.lo text.lo ole2_extract.lo vba_extract.lo msexpand.lo \

-	pe.lo upx.lo htmlnorm.lo chmunpack.lo rebuildpe.lo petite.lo \

-	wwunpack.lo unsp.lo aspack.lo packlibs.lo fsg.lo mew.lo \

-	upack.lo line.lo untar.lo unzip.lo inflate64.lo special.lo \

-	binhex.lo is_tar.lo tnef.lo autoit.lo strlcpy.lo regcomp.lo \

-	regerror.lo regexec.lo regfree.lo unarj.lo bzlib.lo nulsft.lo \

-	pdf.lo spin.lo yc.lo elf.lo sis.lo uuencode.lo pst.lo \

-	phishcheck.lo phish_domaincheck_db.lo phish_whitelist.lo \

-	regex_list.lo mspack.lo cab.lo entconv.lo hashtab.lo dconf.lo \

-	lzma_iface.lo explode.lo

+	textdet.lo filetypes.lo rtf.lo blob.lo mbox.lo message.lo \

+	snprintf.lo table.lo text.lo ole2_extract.lo vba_extract.lo \

+	msexpand.lo pe.lo upx.lo htmlnorm.lo chmunpack.lo rebuildpe.lo \

+	petite.lo wwunpack.lo unsp.lo aspack.lo packlibs.lo fsg.lo \

+	mew.lo upack.lo line.lo untar.lo unzip.lo inflate64.lo \

+	special.lo binhex.lo is_tar.lo tnef.lo autoit.lo strlcpy.lo \

+	regcomp.lo regerror.lo regexec.lo regfree.lo unarj.lo bzlib.lo \

+	nulsft.lo pdf.lo spin.lo yc.lo elf.lo sis.lo uuencode.lo \

+	pst.lo phishcheck.lo phish_domaincheck_db.lo \

+	phish_whitelist.lo regex_list.lo mspack.lo cab.lo entconv.lo \

+	hashtab.lo dconf.lo lzma_iface.lo explode.lo

 libclamav_la_OBJECTS = $(am_libclamav_la_OBJECTS)

 libclamav_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) \

 	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \

@@ -278,6 +278,8 @@ libclamav_la_SOURCES = \

 	str.h \

 	scanners.c \

 	scanners.h \

+	textdet.c \

+	textdet.h \

 	filetypes.c \

 	filetypes.h \

 	filetypes_int.h \

@@ -555,6 +557,7 @@ distclean-compile:

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/strlcpy.Plo@am__quote@

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/table.Plo@am__quote@

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/text.Plo@am__quote@

+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/textdet.Plo@am__quote@

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tnef.Plo@am__quote@

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unarj.Plo@am__quote@

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unsp.Plo@am__quote@

@@ -628,7 +628,7 @@ fileblobAddData(fileblob *fb, const unsigned char *data, size_t len)

 					*ctx->scanned += (unsigned long)len / CL_COUNT_PRECISION;

 				fb->bytes_scanned += (unsigned long)len;

-				if((len > 5) && (cli_scanbuff(data, (unsigned int)len, ctx->virname, ctx->engine, CL_TYPE_UNKNOWN_DATA) == CL_VIRUS)) {

+				if((len > 5) && (cli_scanbuff(data, (unsigned int)len, ctx->virname, ctx->engine, CL_TYPE_BINARY_DATA) == CL_VIRUS)) {

 					cli_dbgmsg("fileblobAddData: found %s\n", *ctx->virname);

 					fb->isInfected = 1;

 				}

@@ -38,6 +38,7 @@

 #include "readdb.h"

 #include "matcher-ac.h"

 #include "str.h"

+#include "textdet.h"

 #include "htmlnorm.h"

 #include "entconv.h"

@@ -46,8 +47,11 @@ static const struct ftmap_s {

     const char *name;

     cli_file_t code;

 } ftmap[] = {

-    { "CL_TYPE_UNKNOWN_TEXT",	CL_TYPE_UNKNOWN_TEXT	},

-    { "CL_TYPE_UNKNOWN_DATA",	CL_TYPE_UNKNOWN_DATA	},

+    { "CL_TYPE_TEXT_ASCII",	CL_TYPE_TEXT_ASCII	},

+    { "CL_TYPE_TEXT_UTF8",	CL_TYPE_TEXT_UTF8	},

+    { "CL_TYPE_TEXT_UTF16LE",	CL_TYPE_TEXT_UTF16LE	},

+    { "CL_TYPE_TEXT_UTF16BE",	CL_TYPE_TEXT_UTF16BE	},

+    { "CL_TYPE_BINARY_DATA",	CL_TYPE_BINARY_DATA	},

     { "CL_TYPE_IGNORED",	CL_TYPE_IGNORED		},

     { "CL_TYPE_MSEXE",		CL_TYPE_MSEXE		},

     { "CL_TYPE_ELF",		CL_TYPE_ELF		},

@@ -83,7 +87,7 @@ static const struct ftmap_s {

     { "CL_TYPE_ARJSFX",		CL_TYPE_ARJSFX		},

     { "CL_TYPE_NULSFT",		CL_TYPE_NULSFT		},

     { "CL_TYPE_AUTOIT",		CL_TYPE_AUTOIT		},

-    { NULL,			CL_TYPE_UNKNOWN_DATA	}

+    { NULL,			CL_TYPE_IGNORED		}

 };

 cli_file_t cli_ftcode(const char *name)

@@ -125,19 +129,7 @@ cli_file_t cli_filetype(const unsigned char *buf, size_t buflen, const struct cl

 	ftype = ftype->next;

     }

-/* FIXME: improve or drop this code

- * https://wwws.clamav.net/bugzilla/show_bug.cgi?id=373

- *

-	int i, text = 1, len;

-    buflen < 25 ? (len = buflen) : (len = 25);

-    for(i = 0; i < len; i++)

-	if(!iscntrl(buf[i]) && !isprint(buf[i]) && !internat[buf[i] & 0xff]) {

-	    text = 0;

-	    break;

-	}

-    return text ? CL_TYPE_UNKNOWN_TEXT : CL_TYPE_UNKNOWN_DATA;

-*/

-    return CL_TYPE_UNKNOWN_TEXT;

+    return cli_texttype(buf, buflen);

 }

 int is_tar(unsigned char *buf, unsigned int nbytes);

@@ -146,16 +138,24 @@ cli_file_t cli_filetype2(int desc, const struct cl_engine *engine)

 {

 	unsigned char smallbuff[MAGIC_BUFFER_SIZE + 1], *decoded, *bigbuff;

 	int bread, sret;

-	cli_file_t ret = CL_TYPE_UNKNOWN_DATA;

+	cli_file_t ret = CL_TYPE_BINARY_DATA;

 	struct cli_matcher *root;

 	struct cli_ac_data mdata;

+    if(!engine) {

+	cli_errmsg("cli_filetype2: engine == NULL\n");

+	return CL_TYPE_ERROR;

+    }

+

     memset(smallbuff, 0, sizeof(smallbuff));

     if((bread = read(desc, smallbuff, MAGIC_BUFFER_SIZE)) > 0)

 	ret = cli_filetype(smallbuff, bread, engine);

-    if(engine && ret == CL_TYPE_UNKNOWN_TEXT) {

+    if(ret >= CL_TYPE_TEXT_ASCII && ret <= CL_TYPE_BINARY_DATA) {

+	/* HTML files may contain special characters and could be

+	 * misidentified as BINARY_DATA by cli_filetype()

+	 */

 	root = engine->root[0];

 	if(!root)

 	    return ret;

@@ -221,7 +221,7 @@ cli_file_t cli_filetype2(int desc, const struct cl_engine *engine)

 	}

     }

-    if(ret == CL_TYPE_UNKNOWN_DATA || ret == CL_TYPE_UNKNOWN_TEXT) {

+    if(ret == CL_TYPE_BINARY_DATA) {

 	if(!(bigbuff = (unsigned char *) cli_calloc(37638 + 1, sizeof(unsigned char))))

 	    return ret;

@@ -243,8 +243,7 @@ cli_file_t cli_filetype2(int desc, const struct cl_engine *engine)

 	    }

 	}

-	if(ret == CL_TYPE_UNKNOWN_DATA || ret == CL_TYPE_UNKNOWN_TEXT) {

-

+	if(ret == CL_TYPE_BINARY_DATA) {

 	    if(!memcmp(bigbuff + 32769, "CD001" , 5) || !memcmp(bigbuff + 37633, "CD001" , 5)) {

 		cli_dbgmsg("Recognized ISO 9660 CD-ROM data\n");

 		ret = CL_TYPE_IGNORED;

@@ -33,8 +33,12 @@

 #define MAX_EMBEDDED_OBJ 10

 typedef enum {

-    CL_TYPE_UNKNOWN_TEXT = CL_TYPENO,

-    CL_TYPE_UNKNOWN_DATA,

+    CL_TYPE_TEXT_ASCII = CL_TYPENO, /* X3.4, ISO-8859, non-ISO ext. ASCII */

+    CL_TYPE_TEXT_UTF8,

+    CL_TYPE_TEXT_UTF16LE,

+    CL_TYPE_TEXT_UTF16BE,

+    CL_TYPE_BINARY_DATA,

+    /* Please do not add any new types above this line */

     CL_TYPE_IGNORED,

     CL_TYPE_ERROR,

     CL_TYPE_MSEXE,

@@ -609,7 +609,7 @@ cli_mbox(const char *dir, int desc, cli_ctx *ctx)

 		type = cli_filetype(start, size, ctx->engine);

-		if((type == CL_TYPE_UNKNOWN_TEXT) &&

+		if((type == CL_TYPE_TEXT_ASCII) &&

 		   (strncmp(start, "Microsoft Mail Internet Headers", 31) == 0))

 			type = CL_TYPE_MAIL;

@@ -1038,7 +1038,7 @@ save_text(cli_ctx *ctx, const char *dir, const char *start, size_t len)

 		 *	in this way. It gets the "filetype" wrong and then

 		 *	doesn't scan correctly

 		 */

-		if(cli_scanbuff((char *)p, len, ctx->virname, ctx->engine, CL_TYPE_UNKNOWN_DATA) == CL_VIRUS) {

+		if(cli_scanbuff((char *)p, len, ctx->virname, ctx->engine, CL_TYPE_BINARY_DATA) == CL_VIRUS) {

 			cli_dbgmsg("save_text: found %s\n", *ctx->virname);

 			return CL_VIRUS;

 		}

@@ -1602,7 +1602,7 @@ static int cli_scanraw(int desc, cli_ctx *ctx, cli_file_t type, uint8_t typercg)

     if(typercg) switch(type) {

-	case CL_TYPE_UNKNOWN_TEXT:

+	case CL_TYPE_TEXT_ASCII:

 	case CL_TYPE_MSEXE:

 	case CL_TYPE_ZIP:

 	    ftrec = 1;

@@ -1620,7 +1620,7 @@ static int cli_scanraw(int desc, cli_ctx *ctx, cli_file_t type, uint8_t typercg)

     if(ret >= CL_TYPENO) {

-	if(type == CL_TYPE_UNKNOWN_TEXT) {

+	if(type == CL_TYPE_TEXT_ASCII) {

 	    lseek(desc, 0, SEEK_SET);

 	    nret = cli_scandesc(desc, ctx, 0, ret, 1, NULL);

@@ -1713,12 +1713,12 @@ static int cli_scanraw(int desc, cli_ctx *ctx, cli_file_t type, uint8_t typercg)

 	if(nret != CL_VIRUS) switch(ret) {

 	    case CL_TYPE_HTML:

-		if(SCAN_HTML && type == CL_TYPE_UNKNOWN_TEXT && (DCONF_DOC & DOC_CONF_HTML))

+		if(SCAN_HTML && type == CL_TYPE_TEXT_ASCII && (DCONF_DOC & DOC_CONF_HTML))

 		    nret = cli_scanhtml(desc, ctx);

 		break;

 	    case CL_TYPE_MAIL:

-		if(SCAN_MAIL && type == CL_TYPE_UNKNOWN_TEXT && (DCONF_MAIL & MAIL_CONF_MBOX))

+		if(SCAN_MAIL && type == CL_TYPE_TEXT_ASCII && (DCONF_MAIL & MAIL_CONF_MBOX))

 		    nret = cli_scanmail(desc, ctx);

 		break;

@@ -1949,7 +1949,7 @@ int cli_magic_scandesc(int desc, cli_ctx *ctx)

 		ret = cli_scansis(desc, ctx);

 	    break;

-	case CL_TYPE_UNKNOWN_DATA:

+	case CL_TYPE_BINARY_DATA:

 	    ret = cli_check_mydoom_log(desc, ctx->virname);

 	    break;

new file mode 100644

@@ -0,0 +1,189 @@

+/*

+ * Text detection based on ascmagic.c from the file(1) utility.

+ * Portions Copyright (C) 2008 Sourcefire, Inc.

+ * Maintained by Tomasz Kojm <tkojm@clamav.net>

+ *

+ * Copyright (c) Ian F. Darwin 1986-1995.

+ * Software written by Ian F. Darwin and others;

+ * maintained 1995-present by Christos Zoulas and others.

+ * 

+ * Redistribution and use in source and binary forms, with or without

+ * modification, are permitted provided that the following conditions

+ * are met:

+ * 1. Redistributions of source code must retain the above copyright

+ *    notice immediately at the beginning of the file, without modification,

+ *    this list of conditions, and the following disclaimer.

+ * 2. Redistributions in binary form must reproduce the above copyright

+ *    notice, this list of conditions and the following disclaimer in the

+ *    documentation and/or other materials provided with the distribution.

+ *  

+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND

+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR

+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS

+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)

+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY

+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF

+ * SUCH DAMAGE.

+ */

+

+#if HAVE_CONFIG_H

+#include "clamav-config.h"

+#endif

+

+#include <stdio.h>

+#include <string.h>

+#include <memory.h>

+#include <ctype.h>

+#include <stdlib.h>

+#ifdef HAVE_UNISTD_H

+#include <unistd.h>

+#endif

+

+#include "filetypes.h"

+#include "textdet.h"

+

+#define F 0   /* character never appears in text */

+#define T 1   /* character appears in plain ASCII text */

+#define I 2   /* character appears in ISO-8859 text */

+#define X 3   /* character appears in non-ISO extended ASCII (Mac, IBM PC) */

+

+static char text_chars[256] = {

+	/*                  BEL BS HT LF    FF CR    */

+	F, F, F, F, F, F, F, T, T, T, T, F, T, T, F, F,  /* 0x0X */

+        /*                              ESC          */

+	F, F, F, F, F, F, F, F, F, F, F, T, F, F, F, F,  /* 0x1X */

+	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x2X */

+	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x3X */

+	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x4X */

+	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x5X */

+	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x6X */

+	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, F,  /* 0x7X */

+	/*            NEL                            */

+	X, X, X, X, X, T, X, X, X, X, X, X, X, X, X, X,  /* 0x8X */

+	X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,  /* 0x9X */

+	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xaX */

+	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xbX */

+	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xcX */

+	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xdX */

+	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xeX */

+	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I   /* 0xfX */

+};

+

+static int td_isascii(const unsigned char *buf, unsigned int len)

+{

+	unsigned int i;

+

+    for(i = 0; i < len; i++)

+	if(text_chars[buf[i]] == F)

+	    return 0;

+

+    return 1;

+}

+

+static int td_isutf8(const unsigned char *buf, unsigned int len)

+{

+	unsigned int i, j, gotone = 0;

+

+

+    for(i = 0; i < len; i++) {

+	if((buf[i] & 0x80) == 0) {  /* 0xxxxxxx is plain ASCII */

+	    /*

+	     * Even if the whole file is valid UTF-8 sequences,

+	     * still reject it if it uses weird control characters.

+	     */

+	    if(text_chars[buf[i]] != T)

+		return 0;

+

+	} else if((buf[i] & 0x40) == 0) { /* 10xxxxxx never 1st byte */

+	    return 0;

+	} else {			   /* 11xxxxxx begins UTF-8 */

+		unsigned int following;

+

+	    if((buf[i] & 0x20) == 0) {		/* 110xxxxx */

+		/* c = buf[i] & 0x1f; */

+		following = 1;

+	    } else if((buf[i] & 0x10) == 0) {	/* 1110xxxx */

+		/* c = buf[i] & 0x0f; */

+		following = 2;

+	    } else if((buf[i] & 0x08) == 0) {	/* 11110xxx */

+		/* c = buf[i] & 0x07; */

+		following = 3;

+	    } else if((buf[i] & 0x04) == 0) {	/* 111110xx */

+		/* c = buf[i] & 0x03; */

+		following = 4;

+	    } else if((buf[i] & 0x02) == 0) {	/* 1111110x */

+		/* c = buf[i] & 0x01; */

+		following = 5;

+	    } else {

+		return 0;

+	    }

+

+	    for(j = 0; j < following; j++) {

+		if(++i >= len)

+		    return gotone;

+

+		if((buf[i] & 0x80) == 0 || (buf[i] & 0x40))

+		    return 0;

+

+		/* c = (c << 6) + (buf[i] & 0x3f); */

+	    }

+

+	    gotone = 1;

+	}

+    }

+

+    return gotone;

+}

+

+static int td_isutf16(const unsigned char *buf, unsigned int len)

+{

+	unsigned int be, i, c;

+

+

+    if(len < 2)

+	return 0;

+

+    if(buf[0] == 0xff && buf[1] == 0xfe)

+	be = 0;

+    else if(buf[0] == 0xfe && buf[1] == 0xff)

+	be = 1;

+    else

+	return 0;

+

+    for(i = 2; i + 1 < len; i += 2) {

+	if(be)

+	    c = buf[i + 1] + 256 * buf[i];

+	else

+	    c = buf[i] + 256 * buf[i + 1];

+

+	if(c == 0xfffe)

+	    return 0;

+

+	if(c < 128 && text_chars[c] != T)

+	    return 0;

+    }

+

+    return 1 + be;

+}

+

+cli_file_t cli_texttype(const unsigned char *buf, unsigned int len)

+{

+	int ret;

+

+    if(td_isutf8(buf, len)) {

+	return CL_TYPE_TEXT_UTF8;

+    } else if((ret = td_isutf16(buf, len))) {

+	if(ret == 1)

+	    return CL_TYPE_TEXT_UTF16LE;

+	else

+	    return CL_TYPE_TEXT_UTF16BE;

+    } else if(td_isascii(buf, len)) {

+	return CL_TYPE_TEXT_ASCII;

+    } else {

+	return CL_TYPE_BINARY_DATA;

+    }

+}

new file mode 100644

@@ -0,0 +1,26 @@

+/*

+ *  Copyright (C) 2008 Sourcefire, Inc.

+ *

+ *  This program is free software; you can redistribute it and/or modify

+ *  it under the terms of the GNU General Public License version 2 as

+ *  published by the Free Software Foundation.

+ *

+ *  This program is distributed in the hope that it will be useful,

+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of

+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ *  GNU General Public License for more details.

+ *

+ *  You should have received a copy of the GNU General Public License

+ *  along with this program; if not, write to the Free Software

+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

+ *  MA 02110-1301, USA.

+ */

+

+#ifndef __TEXTDET_H

+#define __TEXTDET_H

+

+#include "filetypes.h"

+

+cli_file_t cli_texttype(const unsigned char *buf, unsigned int len);

+

+#endif