@@ -1,3 +1,9 @@

+Fri Dec 14 22:55:32 CET 2007 (tk)

+---------------------------------

+  * libclamav: improved filetype detection code; filetype definitions can now

+	       be distributed inside daily.cvd

+  * sigtool/sigtool.c: handle daily.ft

+

 Fri Dec 14 13:02:38 EET 2007 (edwin)

 ------------------------------------

   * libclamav/phishcheck.[ch]:

@@ -54,6 +54,7 @@ libclamav_la_SOURCES = \

 	scanners.h \

 	filetypes.c \

 	filetypes.h \

+	filetypes_int.h \

 	rtf.c \

 	rtf.h \

 	blob.c \

@@ -275,6 +275,7 @@ libclamav_la_SOURCES = \

 	scanners.h \

 	filetypes.c \

 	filetypes.h \

+	filetypes_int.h \

 	rtf.c \

 	rtf.h \

 	blob.c \

@@ -129,6 +129,9 @@ struct cl_engine {

     /* Dynamic configuration */

     void *dconf;

+

+    /* Filetype definitions */

+    void *ftypes;

 };

 struct cl_limits {

@@ -1,4 +1,7 @@

 /*

+ *  Copyright (C) 2007 Sourcefire, Inc.

+ *  Author: Tomasz Kojm <tkojm@clamav.net>

+ *

  *  Copyright (C) 2002 - 2005 Tomasz Kojm <tkojm@clamav.net>

  *  With enhancements from Thomas Lamy <Thomas.Lamy@in-online.net>

  *

@@ -24,7 +27,6 @@

 #include <stdio.h>

 #include <string.h>

 #include <stdlib.h>

-#include <ctype.h>

 #include <sys/types.h>

 #ifdef	HAVE_UNISTD_H

 #include <unistd.h>

@@ -40,214 +42,102 @@

 #include "htmlnorm.h"

 #include "entconv.h"

-struct cli_magic_s {

-    size_t offset;

-    const char *magic;

-    size_t length;

-    const char *descr;

-    cli_file_t type;

+static const struct ftmap_s {

+    const char *name;

+    cli_file_t code;

+} ftmap[] = {

+    { "CL_TYPE_UNKNOWN_TEXT",	CL_TYPE_UNKNOWN_TEXT	},

+    { "CL_TYPE_UNKNOWN_DATA",	CL_TYPE_UNKNOWN_DATA	},

+    { "CL_TYPE_IGNORED",	CL_TYPE_IGNORED		},

+    { "CL_TYPE_MSEXE",		CL_TYPE_MSEXE		},

+    { "CL_TYPE_ELF",		CL_TYPE_ELF		},

+    { "CL_TYPE_POSIX_TAR",	CL_TYPE_POSIX_TAR	},

+    { "CL_TYPE_OLD_TAR",	CL_TYPE_OLD_TAR		},

+    { "CL_TYPE_GZ",		CL_TYPE_GZ		},

+    { "CL_TYPE_ZIP",		CL_TYPE_ZIP		},

+    { "CL_TYPE_BZ",		CL_TYPE_BZ		},

+    { "CL_TYPE_RAR",		CL_TYPE_RAR		},

+    { "CL_TYPE_ARJ",		CL_TYPE_ARJ		},

+    { "CL_TYPE_MSSZDD",		CL_TYPE_MSSZDD		},

+    { "CL_TYPE_MSOLE2",		CL_TYPE_MSOLE2		},

+    { "CL_TYPE_MSCAB",		CL_TYPE_MSCAB		},

+    { "CL_TYPE_MSCHM",		CL_TYPE_MSCHM		},

+    { "CL_TYPE_SIS",		CL_TYPE_SIS		},

+    { "CL_TYPE_SCRENC",		CL_TYPE_SCRENC		},

+    { "CL_TYPE_GRAPHICS",	CL_TYPE_GRAPHICS	},

+    { "CL_TYPE_RIFF",		CL_TYPE_RIFF		},

+    { "CL_TYPE_BINHEX",		CL_TYPE_BINHEX		},

+    { "CL_TYPE_TNEF",		CL_TYPE_TNEF		},

+    { "CL_TYPE_CRYPTFF",	CL_TYPE_CRYPTFF		},

+    { "CL_TYPE_PDF",		CL_TYPE_PDF		},

+    { "CL_TYPE_UUENCODED",	CL_TYPE_UUENCODED	},

+    { "CL_TYPE_PST",		CL_TYPE_PST		},

+    { "CL_TYPE_HTML_UTF16",	CL_TYPE_HTML_UTF16	},

+    { "CL_TYPE_RTF",		CL_TYPE_RTF		},

+    { "CL_TYPE_HTML",		CL_TYPE_HTML		},

+    { "CL_TYPE_MAIL",		CL_TYPE_MAIL		},

+    { "CL_TYPE_SFX",		CL_TYPE_SFX		},

+    { "CL_TYPE_ZIPSFX",		CL_TYPE_ZIPSFX		},

+    { "CL_TYPE_RARSFX",		CL_TYPE_RARSFX		},

+    { "CL_TYPE_CABSFX",		CL_TYPE_CABSFX		},

+    { "CL_TYPE_ARJSFX",		CL_TYPE_ARJSFX		},

+    { "CL_TYPE_NULSFT",		CL_TYPE_NULSFT		},

+    { "CL_TYPE_AUTOIT",		CL_TYPE_AUTOIT		},

+    { NULL,			CL_TYPE_UNKNOWN_DATA	}

 };

-struct cli_smagic_s {

-    const char *sig;

-    const char *descr;

-    cli_file_t type;

-};

+cli_file_t cli_ftcode(const char *name)

+{

+	unsigned int i;

-static const struct cli_magic_s cli_magic[] = {

-

-    /* Executables */

-

-    {0,  "MZ",				2,  "DOS/W32 executable/library/driver", CL_TYPE_MSEXE},

-    {0,	 "\177ELF",			4,  "ELF",		CL_TYPE_ELF},

-

-    /* Archives */

-

-    {0,	    "Rar!",			4,  "RAR",		CL_TYPE_RAR},

-    {0,	    "PK\003\004",		4,  "ZIP",		CL_TYPE_ZIP},

-    {0,	    "PK00PK\003\004",		8,  "ZIP",		CL_TYPE_ZIP},

-    {0,	    "\037\213",			2,  "GZip",		CL_TYPE_GZ},

-    {0,	    "BZh",			3,  "BZip",		CL_TYPE_BZ},

-    {0,	    "\x60\xea",			2,  "ARJ",		CL_TYPE_ARJ},

-    {0,	    "SZDD",			4,  "compress.exe'd",	CL_TYPE_MSSZDD},

-    {0,	    "MSCF",			4,  "MS CAB",		CL_TYPE_MSCAB},

-    {0,	    "ITSF",			4,  "MS CHM",           CL_TYPE_MSCHM},

-    {8,	    "\x19\x04\x00\x10",		4,  "SIS",		CL_TYPE_SIS},

-    {0,	    "\x7a\x1a\x20\x10",		4,  "SIS",		CL_TYPE_SIS},

-    {0,     "#@~^",			4,  "SCRENC",		CL_TYPE_SCRENC},

-    {0,     "(This file must be converted with BinHex 4.0)",

-				       45, "BinHex",		CL_TYPE_BINHEX},

-

-    /* Mail */

-

-    {0,  "From ",			 5, "MBox",		  CL_TYPE_MAIL},

-    {0,  "Received: ",			10, "Raw mail",		  CL_TYPE_MAIL},

-    {0,  "Return-Path: ",		13, "Maildir",		  CL_TYPE_MAIL},

-    {0,  "Return-path: ",		13, "Maildir",		  CL_TYPE_MAIL},

-    {0,  "Delivered-To: ",		14, "Mail",		  CL_TYPE_MAIL},

-    {0,  "X-UIDL: ",			 8, "Mail",		  CL_TYPE_MAIL},

-    {0,  "X-Apparently-To: ",		17, "Mail",		  CL_TYPE_MAIL},

-    {0,  "X-Envelope-From: ",		17, "Mail",		  CL_TYPE_MAIL},

-    {0,  "X-Original-To: ",		15, "Mail",		  CL_TYPE_MAIL},

-    {0,  "X-Symantec-",			11, "Symantec",		  CL_TYPE_MAIL},

-    {0,  "X-EVS",			 5, "EVS mail",		  CL_TYPE_MAIL},

-    {0,  "X-Real-To: ",                 11, "Mail",               CL_TYPE_MAIL},

-    {0,  "X-Sieve: ",			 9, "Mail",		  CL_TYPE_MAIL},

-    {0,  ">From ",			 6, "Mail",		  CL_TYPE_MAIL},

-    {0,  "Date: ",			 6, "Mail",		  CL_TYPE_MAIL},

-    {0,  "Message-Id: ",		12, "Mail",		  CL_TYPE_MAIL},

-    {0,  "Message-ID: ",		12, "Mail",		  CL_TYPE_MAIL},

-    {0,  "Envelope-to: ",		13, "Mail",		  CL_TYPE_MAIL},

-    {0,  "Delivery-date: ",		15, "Mail",		  CL_TYPE_MAIL},

-    {0,  "To: ",			 4, "Mail",		  CL_TYPE_MAIL},

-    {0,  "Subject: ",			 9, "Mail",		  CL_TYPE_MAIL},

-    {0,  "For: ",			 5, "Eserv mail",	  CL_TYPE_MAIL},

-    {0,  "From: ",			 6, "Exim mail",	  CL_TYPE_MAIL},

-    {0,  "v:\015\012Received: ",	14, "VPOP3 Mail (DOS)",	  CL_TYPE_MAIL},

-    {0,  "v:\012Received: ",		13, "VPOP3 Mail (UNIX)",  CL_TYPE_MAIL},

-    {0,  "Hi. This is the qmail-send",  26, "Qmail bounce",	  CL_TYPE_MAIL},

-    {0,  "\170\237\076\042",		 4, "TNEF",               CL_TYPE_TNEF},

-

-    {0,  "begin ",			6,  "UUencoded",	  CL_TYPE_UUENCODED},

-    {0, "\041\102\104\116",		4, "PST",		  CL_TYPE_PST},

-

-    /* Graphics (may contain exploits against MS systems) */

-

-    {0,  "GIF",				 3, "GIF",	    CL_TYPE_GRAPHICS},

-    {0,  "BM",				 2, "BMP",          CL_TYPE_GRAPHICS},

-    {0,  "\377\330\377",		 3, "JPEG",         CL_TYPE_GRAPHICS},

-    {6,  "JFIF",			 4, "JPEG",         CL_TYPE_GRAPHICS},

-    {6,  "Exif",			 4, "JPEG",         CL_TYPE_GRAPHICS},

-    {0,  "\x89PNG",			 4, "PNG",          CL_TYPE_GRAPHICS},

-    {0,  "RIFF",                         4, "RIFF",         CL_TYPE_RIFF},

-    {0,  "RIFX",                         4, "RIFX",         CL_TYPE_RIFF},

-

-    /* Others */

-

-    {0,  "\320\317\021\340\241\261\032\341", 8, "OLE2 container", CL_TYPE_MSOLE2},

-    {0,  "%PDF-",			 5, "PDF document", CL_TYPE_PDF},

-    {0,  "\266\271\254\256\376\377\377\377", 8, "CryptFF", CL_TYPE_CRYPTFF},

-    {0,  "{\\rtf",                           5, "RTF", CL_TYPE_RTF}, 

-

-    /* Ignored types */

-

-    {0,  "\000\000\001\263",             4, "MPEG video stream",  CL_TYPE_DATA},

-    {0,  "\000\000\001\272",             4, "MPEG sys stream",    CL_TYPE_DATA},

-    {0,  "OggS",                         4, "Ogg Stream",         CL_TYPE_DATA},

-    {0,  "ID3",				 3, "MP3",		  CL_TYPE_DATA},

-    {0,  "\377\373\220",		 3, "MP3",		  CL_TYPE_DATA},

-    {0,  "%!PS-Adobe-",			11, "PostScript",	  CL_TYPE_DATA},

-    {0,  "\060\046\262\165\216\146\317", 7, "WMA/WMV/ASF",	  CL_TYPE_DATA},

-    {0,  ".RMF" ,			 4, "Real Media File",	  CL_TYPE_DATA},

-

-    {0, NULL,				 0, NULL,		  CL_TYPE_UNKNOWN_DATA}

-};

+    for(i = 0; ftmap[i].name; i++)

+	if(!strcmp(ftmap[i].name, name))

+	    return ftmap[i].code;

-static const struct cli_smagic_s cli_smagic[] = {

-

-    /* "\nFrom: " * "\nContent-Type: " */

-    {"0a46726f6d3a20{-2048}0a436f6e74656e742d547970653a20", "Mail file", CL_TYPE_MAIL},

-

-    /* "\nReceived: " * "\nContent-Type: " */

-    {"0a52656365697665643a20{-2048}0a436f6e74656e742d547970653a20", "Mail file", CL_TYPE_MAIL},

-

-    /* "\nReceived: " * "\nContent-type: " */

-    {"0a52656365697665643a20{-2048}0a436f6e74656e742d747970653a20", "Mail file", CL_TYPE_MAIL},

-

-    /* "MIME-Version: " * "\nContent-Type: " */

-    {"4d494d452d56657273696f6e3a20{-2048}0a436f6e74656e742d547970653a20", "Mail file", CL_TYPE_MAIL},

-

-    /* remember the matcher is case sensitive */

-    {"3c62723e",       "HTML data", CL_TYPE_HTML},	/* <br> */

-    {"3c42723e",       "HTML data", CL_TYPE_HTML},	/* <Br> */

-    {"3c42523e",       "HTML data", CL_TYPE_HTML},	/* <BR> */

-    {"3c703e",	       "HTML data", CL_TYPE_HTML},	/* <p> */

-    {"3c503e",	       "HTML data", CL_TYPE_HTML},	/* <P> */

-    {"68726566",       "HTML data", CL_TYPE_HTML},	/* href */

-    {"48726566",       "HTML data", CL_TYPE_HTML},	/* Href */

-    {"48524546",       "HTML data", CL_TYPE_HTML},	/* HREF */

-    {"3c68746d6c3e",   "HTML data", CL_TYPE_HTML},      /* <html> */

-    {"3c48544d4c3e",   "HTML data", CL_TYPE_HTML},      /* <HTML> */

-    {"3c48746d6c3e",   "HTML data", CL_TYPE_HTML},      /* <Html> */

-    {"3c686561643e",   "HTML data", CL_TYPE_HTML},      /* <head> */

-    {"3c484541443e",   "HTML data", CL_TYPE_HTML},      /* <HEAD> */

-    {"3c486561643e",   "HTML data", CL_TYPE_HTML},      /* <Head> */

-    {"3c666f6e74",     "HTML data", CL_TYPE_HTML},	/* <font */

-    {"3c466f6e74",     "HTML data", CL_TYPE_HTML},	/* <Font */

-    {"3c464f4e54",     "HTML data", CL_TYPE_HTML},	/* <FONT */

-    {"3c696d67",       "HTML data", CL_TYPE_HTML},      /* <img */

-    {"3c494d47",       "HTML data", CL_TYPE_HTML},      /* <IMG */

-    {"3c496d67",       "HTML data", CL_TYPE_HTML},      /* <Img */

-    {"3c736372697074", "HTML data", CL_TYPE_HTML},	/* <script */

-    {"3c536372697074", "HTML data", CL_TYPE_HTML},	/* <Script */

-    {"3c534352495054", "HTML data", CL_TYPE_HTML},	/* <SCRIPT */

-    {"3c6f626a656374", "HTML data", CL_TYPE_HTML},      /* <object */

-    {"3c4f626a656374", "HTML data", CL_TYPE_HTML},      /* <Object */

-    {"3c4f424a454354", "HTML data", CL_TYPE_HTML},      /* <OBJECT */

-    {"3c696672616d65", "HTML data", CL_TYPE_HTML},      /* <iframe */

-    {"3c494652414d45", "HTML data", CL_TYPE_HTML},      /* <IFRAME */

-    {"3c7461626c65",   "HTML data", CL_TYPE_HTML},	/* <table */

-    {"3c5441424c45",   "HTML data", CL_TYPE_HTML},	/* <TABLE */

-

-    {"526172211a0700", "RAR-SFX", CL_TYPE_RARSFX},

-    {"504b0304", "ZIP-SFX", CL_TYPE_ZIPSFX},

-    {"4d534346", "CAB-SFX", CL_TYPE_CABSFX},

-    {"60ea{7}0002", "ARJ-SFX", CL_TYPE_ARJSFX},

-    {"60ea{7}0102", "ARJ-SFX", CL_TYPE_ARJSFX},

-    {"60ea{7}0202", "ARJ-SFX", CL_TYPE_ARJSFX},

-    {"efbeadde4e756c6c736f6674496e7374", "NSIS", CL_TYPE_NULSFT},

-    {"a3484bbe986c4aa9994c530a86d6487d41553321454130(35|36)", "AUTOIT", CL_TYPE_AUTOIT},

-

-    {"4d5a{60-300}50450000", "PE", CL_TYPE_MSEXE},

-

-    {NULL,  NULL,   CL_TYPE_UNKNOWN_DATA}

-};

+    return CL_TYPE_ERROR;

+}

-static char internat[256] = {

-    /* TODO: Remember to buy a beer to Joerg Wunsch <joerg@FreeBSD.ORG> */

-    0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0,  /* 0x0X */

-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,  /* 0x1X */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 0x2X */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 0x3X */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 0x4X */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 0x5X */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 0x6X */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0,  /* 0x7X */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 0x8X */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 0x9X */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 0xaX */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 0xbX */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 0xcX */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 0xdX */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 0xeX */

-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1   /* 0xfX */

-};

+void cli_ftfree(struct cli_ftype *ftypes)

+{

+	struct cli_ftype *pt;

+

+    while(ftypes) {

+	pt = ftypes;

+	ftypes = ftypes->next;

+	free(pt->magic);

+	free(pt->tname);

+	free(pt);

+    }

+}

-cli_file_t cli_filetype(const unsigned char *buf, size_t buflen)

+cli_file_t cli_filetype(const unsigned char *buf, size_t buflen, const struct cl_engine *engine)

 {

-	int i, text = 1, len;

+	struct cli_ftype *ftype = engine->ftypes;

-    for(i = 0; cli_magic[i].magic; i++) {

-	if(buflen >= cli_magic[i].offset+cli_magic[i].length) {

-	    if(memcmp(buf+cli_magic[i].offset, cli_magic[i].magic, cli_magic[i].length) == 0) {

-		cli_dbgmsg("Recognized %s file\n", cli_magic[i].descr);

-		return cli_magic[i].type;

+    while(ftype) {

+	if(ftype->offset + ftype->length <= buflen) {

+	    if(!memcmp(buf + ftype->offset, ftype->magic, ftype->length)) {

+		cli_dbgmsg("Recognized %s file\n", ftype->tname);

+		return ftype->type;

 	    }

 	}

+	ftype = ftype->next;

     }

-/* improve or drop this code

+/* FIXME: improve or drop this code

  * https://wwws.clamav.net/bugzilla/show_bug.cgi?id=373

  *

+	int i, text = 1, len;

     buflen < 25 ? (len = buflen) : (len = 25);

     for(i = 0; i < len; i++)

 	if(!iscntrl(buf[i]) && !isprint(buf[i]) && !internat[buf[i] & 0xff]) {

 	    text = 0;

 	    break;

 	}

-*/

     return text ? CL_TYPE_UNKNOWN_TEXT : CL_TYPE_UNKNOWN_DATA;

+*/

+    return CL_TYPE_UNKNOWN_TEXT;

 }

 int is_tar(unsigned char *buf, unsigned int nbytes);

@@ -263,7 +153,7 @@ cli_file_t cli_filetype2(int desc, const struct cl_engine *engine)

     memset(smallbuff, 0, sizeof(smallbuff));

     if((bread = read(desc, smallbuff, MAGIC_BUFFER_SIZE)) > 0)

-	ret = cli_filetype(smallbuff, bread);

+	ret = cli_filetype(smallbuff, bread, engine);

     if(engine && ret == CL_TYPE_UNKNOWN_TEXT) {

 	root = engine->root[0];

@@ -357,10 +247,10 @@ cli_file_t cli_filetype2(int desc, const struct cl_engine *engine)

 	    if(!memcmp(bigbuff + 32769, "CD001" , 5) || !memcmp(bigbuff + 37633, "CD001" , 5)) {

 		cli_dbgmsg("Recognized ISO 9660 CD-ROM data\n");

-		ret = CL_TYPE_DATA;

+		ret = CL_TYPE_IGNORED;

 	    } else if(!memcmp(bigbuff + 32776, "CDROM" , 5)) {

 		cli_dbgmsg("Recognized High Sierra CD-ROM data\n");

-		ret = CL_TYPE_DATA;

+		ret = CL_TYPE_IGNORED;

 	    }

 	}

@@ -369,38 +259,3 @@ cli_file_t cli_filetype2(int desc, const struct cl_engine *engine)

     return ret;

 }

-

-int cli_addtypesigs(struct cl_engine *engine)

-{

-	int i, ret;

-	struct cli_matcher *root;

-

-

-    if(!engine->root[0]) {

-	cli_dbgmsg("cli_addtypesigs: Need to allocate AC trie in engine->root[0]\n");

-	root = engine->root[0] = (struct cli_matcher *) cli_calloc(1, sizeof(struct cli_matcher));

-	if(!root) {

-	    cli_errmsg("cli_addtypesigs: Can't initialise AC pattern matcher\n");

-	    return CL_EMEM;

-	}

-

-	if((ret = cli_ac_init(root, cli_ac_mindepth, cli_ac_maxdepth))) {

-	    /* No need to free previously allocated memory here - all engine

-	     * elements will be properly freed by cl_free()

-	     */

-	    cli_errmsg("cli_addtypesigs: Can't initialise AC pattern matcher\n");

-	    return ret;

-	}

-    } else {

-	root = engine->root[0];

-    }

-

-    for(i = 0; cli_smagic[i].sig; i++) {

-	if((ret = cli_parse_add(root, cli_smagic[i].descr, cli_smagic[i].sig, cli_smagic[i].type, NULL, 0))) {

-	    cli_errmsg("cli_addtypesigs: Problem adding signature for %s\n", cli_smagic[i].descr);

-	    return ret;

-	}

-    }

-

-    return 0;

-}

@@ -1,4 +1,7 @@

 /*

+ *  Copyright (C) 2007 Sourcefire, Inc.

+ *  Author: Tomasz Kojm <tkojm@clamav.net>

+ *

  *  Copyright (C) 2002 - 2005 Tomasz Kojm <tkojm@clamav.net>

  *  With enhancements from Thomas Lamy <Thomas.Lamy@in-online.net>

  *

@@ -22,6 +25,9 @@

 #include <sys/types.h>

+#include "clamav.h"

+#include "cltypes.h"

+

 #define MAGIC_BUFFER_SIZE 256

 #define CL_TYPENO 500

 #define MAX_EMBEDDED_OBJ 10

@@ -29,9 +35,10 @@

 typedef enum {

     CL_TYPE_UNKNOWN_TEXT = CL_TYPENO,

     CL_TYPE_UNKNOWN_DATA,

+    CL_TYPE_IGNORED,

+    CL_TYPE_ERROR,

     CL_TYPE_MSEXE,

     CL_TYPE_ELF,

-    CL_TYPE_DATA,

     CL_TYPE_POSIX_TAR,

     CL_TYPE_OLD_TAR,

     CL_TYPE_GZ,

@@ -68,6 +75,15 @@ typedef enum {

     CL_TYPE_AUTOIT

 } cli_file_t;

+struct cli_ftype {

+    cli_file_t type;

+    uint32_t offset;

+    unsigned char *magic;

+    uint16_t length;

+    char *tname;

+    struct cli_ftype *next;

+};

+

 struct cli_matched_type {

     cli_file_t type;

     off_t offset;

@@ -75,7 +91,9 @@ struct cli_matched_type {

     struct cli_matched_type *next;

 };

-cli_file_t cli_filetype(const unsigned char *buf, size_t buflen);

+cli_file_t cli_ftcode(const char *name);

+void cli_ftfree(struct cli_ftype *ftypes);

+cli_file_t cli_filetype(const unsigned char *buf, size_t buflen, const struct cl_engine *engine);

 cli_file_t cli_filetype2(int desc, const struct cl_engine *engine);

 int cli_addtypesigs(struct cl_engine *engine);

new file mode 100644

@@ -0,0 +1,145 @@

+/*

+ *  Static filetype data for use when daily.ft is not available.

+ *  Copyright (C) 2007 Sourcefire, Inc.

+ *  Author: Tomasz Kojm <tkojm@clamav.net>

+ *

+ *  This program is free software; you can redistribute it and/or modify

+ *  it under the terms of the GNU General Public License version 2 as

+ *  published by the Free Software Foundation.

+ *

+ *  This program is distributed in the hope that it will be useful,

+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of

+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ *  GNU General Public License for more details.

+ *

+ *  You should have received a copy of the GNU General Public License

+ *  along with this program; if not, write to the Free Software

+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

+ *  MA 02110-1301, USA.

+ */

+

+#ifndef __FILETYPES_INT_H

+#define __FILETYPES_INT_H

+

+/* Generated with the following perl script:

+#!/usr/bin/perl

+open(FT, "daily.ft") or die "Can't open daily.ft";

+print "static const char *ftypes_int[] = {\n";

+while($line = <FT>) {

+    chomp($line);

+    print "  \"$line\",\n";

+}

+print "  NULL\n};\n"

+*/

+

+static const char *ftypes_int[] = {

+  "0:4d5a:MS-EXE/DLL:CL_TYPE_MSEXE",

+  "0:7f454c46:ELF:CL_TYPE_ELF",

+  "0:52617221:RAR:CL_TYPE_RAR",

+  "0:504b0304:ZIP:CL_TYPE_ZIP",

+  "0:504b3030504b0304:ZIP:CL_TYPE_ZIP",

+  "0:1f8b:GZip:CL_TYPE_GZ",

+  "0:425a68:BZip:CL_TYPE_BZ",

+  "0:60ea:ARJ:CL_TYPE_ARJ",

+  "0:535a4444:compress.exe'd:CL_TYPE_MSSZDD",

+  "0:4d534346:MS CAB:CL_TYPE_MSCAB",

+  "0:49545346:MS CHM:CL_TYPE_MSCHM",

+  "8:19040010:SIS:CL_TYPE_SIS",

+  "0:23407e5e:SCRENC:CL_TYPE_SCRENC",

+  "0:28546869732066696c65206d75737420626520636f6e76657274656420776974682042696e48657820342e3029:BinHex:CL_TYPE_BINHEX",

+  "0:46726f6d20:MBox:CL_TYPE_MAIL",

+  "0:52656365697665643a20:Raw mail:CL_TYPE_MAIL",

+  "0:52657475726e2d506174683a20:Maildir:CL_TYPE_MAIL",

+  "0:52657475726e2d706174683a20:Maildir:CL_TYPE_MAIL",

+  "0:44656c6976657265642d546f3a20:Mail:CL_TYPE_MAIL",

+  "0:582d5549444c3a20:Mail:CL_TYPE_MAIL",

+  "0:582d4170706172656e746c792d546f3a20:Mail:CL_TYPE_MAIL",

+  "0:582d456e76656c6f70652d46726f6d3a20:Mail:CL_TYPE_MAIL",

+  "0:582d4f726967696e616c2d546f3a20:Mail:CL_TYPE_MAIL",

+  "0:582d53796d616e7465632d:Symantec:CL_TYPE_MAIL",

+  "0:582d455653:EVS mail:CL_TYPE_MAIL",

+  "0:582d5265616c2d546f3a20:Mail:CL_TYPE_MAIL",

+  "0:582d53696576653a20:Mail:CL_TYPE_MAIL",

+  "0:3e46726f6d20:Mail:CL_TYPE_MAIL",

+  "0:446174653a20:Mail:CL_TYPE_MAIL",

+  "0:4d6573736167652d49643a20:Mail:CL_TYPE_MAIL",

+  "0:4d6573736167652d49443a20:Mail:CL_TYPE_MAIL",

+  "0:456e76656c6f70652d746f3a20:Mail:CL_TYPE_MAIL",

+  "0:44656c69766572792d646174653a20:Mail:CL_TYPE_MAIL",

+  "0:546f3a20:Mail:CL_TYPE_MAIL",

+  "0:5375626a6563743a20:Mail:CL_TYPE_MAIL",

+  "0:466f723a20:Eserv mail:CL_TYPE_MAIL",

+  "0:46726f6d3a20:Exim mail:CL_TYPE_MAIL",

+  "0:763a0d0a52656365697665643a20:VPOP3 Mail (DOS):CL_TYPE_MAIL",

+  "0:763a0a52656365697665643a20:VPOP3 Mail (UNIX):CL_TYPE_MAIL",

+  "0:48692e20546869732069732074686520716d61696c2d73656e64:Qmail bounce:CL_TYPE_MAIL",

+  "0:789f3e22:TNEF:CL_TYPE_TNEF",

+  "0:626567696e20:UUencoded:CL_TYPE_UUENCODED",

+  "0:2142444e:PST:CL_TYPE_PST",

+  "0:474946:GIF:CL_TYPE_GRAPHICS",

+  "0:424d:BMP:CL_TYPE_GRAPHICS",

+  "0:ffd8ff:JPEG:CL_TYPE_GRAPHICS",

+  "6:4a464946:JPEG:CL_TYPE_GRAPHICS",

+  "6:45786966:JPEG:CL_TYPE_GRAPHICS",

+  "0:89504e47:PNG:CL_TYPE_GRAPHICS",

+  "0:52494646:RIFF:CL_TYPE_RIFF",

+  "0:52494658:RIFX:CL_TYPE_RIFF",

+  "0:d0cf11e0a1b11ae1:OLE2 container:CL_TYPE_MSOLE2",

+  "0:255044462d:PDF document:CL_TYPE_PDF",

+  "0:b6b9acaefeffffff:CryptFF:CL_TYPE_CRYPTFF",

+  "0:7b5c727466:RTF:CL_TYPE_RTF",

+  "0:000001b3:MPEG video stream:CL_TYPE_IGNORED",

+  "0:000001ba:MPEG sys stream:CL_TYPE_IGNORED",

+  "0:4f676753:Ogg Stream:CL_TYPE_IGNORED",

+  "0:494433:MP3:CL_TYPE_IGNORED",

+  "0:fffb90:MP3:CL_TYPE_IGNORED",

+  "0:252150532d41646f62652d:PostScript:CL_TYPE_IGNORED",

+  "0:3026b2758e66cf:WMA/WMV/ASF:CL_TYPE_IGNORED",

+  "0:2e524d46:Real Media File:CL_TYPE_IGNORED",

+  "*:0a46726f6d3a20{-2048}0a436f6e74656e742d547970653a20:Mail file:CL_TYPE_MAIL",

+  "*:0a52656365697665643a20{-2048}0a436f6e74656e742d547970653a20:Mail file:CL_TYPE_MAIL",

+  "*:0a52656365697665643a20{-2048}0a436f6e74656e742d747970653a20:Mail file:CL_TYPE_MAIL",

+  "*:4d494d452d56657273696f6e3a20{-2048}0a436f6e74656e742d547970653a20:Mail file:CL_TYPE_MAIL",

+  "*:3c62723e:HTML data:CL_TYPE_HTML",

+  "*:3c42723e:HTML data:CL_TYPE_HTML",

+  "*:3c42523e:HTML data:CL_TYPE_HTML",

+  "*:3c703e:HTML data:CL_TYPE_HTML",

+  "*:3c503e:HTML data:CL_TYPE_HTML",

+  "*:68726566:HTML data:CL_TYPE_HTML",

+  "*:48726566:HTML data:CL_TYPE_HTML",

+  "*:48524546:HTML data:CL_TYPE_HTML",

+  "*:3c68746d6c3e:HTML data:CL_TYPE_HTML",

+  "*:3c48544d4c3e:HTML data:CL_TYPE_HTML",

+  "*:3c48746d6c3e:HTML data:CL_TYPE_HTML",

+  "*:3c686561643e:HTML data:CL_TYPE_HTML",

+  "*:3c484541443e:HTML data:CL_TYPE_HTML",

+  "*:3c486561643e:HTML data:CL_TYPE_HTML",

+  "*:3c666f6e74:HTML data:CL_TYPE_HTML",

+  "*:3c466f6e74:HTML data:CL_TYPE_HTML",

+  "*:3c464f4e54:HTML data:CL_TYPE_HTML",

+  "*:3c696d67:HTML data:CL_TYPE_HTML",

+  "*:3c494d47:HTML data:CL_TYPE_HTML",

+  "*:3c496d67:HTML data:CL_TYPE_HTML",

+  "*:3c736372697074:HTML data:CL_TYPE_HTML",

+  "*:3c536372697074:HTML data:CL_TYPE_HTML",

+  "*:3c534352495054:HTML data:CL_TYPE_HTML",

+  "*:3c6f626a656374:HTML data:CL_TYPE_HTML",

+  "*:3c4f626a656374:HTML data:CL_TYPE_HTML",

+  "*:3c4f424a454354:HTML data:CL_TYPE_HTML",

+  "*:3c696672616d65:HTML data:CL_TYPE_HTML",

+  "*:3c494652414d45:HTML data:CL_TYPE_HTML",

+  "*:3c7461626c65:HTML data:CL_TYPE_HTML",

+  "*:3c5441424c45:HTML data:CL_TYPE_HTML",

+  "*:526172211a0700:RAR-SFX:CL_TYPE_RARSFX",

+  "*:504b0304:ZIP-SFX:CL_TYPE_ZIPSFX",

+  "*:4d534346:CAB-SFX:CL_TYPE_CABSFX",

+  "*:60ea{7}0002:ARJ-SFX:CL_TYPE_ARJSFX",

+  "*:60ea{7}0102:ARJ-SFX:CL_TYPE_ARJSFX",

+  "*:60ea{7}0202:ARJ-SFX:CL_TYPE_ARJSFX",

+  "*:efbeadde4e756c6c736f6674496e7374:NSIS:CL_TYPE_NULSFT",

+  "*:a3484bbe986c4aa9994c530a86d6487d41553321454130(35|36):AUTOIT:CL_TYPE_AUTOIT",

+  "*:4d5a{60-300}50450000:PE:CL_TYPE_MSEXE",

+  NULL

+};

+

+#endif

@@ -239,7 +239,7 @@ static	int	rfc1341(message *m, const char *dir);

 #endif

 static	bool	usefulHeader(int commandNumber, const char *cmd);

 static	char	*getline_from_mbox(char *buffer, size_t len, FILE *fin);

-static	bool	isBounceStart(const char *line);

+static	bool	isBounceStart(mbox_ctx *mctx, const char *line);

 static	bool	exportBinhexMessage(mbox_ctx *mctx, message *m);

 static	int	exportBounceMessage(mbox_ctx *ctx, text *start);

 static	message	*do_multipart(message *mainMessage, message **messages, int i, mbox_status *rc, mbox_ctx *mctx, message *messageIn, text **tptr, unsigned int recursion_level);

@@ -608,7 +608,7 @@ cli_mbox(const char *dir, int desc, cli_ctx *ctx)

 		free_map();

-		type = cli_filetype(start, size);

+		type = cli_filetype(start, size, ctx->engine);

 		if((type == CL_TYPE_UNKNOWN_TEXT) &&

 		   (strncmp(start, "Microsoft Mail Internet Headers", 31) == 0))

@@ -2848,12 +2848,12 @@ parseEmailBody(message *messageIn, text *textIn, mbox_ctx *mctx, unsigned int re

 			if(lookahead_definately_is_bounce)

 				lookahead_definately_is_bounce = FALSE;

-			else if(!isBounceStart(lineGetData(l)))

+			else if(!isBounceStart(mctx, lineGetData(l)))

 				continue;

 			lookahead = t->t_next;

 			if(lookahead) {

-				if(isBounceStart(lineGetData(lookahead->t_line))) {

+				if(isBounceStart(mctx, lineGetData(lookahead->t_line))) {

 					lookahead_definately_is_bounce = TRUE;

 					/* don't save worthless header lines */

 					continue;

@@ -2957,7 +2957,7 @@ parseEmailBody(message *messageIn, text *textIn, mbox_ctx *mctx, unsigned int re

 				l = t->t_line;

 				if((!inheader) && l) {

 					s = lineGetData(l);

-					if(isBounceStart(s)) {

+					if(isBounceStart(mctx, s)) {

 						cli_dbgmsg("Found the start of another bounce candidate (%s)\n", s);

 						lookahead_definately_is_bounce = TRUE;

 						break;

@@ -4796,7 +4796,7 @@ getline_from_mbox(char *buffer, size_t len, FILE *fin)

  * Is this line a candidate for the start of a bounce message?

  */

 static bool

-isBounceStart(const char *line)

+isBounceStart(mbox_ctx *mctx, const char *line)

 {

 	size_t len;

@@ -4832,7 +4832,7 @@ isBounceStart(const char *line)

 			return FALSE;

 		return TRUE;

 	}

-	return cli_filetype((const unsigned char *)line, len) == CL_TYPE_MAIL;

+	return cli_filetype((const unsigned char *)line, len, mctx->ctx->engine) == CL_TYPE_MAIL;

 }

 /*

@@ -1102,7 +1102,7 @@ messageIsEncoding(message *m)

 		m->encoding = m->body_last;

 	else if((m->bounce == NULL) &&

 		(strncasecmp(line, "Received: ", 10) == 0) &&

-		(cli_filetype((const unsigned char *)line, strlen(line)) == CL_TYPE_MAIL))

+		(cli_filetype((const unsigned char *)line, strlen(line), m->ctx->engine) == CL_TYPE_MAIL))

 			m->bounce = m->body_last;

 		/* Not needed with fast track visa technology */

 	/*else if((m->uuencode == NULL) && isuuencodebegin(line))

@@ -53,6 +53,8 @@

 #include "str.h"

 #include "dconf.h"

 #include "lockdb.h"

+#include "filetypes.h"

+#include "filetypes_int.h"

 #include "readdb.h"

 #include "phishcheck.h"

@@ -76,7 +78,6 @@ static pthread_mutex_t cli_ref_mutex = PTHREAD_MUTEX_INITIALIZER;

 int cl_loaddb(const char *filename, struct cl_engine **engine, unsigned int *signo);

 int cl_loaddbdir(const char *dirname, struct cl_engine **engine, unsigned int *signo);

-

 int cli_parse_add(struct cli_matcher *root, const char *virname, const char *hexsig, unsigned short type, const char *offset, unsigned short target)

 {

 	struct cli_bm_patt *bm_new;

@@ -594,6 +595,99 @@ static int cli_loadndb(FILE *fd, struct cl_engine **engine, unsigned int *signo,

     return CL_SUCCESS;

 }

+#define FT_TOKENS 4

+static int cli_loadft(FILE *fd, struct cl_engine **engine, unsigned int options, unsigned int internal)

+{

+	const char *tokens[FT_TOKENS];

+	char buffer[FILEBUFF];

+	unsigned int line = 0;

+	struct cli_ftype *new;

+	cli_file_t type;

+	int ret;

+

+

+    if((ret = cli_initengine(engine, options))) {

+	cl_free(*engine);

+	return ret;

+    }

+

+    if((ret = cli_initroots(*engine, options))) {

+	cl_free(*engine);

+	return ret;

+    }

+

+    while(1) {

+	if(internal) {

+	    if(!ftypes_int[line])

+		break;

+	    strncpy(buffer, ftypes_int[line], sizeof(buffer));

+	} else {

+	    if(!fgets(buffer, FILEBUFF, fd))

+		break;

+	    cli_chomp(buffer);

+	}

+	line++;

+	cli_strtokenize(buffer, ':', FT_TOKENS, tokens);

+

+	if(!tokens[0] || !tokens[1] || !tokens[2] || !tokens[3]) {

+	    ret = CL_EMALFDB;

+	    break;

+	}

+

+	type = cli_ftcode(tokens[3]);

+	if(type == CL_TYPE_ERROR) {

+	    ret = CL_EMALFDB;

+	    break;

+	}

+

+	if(*tokens[0] == '*') {

+	    if((ret = cli_parse_add((*engine)->root[0], tokens[2], tokens[1], type, NULL, 0)))

+		break;

+

+	} else {

+	    new = (struct cli_ftype *) cli_malloc(sizeof(struct cli_ftype));

+	    if(!new) {

+		ret = CL_EMEM;

+		break;

+	    }

+	    new->type = type;

+	    new->offset = atoi(tokens[0]);

+	    new->magic = (unsigned char *) cli_hex2str(tokens[1]);

+	    if(!new->magic) {

+		cli_errmsg("cli_loadft: Can't decode the hex string\n");

+		ret = CL_EMALFDB;

+		free(new);

+		break;

+	    }

+	    new->length = strlen(tokens[1]) / 2;

+	    new->tname = cli_strdup(tokens[2]);

+	    if(!new->tname) {

+		free(new->magic);

+		free(new);

+		ret = CL_EMEM;

+		break;

+	    }

+	    new->next = (*engine)->ftypes;

+	    (*engine)->ftypes = new;

+	}

+    }

+

+    if(!line) {

+	cli_errmsg("Empty %s filetype database\n", internal ? "built-in" : ".ft");

+	cl_free(*engine);

+	return CL_EMALFDB;

+    }

+

+    if(ret) {

+	cli_errmsg("Problem parsing %s filetype database at line %u\n", internal ? "built-in" : ".ft", line);

+	cl_free(*engine);

+	return ret;

+    }

+

+    cli_dbgmsg("Loaded %u filetype definitions\n", line);

+    return CL_SUCCESS;

+}

+

 static int scomp(const void *a, const void *b)

 {

     return *(const uint32_t *)a - *(const uint32_t *)b;

@@ -1022,6 +1116,9 @@ static int cli_load(const char *filename, struct cl_engine **engine, unsigned in

 	    ret = cli_loadpdb(fd, engine, options);

 	else

 	    skipped = 1;

+    } else if(cli_strbcasestr(filename, ".ft")) {

+	ret = cli_loadft(fd, engine, options, 0);

+

     } else {

 	cli_dbgmsg("cli_load: unknown extension - assuming old database format\n");

 	ret = cli_loaddb(fd, engine, signo, options);

@@ -1061,6 +1158,7 @@ int cl_loaddb(const char *filename, struct cl_engine **engine, unsigned int *sig

 	cli_strbcasestr(ext, ".rmd")   ||	\

 	cli_strbcasestr(ext, ".pdb")   ||	\

 	cli_strbcasestr(ext, ".wdb")   ||	\

+	cli_strbcasestr(ext, ".ft")    ||	\

 	cli_strbcasestr(ext, ".inc")   ||	\

 	cli_strbcasestr(ext, ".cvd")		\

     )

@@ -1509,18 +1607,24 @@ void cl_free(struct cl_engine *engine)

     if(engine->dconf)

 	free(engine->dconf);

+    cli_ftfree(engine->ftypes);

     cli_freelocks();

     free(engine);

 }

 int cl_build(struct cl_engine *engine)

 {

-	int i, ret;

+	unsigned int i;

+	int ret;

 	struct cli_matcher *root;

-    if((ret = cli_addtypesigs(engine)))

-	return ret;

+    if(!engine)

+	return CL_ENULLARG;

+

+    if(!engine->ftypes)

+	if((ret = cli_loadft(NULL, &engine, 0, 1)))

+	    return ret;

     for(i = 0; i < CL_TARGET_TABLE_SIZE; i++)

 	if((root = engine->root[i]))

@@ -1768,7 +1768,7 @@ int cli_magic_scandesc(int desc, cli_ctx *ctx)

     type = cli_filetype2(desc, ctx->engine);

     lseek(desc, 0, SEEK_SET);

-    if(type != CL_TYPE_DATA && ctx->engine->sdb) {

+    if(type != CL_TYPE_IGNORED && ctx->engine->sdb) {

 	if((ret = cli_scanraw(desc, ctx, type, 0)) == CL_VIRUS)

 	    return CL_VIRUS;

 	lseek(desc, 0, SEEK_SET);

@@ -1777,6 +1777,9 @@ int cli_magic_scandesc(int desc, cli_ctx *ctx)

     type == CL_TYPE_MAIL ? ctx->mrec++ : ctx->arec++;

     switch(type) {

+	case CL_TYPE_IGNORED:

+	    break;

+

 	case CL_TYPE_RAR:

 #ifdef ENABLE_UNRAR

 	    if(SCAN_ARCHIVE && (DCONF_ARCH & ARCH_CONF_RAR))

@@ -1922,14 +1925,6 @@ int cli_magic_scandesc(int desc, cli_ctx *ctx)

 		ret = cli_scansis(desc, ctx);

 	    break;

-	case CL_TYPE_DATA:

-	    /* it could be a false positive and a standard DOS .COM file */

-	    {

-		struct stat s;

-		if(fstat(desc, &s) == 0 && S_ISREG(s.st_mode) && s.st_size < 65536)

-		type = CL_TYPE_UNKNOWN_DATA;

-	    }

-

 	case CL_TYPE_UNKNOWN_DATA:

 	    ret = cli_check_mydoom_log(desc, ctx->virname);

 	    break;

@@ -1947,7 +1942,7 @@ int cli_magic_scandesc(int desc, cli_ctx *ctx)

 	}

     }

-    if(type != CL_TYPE_DATA && ret != CL_VIRUS && !ctx->engine->sdb) {