/*

  *  Copyright (C) 2002 - 2004 Tomasz Kojm <tkojm@clamav.net>

  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
  *  the Free Software Foundation; either version 2 of the License, or
  *  (at your option) any later version.
  *
  *  This program is distributed in the hope that it will be useful,
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  *  GNU General Public License for more details.
  *
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, write to the Free Software
  *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 

 #if HAVE_CONFIG_H
 #include "clamav-config.h"
 #endif
 

 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <dirent.h>
 

 #if HAVE_MMAP
 #if HAVE_SYS_MMAN_H
 #include <sys/mman.h>
 #else /* HAVE_SYS_MMAN_H */
 #undef HAVE_MMAP
 #endif
 #endif
 

 #include <mspack.h>
 

 #ifdef CL_THREAD_SAFE
 #  include <pthread.h>
 pthread_mutex_t cli_scanrar_mutex = PTHREAD_MUTEX_INITIALIZER;
 #endif

 int cli_scanrar_inuse = 0;

 extern short cli_leavetemps_flag;
 

 extern int cli_mbox(const char *dir, int desc); /* FIXME */
 

 #include "clamav.h"
 #include "others.h"

 #include "scanners.h"

 #include "matcher-ac.h"
 #include "matcher-bm.h"

 #include "matcher.h"
 #include "unrarlib.h"

 #include "ole2_extract.h"
 #include "vba_extract.h"

 #include "msexpand.h"

 #include "chmunpack.h"

 #include "pe.h"

 #include "filetypes.h"
 #include "htmlnorm.h"

 
 #ifdef HAVE_ZLIB_H
 #include <zlib.h>
 #include <zzip.h>
 #endif
 
 #ifdef HAVE_BZLIB_H
 #include <bzlib.h>
 #endif
 

 #define SCAN_ARCHIVE	    (options & CL_ARCHIVE)
 #define SCAN_MAIL	    (options & CL_MAIL)
 #define SCAN_OLE2	    (options & CL_OLE2)

 #define SCAN_HTML	    (options & CL_HTML)

 #define SCAN_PE		    (options & CL_PE)

 #define DISABLE_RAR	    (options & CL_DISABLERAR)
 #define DETECT_ENCRYPTED    (options & CL_ENCRYPTED)

 #define MAX_MAIL_RECURSION  15

 
 static int cli_scanfile(const char *filename, const char **virname, unsigned long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec);

 #ifdef CL_THREAD_SAFE

 static void cli_unlock_mutex(void *mtx)

 {
     cli_dbgmsg("Pthread cancelled. Unlocking mutex.\n");
     pthread_mutex_unlock(mtx);
 }
 #endif
 

 static int cli_scanrar(int desc, const char **virname, long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)

 {

 	FILE *tmp = NULL;

 	int files = 0, fd, ret = CL_CLEAN, afiles;

 	ArchiveList_struct *rarlist = NULL;

 	ArchiveList_struct *rarlist_head = NULL;

 	char *rar_data_ptr;
 	unsigned long rar_data_size;
 
     cli_dbgmsg("Starting scanrar()\n");
 
 
 #ifdef CL_THREAD_SAFE
     pthread_cleanup_push(cli_unlock_mutex, &cli_scanrar_mutex);
     pthread_mutex_lock(&cli_scanrar_mutex);
     cli_scanrar_inuse = 1;
 #endif
 

     if(! (afiles = urarlib_list(desc, (ArchiveList_struct *) &rarlist))) {

 #ifdef CL_THREAD_SAFE
 	pthread_mutex_unlock(&cli_scanrar_mutex);
 	cli_scanrar_inuse = 0;
 #endif
 	return CL_ERAR;
     }
 

     cli_dbgmsg("Rar -> Number of archived files: %d\n", afiles);
 

     rarlist_head = rarlist;
 

     while(rarlist) {

 	if(DETECT_ENCRYPTED && (rarlist->item.Flags & 4)) {
 	    files++;
 	    cli_dbgmsg("Rar -> Encrypted files found in archive.\n");

 	    lseek(desc, 0, SEEK_SET);

 	    if(cli_scandesc(desc, virname, scanned, root, 0) != CL_VIRUS)

 		*virname = "Encrypted.RAR";

 	    ret = CL_VIRUS;
 	    break;
 	}

 
 	if(limits) {

 	    if(limits->maxfilesize && (rarlist->item.UnpSize > (unsigned int) limits->maxfilesize)) {
 		cli_dbgmsg("RAR->%s: Size exceeded (%u, max: %lu)\n", rarlist->item.Name, (unsigned int) rarlist->item.UnpSize, limits->maxfilesize);

 		rarlist = rarlist->next;
 		files++;

 		/* ret = CL_EMAXSIZE; */

 		continue;
 	    }
 
 	    if(limits->maxfiles && (files > limits->maxfiles)) {
 		cli_dbgmsg("RAR: Files limit reached (max: %d)\n", limits->maxfiles);

 		/* ret = CL_EMAXFILES; */

 		break;
 	    }
 	}
 

         if(!(rarlist->item.FileAttr & RAR_FENTRY_ATTR_DIRECTORY)) {

             rarlist = rarlist->next;
             files++;
             continue;
         }
 

 	if((tmp = tmpfile()) == NULL) {
 	    cli_dbgmsg("RAR -> Can't generate tmpfile().\n");
 #ifdef CL_THREAD_SAFE
 	    pthread_mutex_unlock(&cli_scanrar_mutex);
 	    cli_scanrar_inuse = 0;
 #endif
 	    return CL_ETMPFILE;
 	}
 	fd = fileno(tmp);
 

 	if( urarlib_get(&rar_data_ptr, &rar_data_size, rarlist->item.Name, desc, "clam")) {

 	    cli_dbgmsg("RAR -> Extracted: %s, size: %lu\n", rarlist->item.Name, rar_data_size);

 	    if(fwrite(rar_data_ptr, 1, rar_data_size, tmp) != rar_data_size) {

 		cli_dbgmsg("RAR -> Can't write() file.\n");

 		fclose(tmp);

 		tmp = NULL;
 		ret = CL_ERAR;

 		if(rar_data_ptr) {

 		    free(rar_data_ptr);

 		    rar_data_ptr = NULL;
 		}

 		break;

 	    }
 

 	    if(rar_data_ptr) {

 		free(rar_data_ptr);

 		rar_data_ptr = NULL;
 	    }
 	    if(fflush(tmp) != 0) {
 		cli_dbgmsg("fflush() failed: %s\n", strerror(errno));
 		fclose(tmp);

 		urarlib_freelist(rarlist_head);

 #ifdef CL_THREAD_SAFE
 		pthread_mutex_unlock(&cli_scanrar_mutex);
 		cli_scanrar_inuse = 0;
 #endif
 		return CL_EFSYNC;
 	    }
 
 	    lseek(fd, 0, SEEK_SET);

 	    if((ret = cli_magic_scandesc(fd, virname, scanned, root, limits, options, arec, mrec)) == CL_VIRUS ) {

 		cli_dbgmsg("RAR -> Found %s virus.\n", *virname);

 		fclose(tmp);

 		urarlib_freelist(rarlist);
 #ifdef CL_THREAD_SAFE
 		pthread_mutex_unlock(&cli_scanrar_mutex);
 		cli_scanrar_inuse = 0;
 #endif

   		return ret;

 	    }
 
 	} else {
 	    cli_dbgmsg("RAR -> Can't decompress file %s\n", rarlist->item.Name);

 	    fclose(tmp);

 	    tmp = NULL;

 	    ret = CL_ERAR; /* WinRAR 3.0 ? */
 	    break;

 	}
 

 	fclose(tmp);

 	tmp = NULL;
 	rarlist = rarlist->next;
 	files++;
     }
 

     urarlib_freelist(rarlist_head);

 #ifdef CL_THREAD_SAFE
     pthread_mutex_unlock(&cli_scanrar_mutex);
     cli_scanrar_inuse = 0;
     pthread_cleanup_pop(0);
 #endif

     
     cli_dbgmsg("RAR -> Exit code: %d\n", ret);
 

     return ret;
 }
 
 #ifdef HAVE_ZLIB_H

 static int cli_scanzip(int desc, const char **virname, long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)

 {
 	ZZIP_DIR *zdir;
 	ZZIP_DIRENT zdirent;
 	ZZIP_FILE *zfp;

 	FILE *tmp = NULL;

 	char *buff;

 	int fd, bytes, files = 0, ret = CL_CLEAN;

 	struct stat source;

 	zzip_error_t err;

 
     cli_dbgmsg("Starting scanzip()\n");
 

     if((zdir = zzip_dir_fdopen(dup(desc), &err)) == NULL) {
 	cli_dbgmsg("Zip -> Not supported file format ?.\n");
 	cli_dbgmsg("zzip_dir_fdopen() return code: %d\n", err);

 	/* no return with CL_EZIP due to password protected zips */
 	return CL_CLEAN;

     }
 

     fstat(desc, &source);
 

     if(!(buff = (char *) cli_malloc(FILEBUFF))) {

 	cli_dbgmsg("cli_scanzip(): unable to malloc(%d)\n", FILEBUFF);

 	zzip_dir_close(zdir);
 	return CL_EMEM;
     }
 

     while(zzip_dir_read(zdir, &zdirent)) {

 
 	if(!zdirent.d_name || !strlen(zdirent.d_name)) { /* Mimail fix */
 	    cli_dbgmsg("strlen(zdirent.d_name) == %d\n", strlen(zdirent.d_name));

 	    *virname = "Suspected.Zip";

 	    ret = CL_VIRUS;
 	    break;
 	}
 

 	cli_dbgmsg("Zip -> %s, compressed: %u, normal: %u, ratio: %d (max: %d)\n", zdirent.d_name, zdirent.d_csize, zdirent.st_size, zdirent.st_size / (zdirent.d_csize+1), limits ? limits->maxratio : -1 );

 	if(!zdirent.st_size) { /* omit directories and empty files */

 	    files++;
 	    continue;
 	}
 
 	/* work-around for problematic zips (zziplib crashes with them) */

 	if(zdirent.d_csize <= 0 || zdirent.st_size < 0) {

 	    files++;
 	    cli_dbgmsg("Zip -> Malformed archive detected.\n");

 	    /* ret = CL_EMALFZIP; */
 	    /* report it as a virus */

 	    *virname = "Suspected.Zip";

 	    ret = CL_VIRUS;

 	    break;
 	}
 

 	if(limits && limits->maxratio > 0 && ((unsigned) zdirent.st_size / (unsigned) zdirent.d_csize) >= limits->maxratio) {
 	    *virname = "Oversized.Zip";
 	    ret = CL_VIRUS;
 	    break;
         }
 

 	if(DETECT_ENCRYPTED && (zdirent.d_flags & 1 )) {

 	    files++;
 	    cli_dbgmsg("Zip -> Encrypted files found in archive.\n");

 	    lseek(desc, 0, SEEK_SET);

 	    if(cli_scandesc(desc, virname, scanned, root, 0) != CL_VIRUS)

 		*virname = "Encrypted.Zip";

 	    ret = CL_VIRUS;
 	    break;
 	}
 

 	if(limits) {
 	    if(limits->maxfilesize && (zdirent.st_size > limits->maxfilesize)) {

 		cli_dbgmsg("Zip -> %s: Size exceeded (%d, max: %ld)\n", zdirent.d_name, zdirent.st_size, limits->maxfilesize);

 		files++;

 		/* ret = CL_EMAXSIZE; */

 		continue; /* this is not a bug */

 	    }
 
 	    if(limits->maxfiles && (files > limits->maxfiles)) {
 		cli_dbgmsg("Zip: Files limit reached (max: %d)\n", limits->maxfiles);

 		/* ret = CL_EMAXFILES; */

 		break;
 	    }
 	}
 
 	/* generate temporary file and get its descriptor */
 	if((tmp = tmpfile()) == NULL) {
 	    cli_dbgmsg("Zip -> Can't generate tmpfile().\n");

 	    ret = CL_ETMPFILE;
 	    break;

 	}
 
 	if((zfp = zzip_file_open(zdir, zdirent.d_name, 0)) == NULL) {
 	    cli_dbgmsg("Zip -> %s: Can't open file.\n", zdirent.d_name);
 	    ret = CL_EZIP;

 	    break;

 	}
 

 	while((bytes = zzip_file_read(zfp, buff, FILEBUFF)) > 0) {

 	    if(fwrite(buff, 1, bytes, tmp) != (size_t) bytes) {

 		cli_dbgmsg("Zip -> Can't fwrite() file: %s\n", strerror(errno));

 		zzip_file_close(zfp);

 		zzip_dir_close(zdir);
 		fclose(tmp);

 		free(buff);

 		return CL_EZIP;

 	    }
 	}
 
 	zzip_file_close(zfp);
 

 	if(fflush(tmp) != 0) {

 	    cli_dbgmsg("fflush() failed: %s\n", strerror(errno));

 	    ret = CL_EFSYNC;
 	    break;

 	}
 

 	fd = fileno(tmp);
 

 	lseek(fd, 0, SEEK_SET);

 	if((ret = cli_magic_scandesc(fd, virname, scanned, root, limits, options, arec, mrec)) == CL_VIRUS ) {

 	    cli_dbgmsg("Zip -> Found %s virus.\n", *virname);
 	    ret = CL_VIRUS;
 	    break;
 	} else if(ret == CL_EMALFZIP) {
 	    /* 

 	     * The trick with detection of ZoD only works with higher (>= 5)

 	     * recursion limit level.
 	     */
 	    cli_dbgmsg("Zip -> Malformed Zip, scanning stopped.\n");

 	    *virname = "Suspected.Zip";

 	    ret = CL_VIRUS;
 	    break;
 	}
 

 	if (tmp) {
 	    fclose(tmp);
 	    tmp = NULL;
 	}

 	files++;
     }
 
     zzip_dir_close(zdir);

     if (tmp) {
 	fclose(tmp);
 	tmp = NULL;
     }

 
     free(buff);

     return ret;
 }
 

 static int cli_scangzip(int desc, const char **virname, long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)

 {
 	int fd, bytes, ret = CL_CLEAN;
 	long int size = 0;

 	char *buff;

 	FILE *tmp = NULL;

 	gzFile gd;
 
 

     cli_dbgmsg("in cli_scangzip()\n");
 

     if((gd = gzdopen(dup(desc), "rb")) == NULL) {
 	cli_dbgmsg("Can't gzdopen() descriptor %d.\n", desc);
 	return CL_EGZIP;
     }
 
     if((tmp = tmpfile()) == NULL) {
 	cli_dbgmsg("Can't generate tmpfile().\n");
 	gzclose(gd);
 	return CL_ETMPFILE;
     }
     fd = fileno(tmp);
 

     if(!(buff = (char *) cli_malloc(FILEBUFF))) {

 	cli_dbgmsg("cli_scangzip(): unable to malloc(%d)\n", FILEBUFF);

 	gzclose(gd);

 	return CL_EMEM;

     }

     while((bytes = gzread(gd, buff, FILEBUFF)) > 0) {

 	size += bytes;
 
 	if(limits)

 	    if(limits->maxfilesize && (size + FILEBUFF > limits->maxfilesize)) {

 		cli_dbgmsg("Gzip->desc(%d): Size exceeded (stopped at %ld, max: %ld)\n", desc, size, limits->maxfilesize);

 		/* ret = CL_EMAXSIZE; */

 		break;
 	    }
 

 	if(cli_writen(fd, buff, bytes) != bytes) {

 	    cli_dbgmsg("Gzip -> Can't write() file.\n");

 	    fclose(tmp);

 	    gzclose(gd);

 	    free(buff);

 	    return CL_EGZIP;
 	}
     }
 

     free(buff);

     gzclose(gd);
     if(fsync(fd) == -1) {
 	cli_dbgmsg("fsync() failed for descriptor %d\n", fd);

 	fclose(tmp);

 	return CL_EFSYNC;
     }
 
     lseek(fd, 0, SEEK_SET);

     if((ret = cli_magic_scandesc(fd, virname, scanned, root, limits, options, arec, mrec)) == CL_VIRUS ) {

 	cli_dbgmsg("Gzip -> Found %s virus.\n", *virname);

 	fclose(tmp);

 	return CL_VIRUS;
     }

     fclose(tmp);

 
     return ret;
 }
 #endif
 
 #ifdef HAVE_BZLIB_H
 
 #ifdef NOBZ2PREFIX
 #define BZ2_bzReadOpen bzReadOpen
 #define BZ2_bzReadClose bzReadClose
 #define BZ2_bzRead bzRead
 #endif
 

 static int cli_scanbzip(int desc, const char **virname, long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)

 {
 	int fd, bytes, ret = CL_CLEAN, bzerror = 0;
 	short memlim = 0;
 	long int size = 0;

 	char *buff;

 	FILE *fs, *tmp = NULL;

 	BZFILE *bfd;
 
 

     if((fs = fdopen(dup(desc), "rb")) == NULL) {

 	cli_dbgmsg("Can't fdopen() descriptor %d.\n", desc);

 	return CL_EBZIP;
     }
 
     if(limits)
 	if(limits->archivememlim)
 	    memlim = 1;
 

     if((bfd = BZ2_bzReadOpen(&bzerror, fs, 0, memlim, NULL, 0)) == NULL) {

 	cli_dbgmsg("Can't initialize bzip2 library (descriptor %d).\n", desc);

 	fclose(fs);

 	return CL_EBZIP;
     }
 
     if((tmp = tmpfile()) == NULL) {
 	cli_dbgmsg("Can't generate tmpfile().\n");
 	BZ2_bzReadClose(&bzerror, bfd);

 	fclose(fs);

 	return CL_ETMPFILE;
     }
     fd = fileno(tmp);
 

     if(!(buff = (char *) malloc(FILEBUFF))) {
 	cli_dbgmsg("cli_scanbzip(): unable to malloc(%d)\n", FILEBUFF);
 	fclose(tmp);
 	fclose(fs);
 	BZ2_bzReadClose(&bzerror, bfd);

 	return CL_EMEM;

     }

     while((bytes = BZ2_bzRead(&bzerror, bfd, buff, FILEBUFF)) > 0) {

 	size += bytes;
 
 	if(limits)

 	    if(limits->maxfilesize && (size + FILEBUFF > limits->maxfilesize)) {

 		cli_dbgmsg("Bzip2->desc(%d): Size exceeded (stopped at %ld, max: %ld)\n", desc, size, limits->maxfilesize);

 		/* ret = CL_EMAXSIZE; */

 		break;
 	    }
 

 	if(cli_writen(fd, buff, bytes) != bytes) {

 	    cli_dbgmsg("Bzip2 -> Can't write() file.\n");
 	    BZ2_bzReadClose(&bzerror, bfd);

 	    fclose(tmp);

 	    free(buff);

 	    fclose(fs);

 	    return CL_EGZIP;
 	}
     }
 

     free(buff);

     BZ2_bzReadClose(&bzerror, bfd);
     if(fsync(fd) == -1) {
 	cli_dbgmsg("fsync() failed for descriptor %d\n", fd);

 	fclose(tmp);

 	fclose(fs);

 	return CL_EFSYNC;
     }
 
     lseek(fd, 0, SEEK_SET);

     if((ret = cli_magic_scandesc(fd, virname, scanned, root, limits, options, arec, mrec)) == CL_VIRUS ) {

 	cli_dbgmsg("Bzip2 -> Found %s virus.\n", *virname);
     }

     fclose(tmp);

     fclose(fs);

 
     return ret;
 }
 #endif
 

 static int cli_scanmscomp(int desc, const char **virname, long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)

 {
 	int fd, ret = CL_CLEAN;
 	FILE *tmp = NULL, *in;
 
     cli_dbgmsg("in cli_scanmscomp()\n");
 
     if((in = fdopen(dup(desc), "rb")) == NULL) {
 	cli_dbgmsg("Can't fdopen() descriptor %d.\n", desc);
 	return CL_EMSCOMP;
     }
 
     if((tmp = tmpfile()) == NULL) {
 	cli_dbgmsg("Can't generate tmpfile().\n");
 	fclose(in);
 	return CL_ETMPFILE;
     }
 
     if(cli_msexpand(in, tmp) == -1) {
 	cli_dbgmsg("msexpand failed.\n");
 	return CL_EMSCOMP;
     }
 
     fclose(in);
     if(fflush(tmp)) {
 	cli_dbgmsg("fflush() failed\n");
 	fclose(tmp);
 	return CL_EFSYNC;
     }
 
     fd = fileno(tmp);
     lseek(fd, 0, SEEK_SET);

     if((ret = cli_magic_scandesc(fd, virname, scanned, root, limits, options, arec, mrec)) == CL_VIRUS) {

 	cli_dbgmsg("MSCompress -> Found %s virus.\n", *virname);
 	fclose(tmp);
 	return CL_VIRUS;
     }
 
     fclose(tmp);
     return ret;
 }
 

 static int cli_scanmscab(int desc, const char **virname, long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)

 {
 	struct mscab_decompressor *cabd = NULL;
 	struct mscabd_cabinet *base, *cab;
 	struct mscabd_file *file;
 	const char *tmpdir;
 	char *tempname;
 	int ret = CL_CLEAN;
 
 
     cli_dbgmsg("in cli_scanmscab()\n");
 
     if((cabd = mspack_create_cab_decompressor(NULL)) == NULL) {
 	cli_dbgmsg("Can't create libmspack CAB decompressor\n");
 	return CL_EMSCAB;
     }
 
     if((base = cabd->dsearch(cabd, desc)) == NULL) {
 	cli_dbgmsg("I/O error or no valid cabinets found\n");
 	mspack_destroy_cab_decompressor(cabd);
 	return CL_EMSCAB;
     }
 
     if((tmpdir = getenv("TMPDIR")) == NULL)
 #ifdef P_tmpdir
 	tmpdir = P_tmpdir;
 #else
 	tmpdir = "/tmp";
 #endif
 
     for(cab = base; cab; cab = cab->next) {
 	for(file = cab->files; file; file = file->next) {

 	    tempname = cli_gentemp(tmpdir);

 	    cli_dbgmsg("Extracting data to %s\n", tempname);
 	    if(cabd->extract(cabd, file, tempname)) {
 		cli_dbgmsg("libmscab error code: %d\n", cabd->last_error(cabd));
 	    } else {

 		ret = cli_scanfile(tempname, virname, scanned, root, limits, options, arec, mrec);

 	    }
 	    if(!cli_leavetemps_flag)
 		unlink(tempname);
 	    free(tempname);
 	    if(ret == CL_VIRUS)
 		break;
 	}
 	if(ret == CL_VIRUS)
 	    break;
     }
 
     cabd->close(cabd, base);
     mspack_destroy_cab_decompressor(cabd);
 
     return ret;
 }
 

 static int cli_scanhtml(int desc, const char **virname, long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)

 {
 	unsigned char *membuff, *newbuff, *newbuff2;
 	struct stat statbuf;
 	int ret;
 

     cli_dbgmsg("in cli_scanhtml()\n");
 
     if(fstat(desc, &statbuf) != 0) {
 	cli_dbgmsg("fstat failed\n");
         return CL_EIO;
     }
 
 #ifdef HAVE_MMAP
     membuff = mmap(NULL, statbuf.st_size, PROT_READ, MAP_PRIVATE, desc, 0);
 
     /* TODO: do file operations if mmap fails */
     if(membuff == MAP_FAILED) {
 	cli_dbgmsg("mmap failed\n");
         return CL_EMEM;
     }
 

     newbuff = html_normalize(membuff, statbuf.st_size);

 
     if(newbuff) {
 	newbuff2 = remove_html_comments(newbuff);
 	free(newbuff);
 	newbuff = remove_html_char_ref(newbuff2);
 	free(newbuff2);
 	/* Normalise a second time as the above can leave inconsistent white
 	 * space
 	 */
 	newbuff2 = html_normalize(newbuff, strlen(newbuff));
 	free(newbuff);

 	newbuff = newbuff2;

     }
 

     ret = cl_scanbuff(newbuff, strlen(newbuff), virname, root);

     free(newbuff);

     return ret;

 #else /* FIXME */
     return CL_CLEAN;
 #endif

 }
 

 static int cli_scandir(const char *dirname, const char **virname, long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)

 {

 	DIR *dd;
 	struct dirent *dent;
 	struct stat statbuf;
 	char *fname;

     if((dd = opendir(dirname)) != NULL) {
 	while((dent = readdir(dd))) {

 #ifndef C_INTERIX
 	    if(dent->d_ino)
 #endif
 	    {

 		if(strcmp(dent->d_name, ".") && strcmp(dent->d_name, "..")) {
 		    /* build the full name */
 		    fname = cli_calloc(strlen(dirname) + strlen(dent->d_name) + 2, sizeof(char));
 		    sprintf(fname, "%s/%s", dirname, dent->d_name);

 		    /* stat the file */
 		    if(lstat(fname, &statbuf) != -1) {
 			if(S_ISDIR(statbuf.st_mode) && !S_ISLNK(statbuf.st_mode)) {

 			    if (cli_scandir(fname, virname, scanned, root, limits, options, arec, mrec) == CL_VIRUS) {

 				free(fname);
 				closedir(dd);
 				return CL_VIRUS;
 			    }
 			} else
 			    if(S_ISREG(statbuf.st_mode))

 				if(cli_scanfile(fname, virname, scanned, root, limits, options, arec, mrec) == CL_VIRUS) {

 				    free(fname);
 				    closedir(dd);
 				    return CL_VIRUS;
 				}

 		    }

 		    free(fname);

 		}
 	    }
 	}

     } else {

 	cli_dbgmsg("ScanDir -> Can't open directory %s.\n", dirname);

 	return CL_EOPEN;
     }

     closedir(dd);
     return 0;

 }

 static int cli_vba_scandir(const char *dirname, const char **virname, long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)

 {

 	int ret = CL_CLEAN, i, fd, data_len;
 	vba_project_t *vba_project;

 	DIR *dd;
 	struct dirent *dent;
 	struct stat statbuf;

 	char *fname, *fullname;

 	unsigned char *data;
 
     cli_dbgmsg("VBA scan dir: %s\n", dirname);
     if((vba_project = (vba_project_t *) vba56_dir_read(dirname))) {

 	for(i = 0; i < vba_project->count; i++) {
 	    fullname = (char *) cli_malloc(strlen(vba_project->dir) + strlen(vba_project->name[i]) + 2);
 	    sprintf(fullname, "%s/%s", vba_project->dir, vba_project->name[i]);
 	    fd = open(fullname, O_RDONLY);
 	    if(fd == -1) {

 		cli_dbgmsg("Scan->OLE2 -> Can't open file %s\n", fullname);

 		free(fullname);
 		ret = CL_EOPEN;
 		break;
 	    }
 	    free(fullname);
             cli_dbgmsg("decompress VBA project '%s'\n", vba_project->name[i]);
 	    data = (unsigned char *) vba_decompress(fd, vba_project->offset[i], &data_len);
 	    close(fd);
 
 	    if(!data) {
 		cli_dbgmsg("WARNING: VBA project '%s' decompressed to NULL\n", vba_project->name[i]);
 	    } else {
 		if(cl_scanbuff(data, data_len, virname, root) == CL_VIRUS) {
 		    free(data);
 		    ret = CL_VIRUS;
 		    break;
 		}
 
 		free(data);
 	    }
 	}
 
 	for(i = 0; i < vba_project->count; i++)
 	    free(vba_project->name[i]);
 	free(vba_project->name);
 	free(vba_project->dir);
 	free(vba_project->offset);
 	free(vba_project);

     } else if ((fullname = ppt_vba_read(dirname))) {

     	if(cli_scandir(fullname, virname, scanned, root, limits, options, arec, mrec) == CL_VIRUS) {

 	    ret = CL_VIRUS;
 	}
 	cli_rmdirs(fullname);
     	free(fullname);

     } else if ((vba_project = (vba_project_t *) wm_dir_read(dirname))) {
     	for (i = 0; i < vba_project->count; i++) {
 		fullname = (char *) cli_malloc(strlen(vba_project->dir) + strlen(vba_project->name[i]) + 2);
 		sprintf(fullname, "%s/%s", vba_project->dir, vba_project->name[i]);
 		fd = open(fullname, O_RDONLY);
 		if(fd == -1) {

 			cli_dbgmsg("Scan->OLE2 -> Can't open file %s\n", fullname);

 			free(fullname);
 			ret = CL_EOPEN;
 			break;
 		}
 		free(fullname);
 		cli_dbgmsg("decompress WM project '%s' macro:%d key:%d\n", vba_project->name[i], i, vba_project->key[i]);
 		data = (unsigned char *) wm_decrypt_macro(fd, vba_project->offset[i], vba_project->length[i], vba_project->key[i]);
 		close(fd);
 		
 		if(!data) {
 			cli_dbgmsg("WARNING: WM project '%s' macro %d decrypted to NULL\n", vba_project->name[i], i);
 		} else {
 			if(cl_scanbuff(data, vba_project->length[i], virname, root) == CL_VIRUS) {
 				free(data);
 				ret = CL_VIRUS;
 				break;
 			}
 			free(data);
 		}
 	}
 	for(i = 0; i < vba_project->count; i++)
 	    free(vba_project->name[i]);
 	free(vba_project->key);
 	free(vba_project->length);
 	free(vba_project->offset);
 	free(vba_project->name);
 	free(vba_project->dir);
 	free(vba_project);

     }

     if(ret != CL_CLEAN)
     	return ret;

 
     if((dd = opendir(dirname)) != NULL) {
 	while((dent = readdir(dd))) {

 #ifndef C_INTERIX
 	    if(dent->d_ino)
 #endif
 	    {

 		if(strcmp(dent->d_name, ".") && strcmp(dent->d_name, "..")) {
 		    /* build the full name */
 		    fname = cli_calloc(strlen(dirname) + strlen(dent->d_name) + 2, sizeof(char));
 		    sprintf(fname, "%s/%s", dirname, dent->d_name);
 
 		    /* stat the file */
 		    if(lstat(fname, &statbuf) != -1) {
 			if(S_ISDIR(statbuf.st_mode) && !S_ISLNK(statbuf.st_mode))

 			    if (cli_vba_scandir(fname, virname, scanned, root, limits, options, arec, mrec) == CL_VIRUS) {

 			    	ret = CL_VIRUS;
 				free(fname);
 				break;
 			    }

 		    }
 		    free(fname);
 		}
 	    }
 	}
     } else {

 	cli_dbgmsg("ScanDir -> Can't open directory %s.\n", dirname);

 	return CL_EOPEN;
     }
 
     closedir(dd);

     return ret;
 }
 

 static int cli_scanole2(int desc, const char **virname, long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)

 {
 	const char *tmpdir;

 	char *dir;
 	int ret = CL_CLEAN;

 
     cli_dbgmsg("in cli_scanole2()\n");
 

     if((tmpdir = getenv("TMPDIR")) == NULL)

 #ifdef P_tmpdir
 	tmpdir = P_tmpdir;
 #else
 	tmpdir = "/tmp";
 #endif
 
     /* generate the temporary directory */

     dir = cli_gentemp(tmpdir);

     if(mkdir(dir, 0700)) {

 	cli_dbgmsg("ScanOLE2 -> Can't create temporary directory %s\n", dir);

 	return CL_ETMPDIR;
     }
 

     if((ret = cli_ole2_extract(desc, dir, limits))) {

 	cli_dbgmsg("ScanOLE2 -> %s\n", cl_strerror(ret));

 	cli_rmdirs(dir);
 	free(dir);
 	return ret;
     }
 

     if((ret = cli_vba_scandir(dir, virname, scanned, root, limits, options, arec, mrec)) != CL_VIRUS) {
 	if(cli_scandir(dir, virname, scanned, root, limits, options, arec, mrec) == CL_VIRUS) {

 	    ret = CL_VIRUS;

 	}
     }
 
     cli_rmdirs(dir);
     free(dir);
     return ret;

 }
 

 static int cli_scanmschm(int desc, const char **virname, long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)
 {
 	const char *tmpdir;
 	char *tempname;
 	int ret = CL_CLEAN;
 
 
     cli_dbgmsg("in cli_scanmschm()\n");	
 
     if((tmpdir = getenv("TMPDIR")) == NULL)
 #ifdef P_tmpdir
 	tmpdir = P_tmpdir;
 #else
 	tmpdir = "/tmp";
 #endif
 
     tempname = cli_gentemp(tmpdir);
 
     if(mkdir(tempname, 0700)) {
 	cli_dbgmsg("ScanCHM -> Can't create temporary directory %s\n", tempname);
 	return CL_ETMPDIR;
     }
 
     if(chm_unpack(desc, tempname))
 	ret = cli_scandir(tempname, virname, scanned, root, limits, options, arec, mrec);
 
     if(!cli_leavetemps_flag)
 	cli_rmdirs(tempname);
 
     free(tempname);
     return ret;
 }
 

 static int cli_scanmail(int desc, const char **virname, long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)

 {
 	const char *tmpdir;
 	char *dir;
 	int ret;
 
 

     cli_dbgmsg("Starting cli_scanmail(), mrec == %d, arec == %d\n", *mrec, *arec);

     if((tmpdir = getenv("TMPDIR")) == NULL)

 #ifdef P_tmpdir
 	tmpdir = P_tmpdir;
 #else
 	tmpdir = "/tmp";
 #endif
 
 	/* generate the temporary directory */

 	dir = cli_gentemp(tmpdir);

 	if(mkdir(dir, 0700)) {

 	    cli_dbgmsg("ScanMail -> Can't create temporary directory %s\n", dir);

 	    return CL_ETMPDIR;
 	}
 
 	/*
 	 * Extract the attachments into the temporary directory
 	 */

 	ret = cli_mbox(dir, desc);

 	/* FIXME: check mbox return code */
 

 	ret = cli_scandir(dir, virname, scanned, root, limits, options, arec, mrec);

 
 	cli_rmdirs(dir);
 	free(dir);
 
 	return ret;
 }
 

 int cli_magic_scandesc(int desc, const char **virname, long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)

 {

 	char magic[MAGIC_BUFFER_SIZE+1];

 	int ret = CL_CLEAN, nret;

 	int bread = 0;

 	cli_file_t type;

 
 
     if(!root) {

 	cli_errmsg("CRITICAL: root == NULL\n");

 	return -1;
     }
 

     if(!options) { /* raw mode (stdin, etc.) */

 	cli_dbgmsg("Raw mode: no support for archives.\n");

 	if((ret = cli_scandesc(desc, virname, scanned, root, 0) == CL_VIRUS))

 	    cli_dbgmsg("%s virus found in descriptor %d.\n", *virname, desc);
 	return ret;
     }
 

     if(SCAN_ARCHIVE && limits && limits->maxreclevel)

 	if(*arec > limits->maxreclevel) {
 	    cli_dbgmsg("Archive recursion limit exceeded (arec == %d).\n", *arec);

 	    /* return CL_EMAXREC; */
 	    return CL_CLEAN;

 	}

     if(SCAN_MAIL)
 	if(*mrec > MAX_MAIL_RECURSION) {
 	    cli_dbgmsg("Mail recursion level exceeded (mrec == %d).\n", *mrec);
 	    /* return CL_EMAXREC; */
 	    return CL_CLEAN;
 	}

     lseek(desc, 0, SEEK_SET);
     bread = read(desc, magic, MAGIC_BUFFER_SIZE);
     magic[MAGIC_BUFFER_SIZE] = '\0';
     lseek(desc, 0, SEEK_SET);

     if(bread != MAGIC_BUFFER_SIZE) {

 	/* short read: No need to do magic */

 	if((ret = cli_scandesc(desc, virname, scanned, root, 0) == CL_VIRUS))
 	    cli_dbgmsg("%s virus found in descriptor %d.\n", *virname, desc);

 	return ret;
     }

     type = cli_filetype(magic, bread);

     type == CL_MAILFILE ? (*mrec)++ : (*arec)++;

     switch(type) {
 	case CL_RARFILE:
 	    if(!DISABLE_RAR && SCAN_ARCHIVE && !cli_scanrar_inuse)

 		ret = cli_scanrar(desc, virname, scanned, root, limits, options, arec, mrec);

 	    break;

 	case CL_ZIPFILE:
 	    if(SCAN_ARCHIVE)

 		ret = cli_scanzip(desc, virname, scanned, root, limits, options, arec, mrec);

 	    break;

 	case CL_GZFILE:
 	    if(SCAN_ARCHIVE)

 		ret = cli_scangzip(desc, virname, scanned, root, limits, options, arec, mrec);

 	    break;

 	case CL_BZFILE:

 #ifdef HAVE_BZLIB_H

 	    if(SCAN_ARCHIVE)

 		ret = cli_scanbzip(desc, virname, scanned, root, limits, options, arec, mrec);

 #endif

 	    break;

 	case CL_MSCFILE:
 	    if(SCAN_ARCHIVE)

 		ret = cli_scanmscomp(desc, virname, scanned, root, limits, options, arec, mrec);

 	    break;
 

 	case CL_MSCABFILE:
 	    if(SCAN_ARCHIVE)

 		ret = cli_scanmscab(desc, virname, scanned, root, limits, options, arec, mrec);

 	    break;
 

 	case CL_MAILFILE:
 	    if(SCAN_MAIL)

 		ret = cli_scanmail(desc, virname, scanned, root, limits, options, arec, mrec);

 	    break;

 	case CL_MSCHMFILE:
 	    if(SCAN_ARCHIVE)
 		ret = cli_scanmschm(desc, virname, scanned, root, limits, options, arec, mrec);
 	    break;
 

 	case CL_OLE2FILE:
 	    if(SCAN_OLE2)

 		ret = cli_scanole2(desc, virname, scanned, root, limits, options, arec, mrec);

 	    break;

 	case CL_DATAFILE:
 	    /* it could be a false positive and a standard DOS .COM file */
 	    {
 		struct stat s;
 		if(fstat(desc, &s) == 0 && S_ISREG(s.st_mode) && s.st_size < 65536)
 		type = CL_UNKNOWN_TYPE;
 	    }
 

 	default:

 	    break;

     }
 

     type == CL_MAILFILE ? (*mrec)-- : (*arec)--;

 
     if(type != CL_DATAFILE && ret != CL_VIRUS) { /* scan the raw file */

 	    int typerec;

 	type == CL_UNKNOWN_TYPE ? (typerec = 1) : (typerec = 0);
 	lseek(desc, 0, SEEK_SET);

 	if((nret = cli_scandesc(desc, virname, scanned, root, typerec)) == CL_VIRUS) {

 	    cli_dbgmsg("%s virus found in descriptor %d.\n", *virname, desc);
 	    return CL_VIRUS;

 
 	} else if(nret >= CL_TYPENO) {
 	    lseek(desc, 0, SEEK_SET);
 
 	    switch(nret) {
 		case CL_HTMLFILE:
 		    if(SCAN_HTML)

 			if(cli_scanhtml(desc, virname, scanned, root, limits, options, arec, mrec) == CL_VIRUS)

 			    return CL_VIRUS;
 		    break;

 
 		case CL_MAILFILE:
 		    if(SCAN_MAIL)

 			if(cli_scanmail(desc, virname, scanned, root, limits, options, arec, mrec) == CL_VIRUS)

 			    return CL_VIRUS;
 		    break;

 	    }

 	}

     }

     (*arec)++;
     lseek(desc, 0, SEEK_SET);
     switch(type) {
 	/* Due to performance reasons all executables were first scanned
 	 * in raw mode. Now we will try to unpack them
 	 */
 	case CL_DOSEXE:
 	    if(SCAN_PE)
 		ret = cli_scanpe(desc, virname, scanned, root, limits, options, arec, mrec);
 	    break;

 
 	default:
 	    break;

     }
     (*arec)--;
 
 

     return ret;
 }
 

 int cl_scandesc(int desc, const char **virname, unsigned long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options)

 {

 	int arec = 0, mrec = 0;

     return cli_magic_scandesc(desc, virname, scanned, root, limits, options, &arec, &mrec);

 }
 

 static int cli_scanfile(const char *filename, const char **virname, unsigned long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, int *arec, int *mrec)

 {
 	int fd, ret;
 

     /* internal version of cl_scanfile with arec/mrec preserved */

     if((fd = open(filename, O_RDONLY)) == -1)
 	return CL_EOPEN;
 

     ret = cli_magic_scandesc(fd, virname, scanned, root, limits, options, arec, mrec);

 
     close(fd);
     return ret;
 }
 

 int cl_scanfile(const char *filename, const char **virname, unsigned long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options)

 {
 	int fd, ret;
 

     if((fd = open(filename, O_RDONLY)) == -1)
 	return CL_EOPEN;
 
     ret = cl_scandesc(fd, virname, scanned, root, limits, options);
     close(fd);
 
     return ret;
 }