@@ -1,3 +1,7 @@

+Tue Jul 28 20:19:08 CEST 2009 (tk)

+----------------------------------

+ * libclamav: improve loading speed of compressed databases (bb#1105)

+

 Mon Jul 27 13:53:15 CEST 2009 (tk)

 ----------------------------------

  * libclamav/macho.c: improve detection of Universal Binaries

@@ -1,5 +1,5 @@

 /*

- *  Copyright (C) 2007-2008 Sourcefire, Inc.

+ *  Copyright (C) 2007-2009 Sourcefire, Inc.

  *

  *  Authors: Tomasz Kojm

  *

@@ -41,6 +41,7 @@

 #include "str.h"

 #include "cvd.h"

 #include "readdb.h"

+#include "default.h"

 #define TAR_BLOCKSIZE 512

@@ -227,6 +228,17 @@ static int cli_tgzload(int fd, struct cl_engine *engine, unsigned int *signo, un

 	dbio.gzs = NULL;

     }

+    dbio.bufsize = CLI_DEFAULT_DBIO_BUFSIZE;

+    dbio.buf = cli_malloc(dbio.bufsize);

+    if(!dbio.buf) {

+	cli_errmsg("cli_tgzload: Can't allocate memory for dbio.buf\n");

+	CLOSE_DBIO;

+	return CL_EMALFDB;

+    }

+    dbio.bufpt = NULL;

+    dbio.usebuf = 1;

+    dbio.readpt = dbio.buf;

+

     while(1) {

 	if(compr)

@@ -239,6 +251,7 @@ static int cli_tgzload(int fd, struct cl_engine *engine, unsigned int *signo, un

 	if(nread != TAR_BLOCKSIZE) {

 	    cli_errmsg("cli_tgzload: Incomplete block read\n");

+	    free(dbio.buf);

 	    CLOSE_DBIO;

 	    return CL_EMALFDB;

 	}

@@ -251,6 +264,7 @@ static int cli_tgzload(int fd, struct cl_engine *engine, unsigned int *signo, un

 	if(strchr(name, '/')) {

 	    cli_errmsg("cli_tgzload: Slash separators are not allowed in CVD\n");

+	    free(dbio.buf);

 	    CLOSE_DBIO;

 	    return CL_EMALFDB;

 	}

@@ -263,10 +277,12 @@ static int cli_tgzload(int fd, struct cl_engine *engine, unsigned int *signo, un

 		break;

 	    case '5':

 		cli_errmsg("cli_tgzload: Directories are not supported in CVD\n");

+		free(dbio.buf);

 		CLOSE_DBIO;

 		return CL_EMALFDB;

 	    default:

 		cli_errmsg("cli_tgzload: Unknown type flag '%c'\n", type);

+		free(dbio.buf);

 		CLOSE_DBIO;

 		return CL_EMALFDB;

 	}

@@ -276,10 +292,14 @@ static int cli_tgzload(int fd, struct cl_engine *engine, unsigned int *signo, un

 	if((sscanf(osize, "%o", &size)) == 0) {

 	    cli_errmsg("cli_tgzload: Invalid size in header\n");

+	    free(dbio.buf);

 	    CLOSE_DBIO;

 	    return CL_EMALFDB;

 	}

 	dbio.size = size;

+	dbio.readsize = dbio.size < dbio.bufsize ? dbio.size : dbio.bufsize - 1;

+	dbio.bufpt = NULL;

+	dbio.readpt = dbio.buf;

 	/* cli_dbgmsg("cli_tgzload: Loading %s, size: %u\n", name, size); */

 	if(compr)

@@ -291,6 +311,7 @@ static int cli_tgzload(int fd, struct cl_engine *engine, unsigned int *signo, un

 	    ret = cli_load(name, engine, signo, options, &dbio);

 	    if(ret) {

 		cli_errmsg("cli_tgzload: Can't load %s\n", name);

+		free(dbio.buf);

 		CLOSE_DBIO;

 		return CL_EMALFDB;

 	    }

@@ -309,6 +330,7 @@ static int cli_tgzload(int fd, struct cl_engine *engine, unsigned int *signo, un

 	}

     }

+    free(dbio.buf);

     CLOSE_DBIO;

     return CL_SUCCESS;

 }

@@ -1,5 +1,5 @@

 /*

- *  Copyright (C) 2007-2008 Sourcefire, Inc.

+ *  Copyright (C) 2007-2009 Sourcefire, Inc.

  *

  *  Authors: Tomasz Kojm

  *

@@ -29,6 +29,9 @@ struct cli_dbio {

     gzFile *gzs;

     FILE *fs;

     unsigned int size;

+    char *buf, *bufpt, *readpt;

+    unsigned int usebuf, bufsize, readsize;

+

 };

 int cli_cvdload(FILE *fs, struct cl_engine *engine, unsigned int *signo, unsigned int daily, unsigned int options, unsigned int cld);

@@ -26,6 +26,7 @@

 #define CLI_DEFAULT_AC_TRACKLEN	    8

 #define CLI_DEFAULT_LSIG_BUFSIZE    32768

+#define CLI_DEFAULT_DBIO_BUFSIZE    CLI_DEFAULT_LSIG_BUFSIZE + 1

 #define CLI_DEFAULT_MAXSCANSIZE	    104857600

 #define CLI_DEFAULT_MAXFILESIZE	    26214400

@@ -1,5 +1,5 @@

 /*

- *  Copyright (C) 2007-2008 Sourcefire, Inc.

+ *  Copyright (C) 2007-2009 Sourcefire, Inc.

  *

  *  Authors: Tomasz Kojm

  *

@@ -340,10 +340,68 @@ static int cli_initroots(struct cl_engine *engine, unsigned int options)

 char *cli_dbgets(char *buff, unsigned int size, FILE *fs, struct cli_dbio *dbio)

 {

-    if(fs) {

+    if(fs)

 	return fgets(buff, size, fs);

-    } else {

+    if(dbio->usebuf) {

+	    int bread;

+	    char *nl;

+

+	while(1) {

+	    if(!dbio->bufpt) {

+		if(!dbio->size)

+		    return NULL;

+

+		if(dbio->gzs) {

+		    bread = gzread(dbio->gzs, dbio->readpt, dbio->readsize);

+		    if(bread == -1) {

+			cli_errmsg("cli_dbgets: gzread() failed\n");

+			return NULL;

+		    }

+		} else {

+		    bread = fread(dbio->readpt, 1, dbio->readsize, dbio->fs);

+		    if(!bread && ferror(dbio->fs)) {

+			cli_errmsg("cli_dbgets: gzread() failed\n");

+			return NULL;

+		    }

+		}

+		if(!bread)

+		    return NULL;

+		dbio->readpt[bread] = 0;

+		dbio->bufpt = dbio->buf;

+		dbio->size -= bread;

+	    }

+	    nl = strchr(dbio->bufpt, '\n');

+	    if(nl) {

+		if(nl - dbio->bufpt >= size) {

+		    cli_errmsg("cli_dbgets: Line too long for provided buffer\n");

+		    return NULL;

+		}

+		strncpy(buff, dbio->bufpt, nl - dbio->bufpt);

+		buff[nl - dbio->bufpt] = 0;

+		if(nl < dbio->buf + dbio->bufsize) {

+		    dbio->bufpt = ++nl;

+		} else {

+		    dbio->bufpt = NULL;

+		    dbio->readpt = dbio->buf;

+		    dbio->readsize = dbio->size < dbio->bufsize ? dbio->size : dbio->bufsize - 1;

+		}

+		return buff;

+	    } else {

+		    unsigned int remain = dbio->buf + dbio->bufsize - 1 - dbio->bufpt;

+

+		if(dbio->bufpt == dbio->buf) {

+		    cli_errmsg("cli_dbgets: Invalid data or internal buffer too small\n");

+		    return NULL;

+		}

+		memmove(dbio->buf, dbio->bufpt, remain);

+		dbio->readpt = dbio->buf + remain;

+		dbio->readsize = dbio->bufsize - remain;

+		dbio->readsize = dbio->size < dbio->bufsize - remain ? dbio->size : dbio->bufsize - remain - 1;

+		dbio->bufpt = NULL;

+	    }

+	}

+    } else { /* use gzgets/fgets */

 	    char *pt;

 	    unsigned int bs;