@@ -1,3 +1,7 @@

+Fri Nov 8 17:08:09 2013 EDT 2013 (morgan)

+------------------------------------

+ * Add ForceToDisk option for clamd and force-to-disk arg for clamscan

+

 Wed Oct 31 12:48:00 2013 EDT 2013 (morgan)

 ------------------------------------

  * libclamav: bb#5341 - Change floating point byte order check from compile time to run time.

@@ -431,6 +431,9 @@ int main(int argc, char **argv)

     if(optget(opts, "LeaveTemporaryFiles")->enabled)

 	cl_engine_set_num(engine, CL_ENGINE_KEEPTMP, 1);

+    if(optget(opts, "ForceToDisk")->enabled)

+	cl_engine_set_num(engine, CL_ENGINE_FORCETODISK, 1);

+

     if(optget(opts, "PhishingSignatures")->enabled)

 	dboptions |= CL_DB_PHISHING;

     else

@@ -631,6 +631,9 @@ int scanmanager(const struct optstruct *opts)

     if(optget(opts, "leave-temps")->enabled)

 	cl_engine_set_num(engine, CL_ENGINE_KEEPTMP, 1);

+    if(optget(opts, "force-to-disk")->enabled)

+	cl_engine_set_num(engine, CL_ENGINE_FORCETODISK, 1);

+

     if(optget(opts, "bytecode-unsigned")->enabled)

 	dboptions |= CL_DB_BYTECODE_UNSIGNED;

@@ -241,6 +241,10 @@ Example

 # Default: yes

 #AlgorithmicDetection yes

+# This option causes memory or nested map scans to dump the content to disk.

+# If you turn on this option, more data is written to disk and is available

+# when the LeaveTemporaryFiles option is enabled.

+#ForceToDisk yes

 ##

 ## Executable files

@@ -193,7 +193,8 @@ enum cl_engine_field {

     CL_ENGINE_MAX_HTMLNORMALIZE,    /* uint64_t */

     CL_ENGINE_MAX_HTMLNOTAGS,       /* uint64_t */

     CL_ENGINE_MAX_SCRIPTNORMALIZE,  /* uint64_t */

-    CL_ENGINE_MAX_ZIPTYPERCG        /* uint64_t */

+    CL_ENGINE_MAX_ZIPTYPERCG,       /* uint64_t */

+    CL_ENGINE_FORCETODISK           /* uint32_t */

 };

 enum bytecode_security {

@@ -157,7 +157,7 @@ int cli_scancpio_old(cli_ctx *ctx)

 	    if(ret == CL_EMAXFILES) {

 		return ret;

 	    } else if(ret == CL_SUCCESS) {

-		ret = cli_map_scandesc(*ctx->fmap, pos, filesize, ctx);

+		ret = cli_map_scan(*ctx->fmap, pos, filesize, ctx);

 		if(ret == CL_VIRUS)

 		    return ret;

 	    }

@@ -234,7 +234,7 @@ int cli_scancpio_odc(cli_ctx *ctx)

 	if(ret == CL_EMAXFILES) {

 	    return ret;

 	} else if(ret == CL_SUCCESS) {

-	    ret = cli_map_scandesc(*ctx->fmap, pos, filesize, ctx);

+	    ret = cli_map_scan(*ctx->fmap, pos, filesize, ctx);

 	    if(ret == CL_VIRUS)

 		return ret;

 	}

@@ -313,7 +313,7 @@ int cli_scancpio_newc(cli_ctx *ctx, int crc)

 	if(ret == CL_EMAXFILES) {

 	    return ret;

 	} else if(ret == CL_SUCCESS) {

-	    ret = cli_map_scandesc(*ctx->fmap, pos, filesize, ctx);

+	    ret = cli_map_scan(*ctx->fmap, pos, filesize, ctx);

 	    if(ret == CL_VIRUS)

 		return ret;

 	}

@@ -168,7 +168,7 @@ int cli_scandmg(cli_ctx *ctx)

     cli_dbgmsg("cli_scandmg: Extracting into %s\n", dirname);

     /* Dump XML to tempfile, if needed */

-    if (ctx->engine->keeptmp) {

+    if (ctx->engine->keeptmp && !ctx->engine->forcetodisk) {

         int xret;

         xret = dmg_extract_xml(ctx, dirname, &hdr);

@@ -180,7 +180,7 @@ int cli_scandmg(cli_ctx *ctx)

     }

     /* scan XML with cli_map_scandesc */

-    ret = cli_map_scandesc(*ctx->fmap, (off_t)hdr.xmlOffset, (size_t)hdr.xmlLength, ctx);

+    ret = cli_map_scan(*ctx->fmap, (off_t)hdr.xmlOffset, (size_t)hdr.xmlLength, ctx);

     if (ret != CL_CLEAN) {

         cli_dbgmsg("cli_scandmg: retcode from scanning TOC xml: %s\n", cl_strerror(ret));

         if (!ctx->engine->keeptmp)

@@ -553,7 +553,7 @@ int cli_scanmacho_unibin(cli_ctx *ctx)

 	cli_dbgmsg("UNIBIN: Binary %u of %u\n", i + 1, fat_header.nfats);

 	cli_dbgmsg("UNIBIN: File offset: %u\n", fat_arch.offset);

 	cli_dbgmsg("UNIBIN: File size: %u\n", fat_arch.size);

-	ret = cli_map_scandesc(map, fat_arch.offset, fat_arch.size, ctx);

+	ret = cli_map_scan(map, fat_arch.offset, fat_arch.size, ctx);

 	if(ret == CL_VIRUS)

 	    break;

     }

@@ -461,6 +461,12 @@ int cl_engine_set_num(struct cl_engine *engine, enum cl_engine_field field, long

 	case CL_ENGINE_KEEPTMP:

 	    engine->keeptmp = num;

 	    break;

+	case CL_ENGINE_FORCETODISK:

+	    if(num)

+	        engine->forcetodisk = 1;

+	    else

+	        engine->forcetodisk = 0;

+	    break;

 	case CL_ENGINE_BYTECODE_SECURITY:

 	    if (engine->dboptions & CL_DB_COMPILED) {

 		cli_errmsg("cl_engine_set_num: CL_ENGINE_BYTECODE_SECURITY cannot be set after engine was compiled\n");

@@ -541,6 +547,8 @@ long long cl_engine_get_num(const struct cl_engine *engine, enum cl_engine_field

 	    return engine->ac_maxdepth;

 	case CL_ENGINE_KEEPTMP:

 	    return engine->keeptmp;

+	case CL_ENGINE_FORCETODISK:

+	    return engine->forcetodisk;

 	case CL_ENGINE_BYTECODE_SECURITY:

 	    return engine->bytecode_security;

 	case CL_ENGINE_BYTECODE_TIMEOUT:

@@ -619,6 +627,7 @@ struct cl_settings *cl_engine_settings_copy(const struct cl_engine *engine)

     settings->ac_maxdepth = engine->ac_maxdepth;

     settings->tmpdir = engine->tmpdir ? strdup(engine->tmpdir) : NULL;

     settings->keeptmp = engine->keeptmp;

+    settings->forcetodisk = engine->forcetodisk;

     settings->maxscansize = engine->maxscansize;

     settings->maxfilesize = engine->maxfilesize;

     settings->maxreclevel = engine->maxreclevel;

@@ -652,6 +661,7 @@ int cl_engine_settings_apply(struct cl_engine *engine, const struct cl_settings

     engine->ac_mindepth = settings->ac_mindepth;

     engine->ac_maxdepth = settings->ac_maxdepth;

     engine->keeptmp = settings->keeptmp;

+    engine->forcetodisk = settings->forcetodisk;

     engine->maxscansize = settings->maxscansize;

     engine->maxfilesize = settings->maxfilesize;

     engine->maxreclevel = settings->maxreclevel;

@@ -55,7 +55,7 @@

  * in re-enabling affected modules.

  */

-#define CL_FLEVEL 75

+#define CL_FLEVEL 77

 #define CL_FLEVEL_DCONF	CL_FLEVEL

 #define CL_FLEVEL_SIGTOOL CL_FLEVEL

@@ -283,6 +283,8 @@ struct cl_engine {

     uint64_t maxhtmlnotags; /* max size for scanning normalized HTML */

     uint64_t maxscriptnormalize; /* max size to normalize scripts */

     uint64_t maxziptypercg; /* max size to re-do zip filetype */

+

+    uint32_t forcetodisk; /* cause memory or map scans to dump to disk first */

 };

 struct cl_settings {

@@ -322,6 +324,8 @@ struct cl_settings {

     uint64_t maxhtmlnotags; /* max size for scanning normalized HTML */

     uint64_t maxscriptnormalize; /* max size to normalize scripts */

     uint64_t maxziptypercg; /* max size to re-do zip filetype */

+

+    uint32_t forcetodisk; /* cause memory or map scans to dump to disk first */

 };

 extern int (*cli_unrar_open)(int fd, const char *dirname, unrar_state_t *state);

@@ -2964,7 +2964,67 @@ int cl_scandesc(int desc, const char **virname, unsigned long int *scanned, cons

     return cl_scandesc_callback(desc, virname, scanned, engine, scanoptions, NULL);

 }

-/* length = 0, till the end */

+/* For map scans that may be forced to disk */

+int cli_map_scan(cl_fmap_t *map, off_t offset, size_t length, cli_ctx *ctx)

+{

+    off_t old_off = map->nested_offset;

+    size_t old_len = map->len;

+    int ret = CL_CLEAN;

+

+    cli_dbgmsg("cli_map_scan: [%ld, +%lu)\n",

+	       (long)offset, (unsigned long)length);

+    if (offset < 0 || offset >= map->len) {

+	cli_dbgmsg("Invalid offset: %ld\n", (long)offset);

+	return CL_CLEAN;

+    }

+

+    if (ctx->engine->forcetodisk) {

+        /* if this is forced to disk, then need to write the nested map and scan it */

+        const uint8_t *mapdata = NULL;

+        char *tempfile = NULL;

+        int fd = -1;

+        size_t nread = 0;

+

+        mapdata = fmap_need_off_once_len(map, offset, length, &nread);

+        if (!mapdata || (nread != length)) {

+            cli_errmsg("cli_map_scan: could not map sub-file\n");

+            return CL_EMAP;

+        }

+

+        ret = cli_gentempfd(ctx->engine->tmpdir, &tempfile, &fd);

+        if (ret != CL_SUCCESS) {

+            return ret;

+        }

+

+        cli_dbgmsg("cli_map_scan: writing nested map content to temp file %s\n", tempfile);

+        if (cli_writen(fd, mapdata, length) < 0) {

+            cli_errmsg("cli_map_scan: cli_writen error writing subdoc temporary file.\n");

+            ret = CL_EWRITE;

+        }

+

+        /* scan the temp file */

+        ret = cli_base_scandesc(fd, ctx, CL_TYPE_ANY);

+

+        /* remove the temp file, if needed */

+        if (fd > -1) {

+            close(fd);

+        }

+        if(!ctx->engine->keeptmp) {

+            if (cli_unlink(tempfile)) {

+                cli_errmsg("cli_map_scan: error unlinking tempfile %s\n", tempfile);

+                ret = CL_EUNLINK;

+            }

+        }

+        free(tempfile);

+    }

+    else {

+        /* Not forced to disk, use nested map */

+        ret = cli_map_scandesc(map, offset, length, ctx);

+    }

+    return ret;

+}

+

+/* For map scans that are not forced to disk */

 int cli_map_scandesc(cl_fmap_t *map, off_t offset, size_t length, cli_ctx *ctx)

 {

     off_t old_off = map->nested_offset;

@@ -29,6 +29,7 @@ int cli_magic_scandesc(int desc, cli_ctx *ctx);

 int cli_partition_scandesc(int desc, cli_ctx *ctx);

 int cli_magic_scandesc_type(cli_ctx *ctx, cli_file_t type);

 int cli_map_scandesc(cl_fmap_t *map, off_t offset, size_t length, cli_ctx *ctx);

+int cli_map_scan(cl_fmap_t *map, off_t offset, size_t length, cli_ctx *ctx);

 int cli_mem_scandesc(const void *buffer, size_t length, cli_ctx *ctx);

 int cli_found_possibly_unwanted(cli_ctx* ctx);

@@ -338,6 +338,8 @@ const struct clam_option __clam_options[] = {

     { "ArchiveBlockEncrypted", "block-encrypted", 0, TYPE_BOOL, MATCH_BOOL, 0, NULL, 0, OPT_CLAMD | OPT_CLAMSCAN, "Mark encrypted archives as viruses (Encrypted.Zip, Encrypted.RAR).", "no" },

+    { "ForceToDisk", "force-to-disk", 0, TYPE_BOOL, MATCH_BOOL, 0, NULL, 0, OPT_CLAMD | OPT_CLAMSCAN, "This option causes memory or nested map scans to dump the content to disk.\nIf you turn on this option, more data is written to disk and is available\nwhen the leave-temps option is enabled at the cost of more disk writes.", "no" },

+

     { "MaxScanSize", "max-scansize", 0, TYPE_SIZE, MATCH_SIZE, CLI_DEFAULT_MAXSCANSIZE, NULL, 0, OPT_CLAMD | OPT_CLAMSCAN, "This option sets the maximum amount of data to be scanned for each input file.\nArchives and other containers are recursively extracted and scanned up to this\nvalue.\nThe value of 0 disables the limit.\nWARNING: disabling this limit or setting it too high may result in severe\ndamage.", "100M" },

     { "MaxFileSize", "max-filesize", 0, TYPE_SIZE, MATCH_SIZE, CLI_DEFAULT_MAXFILESIZE, NULL, 0, OPT_CLAMD | OPT_MILTER | OPT_CLAMSCAN, "Files/messages larger than this limit won't be scanned. Affects the input\nfile itself as well as files contained inside it (when the input file is\nan archive, a document or some other kind of container).\nThe value of 0 disables the limit.\nWARNING: disabling this limit or setting it too high may result in severe\ndamage to the system.", "25M" },