#ifdef CL_THREAD_SAFE
 #include <pthread.h>
 #endif
 

 #include "clamav.h"
 #include "others.h"
 #include "htmlnorm.h"
 #include "hashtab.h"
 #include "entconv.h"
 #include "entitylist.h"

 #include "cltypes.h"

 
 #ifdef HAVE_ICONV_H
 #include <iconv.h>
 #endif

 #include "encoding_aliases.h"
 

 
 #define MAX_LINE 1024
 

 #ifndef EILSEQ
 #define EILSEQ 84
 #endif

 
 unsigned char* entity_norm(const struct entity_conv* conv,const unsigned char* entity)
 {
 	struct element* e = hashtab_find(conv->ht,entity,strlen((const char*)entity));
 	if(e && e->key) {
 		const int val = e->data;
 		if(val == '<')/* this was an escaped <, so output it escaped*/
 			return (unsigned char*)cli_strdup("&lt;");
 		else if(val == '>')/* see above */
 			return (unsigned char*)cli_strdup("&gt;");
 		else if(val<127) {
 			unsigned char *e_out = cli_malloc(2);
 
 			if(!e_out)
 			    return NULL;
 
 			e_out[0] = (unsigned char)val;
 			e_out[1] = '\0';
 			return e_out;
 		}
 		else if(val==160)
 			return (unsigned char*)cli_strdup(" ");
 		else {
 			unsigned char *ent_out = cli_malloc(10);
 
 			if(!ent_out)
 			    return NULL;
 
 			snprintf((char*)ent_out,9,"&#%d;",val);
 			ent_out[9] = '\0';
 			return ent_out;
 		}
 	}
 	else
 		return NULL;
 }
 
 /* sane default, must be larger, than the longest possible return string,
  * which is
  * &#xxx;*/
 #define MIN_BUFFER_SIZE 32
 
 int init_entity_converter(struct entity_conv* conv,const unsigned char* encoding,size_t buffer_size)
 {
 	if(buffer_size < MIN_BUFFER_SIZE) {

 		cli_warnmsg("Entity converter: Supplied buffer size:%lu, smaller than minimum required: %d\n",(unsigned long)buffer_size,MIN_BUFFER_SIZE);

 		return CL_ENULLARG;
 	}
 	if(conv) {
 		conv->encoding = (unsigned char*) cli_strdup("ISO-8859-1");
 		conv->autodetected = OTHER;
 		conv->bom_cnt = 0;
 		conv->buffer_cnt = 0;
 		conv->bytes_read = 0;
 		conv->partial = 0;
 		conv->entity_buffcnt = 0;
 		conv->buffer_size = buffer_size;
 		conv->priority = NOPRIO;
 
 		conv->tmp_area.offset = 0;
 		conv->tmp_area.length = 0;
 		conv->tmp_area.buffer  =  cli_malloc(buffer_size);
 		if(!conv->tmp_area.buffer) {
 			return CL_EMEM;
 		}
 
 		conv->out_area.offset = 0;
 		conv->out_area.length = 0;
 		conv->out_area.buffer = cli_malloc(buffer_size);
 		if(!conv->out_area.buffer) {
 			free(conv->tmp_area.buffer);
 			return CL_EMEM;
 		}
 
 		conv->norm_area.offset = 0;
 		conv->norm_area.length = 0;
 		conv->norm_area.buffer = cli_malloc(buffer_size);
 		if(!conv->norm_area.buffer) {
 			free(conv->tmp_area.buffer);
 			free(conv->out_area.buffer);
 			return CL_EMEM;
 		}
 
 		conv->ht = &entities_htable;

 		conv->msg_zero_shown = 0;

 
 		return 0;
 	}
 	else 
 		return CL_ENULLARG;
 }
 

 static size_t encoding_bytes(const unsigned char* fromcode, enum encodings* encoding)

 {
 	const unsigned char* from = (const unsigned char*) fromcode;
 	/* special case for these unusual byteorders */

 	*encoding=E_OTHER;

 	if(from == UCS4_2143)

 		*encoding = E_UCS4_2134;

 	else if (from == UCS4_3412)

 		*encoding = E_UCS4_3412;

 	else {

 		struct element * e = hashtab_find(&aliases_htable,from,strlen((const char*)fromcode));

 		if(e && e->key) {

 			*encoding = e->data;

 		}
 	}
 

 	switch(*encoding) {

 		case E_UCS4:
 		case E_UCS4_1234:
 		case E_UCS4_4321:
 		case E_UCS4_2134:
 		case E_UCS4_3412:

 			return 4;

 		case E_UTF16:
 		case E_UTF16_BE:
 		case E_UTF16_LE:

 			return 2;

 		case E_UTF8:
 		case E_UNKNOWN:
 		case E_OTHER:
 		default:

 			return 1;

 	}

 	}
 
 #ifndef HAVE_ICONV_H
 typedef struct {
 	enum encodings encoding;
 	size_t size;
 } * iconv_t;
 
 static iconv_t iconv_open(const char *tocode, const char* fromcode)
 {
 	iconv_t iconv = cli_malloc(sizeof(*iconv));
 	if(!iconv)
 		return NULL;
 	/* TODO: check that tocode is UTF16BE */
 	iconv->size = encoding_bytes(fromcode,&iconv->encoding);

 	return iconv;
 }
 
 static int iconv_close(iconv_t cd)
 {
 	if(cd)
 		free(cd);
 	return 0;
 }
 
 
 static int iconv(iconv_t iconv_struct,char **inbuf, size_t *inbytesleft,
 		char** outbuf, size_t *outbytesleft)
 {
 	const size_t maxcopy = (*inbytesleft > *outbytesleft ? *outbytesleft  : *inbytesleft) & ~(iconv_struct->size - 1);
 	const uint8_t* input = (const uint8_t*)*inbuf;
 	uint8_t* output = (uint8_t*)*outbuf;
 	size_t i;
 
 	/*,maxcopy is aligned to data size */
 	/* output is always utf16be !*/
 	switch(iconv_struct->encoding) {
 		case E_UCS4:
 		case E_UCS4_1234:			
 			{
 				for(i=0;i < maxcopy; i += 4) {
 					if(!input[i+2] && !input[i+3]) {
 						output[i/2] = input[i+1]; /* is compiler smart enough to replace /2, with >>1 ? */
 						output[i/2+1] = input[i];
 					}
 					else {
 						cli_dbgmsg("Warning: unicode character out of utf16 range!\n");
 						output[i/2] = 0xff;
 						output[i/2+1] = 0xff;
 					}
 				}
 				break;
 			}
 		case E_UCS4_4321:
 			{
 				const uint16_t *in = (const uint16_t*)input;/*UCS4_4321, and UTF16_BE have same endianness, no need for byteswap here*/
 				uint16_t *out = (uint16_t*)output;
 				for(i=0;i<maxcopy/2; i+=2) {
 					if(!in[i]) {
 						out[i/2] = in[i+1];
 					}
 					else {
 						out[i/2] = 0xffff;
 					}
 				}
 				break;
 			}
 		case E_UCS4_2134: 
 			{
 				const uint16_t *in = (const uint16_t*)input;
 				uint16_t* out = (uint16_t*)output;
 				for(i=0;i<maxcopy/2;i+=2) {
 					if(!in[i+1])
 						out[i/2] = in[i];
 					else
 						out[i/2] = 0xffff;
 				}
 				break;
 			}
 		case E_UCS4_3412:
 			{
 				for(i=0;i < maxcopy;i += 4) {
 					if(!input[i] && !input[i+1]) {
 						output[i/2] = input[i+3];
 						output[i/2+1] = input[i+2];
 					}
 					else {
 						output[i/2] = 0xff;
 						output[i/2+1] = 0xff;
 					}
 				}
 				break;
 			}
 		case E_UTF16:
 		case E_UTF16_LE:
 			{
 				for(i=0;i < maxcopy;i += 2) {

 					output[i] = input[i+1];
 					output[i+1] = input[i];

 				}
 				break;
 			}
 		case E_UTF16_BE:

 			memcpy(output,input,maxcopy);

 			break;
 		case E_UNKNOWN:
 		case E_OTHER:
 			{
 				const size_t max_copy = *inbytesleft > (*outbytesleft/2) ? (*outbytesleft/2) : *inbytesleft;
 				for(i=0;i<max_copy;i++) {
 					output[i*2]   = 0;
 					output[i*2+1] = input[i];
 				}
 				*outbytesleft -= max_copy*2;
 				*inbytesleft  -= max_copy;
 				*inbuf += max_copy;
 				*outbuf += max_copy*2;
 				if(*inbytesleft)
 					return E2BIG;
 				return 0;
 			}
 		case E_UTF8:
 			{
 				const size_t maxread  = *inbytesleft;
 				const size_t maxwrite = *outbytesleft;
 				size_t j;
 				for(i=0,j=0 ; i < maxread && j < maxwrite;) {
 					if(input[i] < 0x7F)  {

 						output[j++] = 0;
 						output[j++] = input[i++];

 							}
 					else if( (input[i]&0xE0) == 0xC0 ) {
 						if ((input[i+1]&0xC0) == 0x80) {
 							/* 2 bytes long 110yyyyy zzzzzzzz -> 00000yyy yyzzzzzz*/

 							output[j++] = ((input[i] & 0x1F) >> 2) & 0x07;

 							output[j++] = ((input[i] & 0x1F) << 6) | (input[i+1] & 0x3F);
 						}

 						else {

 							cli_dbgmsg("invalid UTF8 character encountered\n");

 							break;
 						}

 						i+=2;
 					}
 					else if( (input[i]&0xE0) == 0xE0) {
 						if( (input[i+1]&0xC0) == 0x80 && (input[i+2]&0xC0) == 0x80) {
 							/* 3 bytes long 1110xxxx 10yyyyyy 10zzzzzzzz -> xxxxyyyy yyzzzzzz*/

 							output[j++] = (input[i] << 4) | ((input[i+1] >> 2) & 0x0F);
 							output[j++] = (input[i+1] << 6) | (input[i+2] & 0x3F);

 						}

 						else {

 							cli_dbgmsg("invalid UTF8 character encountered\n");

 							break;
 						}

 						i+=3;
 					}
 					else if( (input[i]&0xF8) == 0xF0) {
 						if((input[i+1]&0xC0) == 0x80 && (input[i+2]&0xC0) == 0x80 && (input[i+3]&0xC0) == 0x80) {
 							/* 4 bytes long 11110www 10xxxxxx 10yyyyyy 10zzzzzz -> 000wwwxx xxxxyyyy yyzzzzzz*/

 							cli_dbgmsg("UTF8 character out of UTF16 range encountered");
 							output[j++] = 0xff;
 							output[j++] = 0xff;
 
 							/*out[j++] = ((input[i] & 0x07) << 2) | ((input[i+1] >> 4) & 0x3);
 							out[j++] = (input[i+1] << 4) | ((input[i+2] >> 2) & 0x0F);
 							out[j++] = (input[i+2] << 6) | (input[i+2] & 0x3F);*/

 						}

 						else {

 							cli_dbgmsg("invalid UTF8 character encountered\n");

 							break;
 						}

 						i+=4;
 					}
 					else {
 						cli_dbgmsg("invalid UTF8 character encountered\n");

 						break;

 					}							
 				}
 				*inbytesleft -= i;
 				*outbytesleft -= j;
 				*inbuf += i;
 				*outbuf += j;

 				if(*inbytesleft && *outbytesleft) {
 					errno = EILSEQ;/* we had an early exit */
 					return -1;
 				}
 				if(*inbytesleft) {
 					errno = E2BIG;
 					return -1;
 				}

 				return 0;
 			}
 	}
 	
 	*outbytesleft -= maxcopy;
 	*inbytesleft  -= maxcopy;
 	*inbuf += maxcopy;
 	*outbuf += maxcopy;

 	if(*inbytesleft) {
 		errno = E2BIG;
 		return -1;
 	}

 	return  0;
 }
 

 #else
 
 
 

 #endif
 
 /* new iconv() version */
 static inline void process_bom(struct entity_conv* conv)
 {
 	const unsigned char* bom = conv->bom;
 	const unsigned char* encoding = OTHER;
 	int has_bom = 0;
 	uint8_t enc_bytes = 4;/* default is UTF8, which has a maximum of 4 bytes*/
 
 	switch(bom[0]) {
 		case 0x00:
 			if(bom[1] == 0x00) {
 				if(bom[2] == 0xFE && bom[3] == 0xFF) {
 					encoding = UCS4_1234;/* UCS-4 big-endian*/
 					has_bom = 1;
 				}
 				else if(bom[2] == 0xFF && bom[3] == 0xFE) {
 					encoding = UCS4_2143;/* UCS-4 unusual order 2143 */
 					has_bom = 1;
 				}
 				else if(bom[2] == 0x00 && bom[3] == 0x3C) {
 					encoding = UNDECIDED_32_1234;
 				} 
 				else if(bom[2] == 0x3C && bom[3] == 0x00) {
 					encoding = UNDECIDED_32_2143;
 				}
 			}/* 0x00 0x00 */
 			else if(bom[1] == 0x3C) {
 				if(bom[2] == 0x00) {
 					if(bom[3] == 0x00) {
 						encoding = UNDECIDED_32_3412;
 					}
 					else if(bom[3] == 0x3F) {
 						encoding = UNDECIDED_16_BE;
 						enc_bytes = 2;
 					}
 				}/*0x00 0x3C 0x00*/
 			}/*0x00 0x3C*/
 			break;
 		case 0xFF:
 			if(bom[1] == 0xFE) {
 				if(bom[2] == 0x00 && bom[3] == 0x00) {
 					encoding = UCS4_4321;
 					has_bom = 1;
 				}
 				else {
 					encoding = UTF16_LE;
 					has_bom = 1;
 					enc_bytes = 2;
 				}
 			}/*0xFF 0xFE*/
 			break;
 		case 0xFE: 
 			if(bom[1] == 0xFF) {
 					if(bom[2] == 0x00 && bom[3] == 0x00) {
 						encoding = UCS4_3412;
 						has_bom = 1;
 					}
 					else {
 						encoding = UTF16_BE;
 						has_bom = 1;
 						enc_bytes = 2;
 					}					
 			}/*0xFE 0xFF*/
 			break;
 		case 0xEF: 
 			if(bom[1] == 0xBB && bom[2] == 0xBF)  {
 					encoding = UTF8;
 					has_bom = 1;
 					/*enc_bytes = 4;- default, maximum 4 bytes*/
 			}/*0xEF 0xBB 0xBF*/				
 			break;
 		case 0x3C: 
 				if(bom[1] == 0x00) {
 					if(bom[2] == 0x00 && bom[3] == 0x00) {
 						encoding = UNDECIDED_32_4321;
 					}
 					else if(bom[2] == 0x3F && bom[3] == 0x00) {
 						encoding = UNDECIDED_16_LE;
 						enc_bytes = 2;
 					}
 				}/*0x3C 0x00*/
 				else if(bom[1] == 0x3F && bom[2] == 0x78 && bom[3]==0x6D) {
 					encoding = UNDECIDED_8;
 					enc_bytes = 1;
 				}/*0x3C 3F 78 6D*/
 				break;
 		case 0x4C: 
 				if(bom[1] == 0x6F && bom[2] == 0xA7 && bom[3] == 0x94) {
 					encoding = EBCDIC;
 					enc_bytes = 1;
 				}/*4C 6F A7 94*/
 				break;
 	}/*switch*/
 	conv->autodetected = encoding;
 	conv->enc_bytes = enc_bytes;
 	conv->has_bom = has_bom;
 }
 
 static unsigned char* normalize_encoding(const unsigned char* enc)
 {
 	unsigned char* norm; 
 	size_t i;
 	const size_t len = strlen((const char*)enc);
 	norm = cli_malloc( len+1);
 	if(!norm)
 		return NULL;
 	if(enc == OTHER)
 		enc = (const unsigned char*)"ISO-8859-1";
 	for(i=0;i < strlen((const char*)enc); i++)
 		norm[i] = toupper(enc[i]);
 	norm[len]='\0';
 	return norm;
 }
 

 static const unsigned char* encoding_name(unsigned char* encoding)

 {
 	if(!encoding)

 		return (const unsigned char*)"ISO-8859-1";

 	else

 		return encoding;

 }
 
 void process_encoding_set(struct entity_conv* conv,const unsigned char* encoding,enum encoding_priority prio)
 {

 	unsigned char *tmp_encoding;
 	enum encodings tmp;
 	size_t new_size,old_size;
 

 	cli_dbgmsg("Setting encoding for %p  to %s, priority: %d\n",(void*)conv, encoding, prio);

 	if(encoding == OTHER)
 		return;
 	if(conv->priority == CONTENT_TYPE)
 		return;/* Content-type in header is highest priority, no overrides possible*/
 	if(conv->priority ==  BOM && prio == NOBOM_AUTODETECT)
 		return;

 
 	tmp_encoding = normalize_encoding(encoding);/* FIXME: better obey priorities*/

 	if(prio == META) {

 	old_size = encoding_bytes(conv->encoding,&tmp);
 	new_size = encoding_bytes(tmp_encoding,&tmp);
 	if(old_size != new_size)  {

 		/* on x86 gcc wants %u for size_t, on x86_64 it wants %lu for size_t. So just cast to unsigned long to make warnings go away. */
 		cli_dbgmsg("process_encoding_set: refusing to override encoding - new encoding size differs: %s(%lu) != %s(%lu)\n",conv->encoding,(unsigned long)old_size,tmp_encoding,(unsigned long)new_size);

 		free(tmp_encoding);
 		return;
 	}

 	}

 	free(conv->encoding);

 	conv->encoding = tmp_encoding;

 	cli_dbgmsg("New encoding for %p:%s\n",(void*)conv,conv->encoding);

 	/* reset stream */
 }
 
 static int encoding_norm_done(struct entity_conv* conv)
 {
 	if(conv->encoding) {
 		free(conv->encoding);
 		conv->encoding = NULL;
 	}
 	conv->buffer_size = 0;
 	if(conv->tmp_area.buffer) {
 		free(conv->tmp_area.buffer);
 		conv->tmp_area.buffer = NULL;
 	}
 	if(conv->out_area.buffer) {
 		free(conv->out_area.buffer);
 		conv->out_area.buffer = NULL;
 	}
 	if(conv->norm_area.buffer) {
 		free(conv->norm_area.buffer);
 		conv->norm_area.buffer = NULL;
 	}
 	return 0;
 }
 
 int entity_norm_done(struct entity_conv* conv)
 {
 	return encoding_norm_done(conv);
 }
 

 static size_t read_raw(FILE *stream, m_area_t *m_area, int max_len, unsigned char* outbuff)

 {
 
 	/* Try and use the memory buffer first */
 	if (m_area) {
 		size_t area_maxcopy;
 		const unsigned char* src;
 		size_t copied;
 		if(m_area->offset >= m_area->length)
 			return 0;
 		area_maxcopy = (m_area->length > m_area->offset + max_len) ? max_len : m_area->length - m_area->offset;
 		src = m_area->buffer + m_area->offset;
 		m_area->offset += area_maxcopy;
 		copied = area_maxcopy;
 		while(area_maxcopy && *src != '\n') {
 			*outbuff++ = *src++;
 			area_maxcopy--;
 		}
 		if(area_maxcopy > 3) {
 			/*copy 3 more bytes, just in case its ucs4 */
 			*outbuff++ = *src++;
 			*outbuff++ = *src++;
 			*outbuff++ = *src++;
 			area_maxcopy -= 3;
 		}
 		m_area->offset -= area_maxcopy;
 		copied -= area_maxcopy;
 		return copied;
 	} else {
 		if (!stream) {
 			cli_dbgmsg("No HTML stream\n");
 			return 0;
 		}
 		else {
 			const size_t iread = fread(outbuff, 1, max_len, stream);
 			size_t i;
 			if(ferror(stream)) {
 				cli_errmsg("Error while reading HTML stream\n");
 			}
 			for(i=0; i < iread; i++)
 				if(outbuff[i] == '\n') {
 					return i+3 > iread ?  iread : i+3;
 				}
 			return iread;
 		}
 	}
 }
 

 static void output_first(struct entity_conv* conv,unsigned char** out, unsigned char** in,size_t* inleft)

 {
 	if(conv->has_bom) {
 		switch(conv->enc_bytes) {
 			case 1:

 				if(conv->autodetected == UTF8) {

 					*in += 3;

 					*inleft -= 3;
 				}

 				break;
 			case 2:
 				*in += 2;

 				*inleft -= 2;

 				break;
 			case 4:
 				*in += 4;

 				*inleft -= 4;

 				break;
 		}
 	}
 }
 

 /* sarge leaks on iconv_open/iconv_close, so lets not open/close so many times,
  * just keep on each thread its own pool of iconvs*/
 
 struct iconv_cache {
 	iconv_t* tab;
 	size_t     len;
 	size_t   last;
 	struct   hashtable hashtab;
 };
 
 static void iconv_cache_init(struct iconv_cache* cache)
 {
 /*	cache->tab = NULL;
 	cache->len = 0;
 	cache->used = 0; - already done by memset*/

 	cli_dbgmsg("Initializing iconv pool:%p\n",(void*)cache);

 	hashtab_init(&cache->hashtab, 32);
 }
 
 static void iconv_cache_destroy(struct iconv_cache* cache)
 {
 	size_t i;

 	cli_dbgmsg("Destroying iconv pool:%p\n",(void*)cache);

 	for(i=0;i < cache->last;i++) {
 		cli_dbgmsg("closing iconv:%p\n",cache->tab[i]);
 		iconv_close(cache->tab[i]);
 	}
 	hashtab_clear(&cache->hashtab);
 	free(cache->hashtab.htable);
 	free(cache->tab);
 	free(cache);
 }
 
 
 #ifdef CL_THREAD_SAFE
 static pthread_key_t iconv_pool_tls_key;
 static pthread_once_t iconv_pool_tls_key_once = PTHREAD_ONCE_INIT;
 
 /* destructor called for all threads that exit via pthread_exit, or cancellation. Unfortunately that doesn't include
  * the main thread, so we have to call this manually for the main thread.*/
 
 static int cache_atexit_registered = 0;
 
 static void iconv_pool_tls_instance_destroy(void* ptr)
 {
 	if(ptr) {
 		iconv_cache_destroy(ptr);
 	}
 }
 
 static void iconv_cache_cleanup_main(void)
 {
 	struct iconv_cache* cache = pthread_getspecific(iconv_pool_tls_key);
 	if(cache) {
 		iconv_pool_tls_instance_destroy(cache);
 		pthread_setspecific(iconv_pool_tls_key,NULL);
 	}
 	pthread_key_delete(iconv_pool_tls_key);
 }
 
 static void iconv_pool_tls_key_alloc(void)
 {
 	pthread_key_create(&iconv_pool_tls_key, iconv_pool_tls_instance_destroy);
 	if(!cache_atexit_registered) {
 		cli_dbgmsg("iconv:registering atexit\n");
 		if(atexit(iconv_cache_cleanup_main)) {
 			cli_dbgmsg("failed to register atexit\n");
 		}
 		cache_atexit_registered = 1;
 	}
 }
 
 static void init_iconv_pool_ifneeded(void)
 {
 	pthread_once(&iconv_pool_tls_key_once, iconv_pool_tls_key_alloc);
 }
 
 static inline struct iconv_cache* cache_get_tls_instance(void)
 {
 	struct iconv_cache* cache = pthread_getspecific(iconv_pool_tls_key);
 	if(!cache) {
 		cache = cli_calloc(1,sizeof(*cache));
 		if(!cache) {
 			cli_dbgmsg("!Out of memory allocating TLS iconv instance\n");
 			return NULL;
 		}
 		iconv_cache_init(cache);
 		pthread_setspecific(iconv_pool_tls_key, cache);
 	}
 	return cache;
 }
 
 #else
 
 static struct iconv_cache* global_iconv_cache = NULL;
 static int    iconv_global_inited = 0;
 
 
 static void iconv_cache_cleanup_main(void)
 {
 	iconv_cache_destroy(global_iconv_cache);
 }
 
 static inline void init_iconv_pool_ifneeded() 
 {
 	if(!iconv_global_inited) {
 		global_iconv_cache = cli_calloc(1,sizeof(*global_iconv_cache));
 		if(global_iconv_cache) {
 			iconv_cache_init(global_iconv_cache);
 			atexit(iconv_cache_cleanup_main);
 			iconv_global_inited = 1;
 		}
 	}
 }
 
 
 static inline struct iconv_cache* cache_get_tls_instance(void)
 {
 	return global_iconv_cache;
 }
 
 #endif
 
 static iconv_t iconv_open_cached(const unsigned char* fromcode)
 {
 	struct iconv_cache * cache;
 	size_t idx;
 	const size_t fromcode_len = strlen((const char*)fromcode);
 	struct element * e;

 	iconv_t  iconv_struct;

 
 	init_iconv_pool_ifneeded();
 	cache = cache_get_tls_instance();/* gets TLS iconv pool */
 	if(!cache) {
 		cli_dbgmsg("!Unable to get TLS iconv cache!\n");
 		errno = EINVAL;
 		return (iconv_t)-1;
 	}
 
 	e = hashtab_find(&cache->hashtab, fromcode, fromcode_len);
 	if(e && (e->data < 0 || (size_t)e->data > cache->len)) {
 		e = NULL;
 	}
 	if(e) {
 		return cache->tab[e->data];
 	}
 	cli_dbgmsg("iconv not found in cache, for encoding:%s\n",fromcode);

 	iconv_struct = iconv_open("UTF-16BE",(const char*)fromcode);
 	if(iconv_struct != (iconv_t)-1) {

 	idx = cache->last++;
 	if(idx >= cache->len) {
 		cache->len += 16;

 		cache->tab = cli_realloc2(cache->tab, cache->len*sizeof(cache->tab[0]));

 		if(!cache->tab) {
 			cli_dbgmsg("!Out of mem in iconv-pool\n");
 			errno = ENOMEM;
 			return (iconv_t)-1;
 		}
 	}
 
 	hashtab_insert(&cache->hashtab, fromcode, fromcode_len, idx);

 		cache->tab[idx] = iconv_struct;

 	cli_dbgmsg("iconv_open(),for:%s -> %p\n",fromcode,(void*)cache->tab[idx]);
 	return cache->tab[idx];
 }

 	return (iconv_t)-1;
 }

 /* tmp_m_area and conv->out_area are of size maxlen */
 unsigned char* encoding_norm_readline(struct entity_conv* conv, FILE* stream_in, m_area_t* in_m_area, const size_t maxlen)
 {

 	if(!conv || !conv->out_area.buffer || !conv->tmp_area.buffer || maxlen<2 )

 		return NULL;
 	else {
 		/* stream_in|in_m_area ->(read_raw) conv->tmp_area -> (iconv) conv->out_area -> (normalize) conv->norm_area -> (cli_readline) return value*/
 		const size_t tmp_move = conv->tmp_area.length - conv->tmp_area.offset;
 		const size_t tmp_available = conv->buffer_size - tmp_move;
 		const size_t max_read = maxlen < tmp_available ? maxlen : tmp_available;
 		unsigned char* tmpbuff = &conv->tmp_area.buffer[tmp_move];
 	
 		const size_t out_move = conv->out_area.length < conv->out_area.offset ? 0 : conv->out_area.length - conv->out_area.offset;
 		size_t outleft = conv->buffer_size - out_move;
 		unsigned char* out = &conv->out_area.buffer[out_move];
 
 		const size_t norm_move = conv->norm_area.length - conv->norm_area.offset;
 
 		unsigned char* norm;
 		const unsigned char* norm_end;
 		iconv_t iconv_struct;
 
 		size_t rc, inleft;
 		ssize_t i;
 

 		signed char alignfix;

 
 		/* move whatever left in conv->tmp_area to beginning */
 		if(tmp_move)
 			memmove(conv->tmp_area.buffer, conv->tmp_area.buffer + conv->tmp_area.offset, tmp_move);
 		conv->tmp_area.offset = 0;
 
 		/* read raw data from stream, or in_m_area into conv->tmp_area*/
 		conv->tmp_area.length = tmp_move + read_raw(stream_in, in_m_area, max_read, tmpbuff);
 
 		/* move whatever left in conv->out_area to beginning */
 		if(out_move)
 			memmove(conv->out_area.buffer, conv->out_area.buffer + conv->out_area.offset, out_move);
 		conv->out_area.offset = 0;
 
 		tmpbuff = conv->tmp_area.buffer;

 		inleft = conv->tmp_area.length;

 		if(!conv->bom_cnt && conv->tmp_area.length >= 4) {/* detect Byte Order Mark */
 			memcpy( conv->bom, tmpbuff, 4);
 			process_bom(conv);
 			process_encoding_set(conv,conv->autodetected,conv->has_bom ? BOM : NOBOM_AUTODETECT);

 			output_first(conv,&out,&tmpbuff,&inleft);

 			conv->bom_cnt++;
 		}
 
 		/* convert encoding conv->tmp_area. conv->out_area */
 		alignfix = inleft%4;/* iconv gives an error if we give him 3 bytes to convert, 
 				       and we are using ucs4, ditto for utf16, and 1 byte*/
 		inleft -= alignfix;
 
 		if(!inleft && alignfix) {
 			size_t k;
 			for(k=0;k+alignfix < 4;k++)
 				tmpbuff[alignfix+k] = '\0';
 			inleft = 4;
 			alignfix = -inleft;
 		}
 

 		iconv_struct = iconv_open_cached(encoding_name(conv->encoding));

 
 		if(iconv_struct == (iconv_t)-1) {
 			cli_dbgmsg("Iconv init problem for encoding:%s, falling back to iso encoding!\n",encoding_name(conv->encoding));

 			/* message shown only once/file */

 			/* what can we do? just fall back for it being an ISO-8859-1 */

 		        free(conv->encoding);
 			conv->encoding = (unsigned char*) cli_strdup("ISO-8859-1");
 			iconv_struct = iconv_open_cached(conv->encoding);

 			if(iconv_struct == (iconv_t)-1) {
 				cli_dbgmsg("fallback failed... bail out\n");
 				return cli_readline(NULL,&conv->tmp_area,maxlen);
 			}
 		}
 

 		if(inleft && outleft > conv->buffer_size/2 ) /* iconv doesn't like inleft to be 0 */ {

 			rc = iconv(iconv_struct, (char**) &tmpbuff,  &inleft, (char**) &out, &outleft);	

 		}

 		else
 			rc = 0;
 

 #if 0
 		 iconv_close(iconv_struct);/* - don't close, we are using a cached instance */
 #endif

 
 		if(rc==(size_t)-1 && errno != E2BIG) {

 				cli_dbgmsg("iconv error:%s, silently resuming (%ld,%ld,%lu,%lu)\n",strerror(errno),(long)(out-conv->out_area.buffer),(long)(tmpbuff-conv->tmp_area.buffer),(unsigned long)inleft,(unsigned long)outleft);

 				/* output raw byte, and resume at next byte */
 				*out++ = 0;
 				*out++ = *tmpbuff++;
 				inleft--;
 /*				return cli_readline(NULL, &conv->norm_area, maxlen);*/

 		}
 
 		conv->tmp_area.length = inleft + (alignfix > 0 ? alignfix : 0);

 		conv->out_area.length = out - conv->out_area.buffer - out_move;

 
 		conv->tmp_area.offset = tmpbuff - conv->tmp_area.buffer;
 		conv->tmp_area.length += conv->tmp_area.offset;
 
 
 		/* move whatever left in conv->norm_area to beginning */

 		if(norm_move) {
 			if(norm_move < conv->buffer_size/2) {

 			memmove(conv->norm_area.buffer, conv->norm_area.buffer + conv->norm_area.offset, norm_move);
 		conv->norm_area.offset = 0;

 				norm = conv->norm_area.buffer + norm_move;
 			}
 			else {
 				/* don't modify offset here */
 				norm = conv->norm_area.buffer + conv->norm_area.length;
 			}
 		}
 		else {
 			conv->norm_area.offset = 0;
 			norm = conv->norm_area.buffer;	
 		}

 
 		/* now do the real normalization */
 		out = conv->out_area.buffer;/* skip over utf16 bom, FIXME: check if iconv really outputted a BOM */
 		norm_end = conv->norm_area.buffer + conv->buffer_size;
 		if(conv->out_area.length>0 && out[0] == 0xFF && out[1] == 0xFE)
 			i = 2;
 		else
 			i = 0;
 		for(; i < conv->out_area.length; i += 2) {
 			uint16_t u16 = ( ((uint16_t)out[i]) << 8 ) | out[i+1];
 			if(!u16) {

 				if(alignfix >= 0 && !conv->msg_zero_shown) /* if alignfix is negative, this 0 byte is on-purpose, its padding */ {
 					conv->msg_zero_shown = 1;

 					cli_dbgmsg("Skipping null character in html stream\n");
 			}

 			}

 			else if(u16 < 0x80) {

 				if(norm >= norm_end)
 					break;
 				if((unsigned char)u16 ==0)
 					cli_dbgmsg("Impossible\n");
 				*norm++ = (unsigned char)u16;
 			}
 			else if (u16 == 160)  {/*nbsp*/

 				if(norm >= norm_end)
 					break;

 				*norm++ = 0x20;
 			}
 			else {
 				char buff[10];

 				int len;

 				snprintf(buff,9,"&#%d;",u16);
 				buff[9] = '\0';

 				len = strlen(buff);

 				if((norm_end - norm) <= len)
 					/* prevent buffer overflow */

 					break;

 				memcpy((char*)norm, buff, len);
 				norm += len;

 			}	
 		}
 		conv->out_area.offset = i; /* so that we can resume next time from here */

 		conv->norm_area.length = norm - conv->norm_area.buffer;
 /*
 		conv->norm_area.buffer[conv->buffer_size-1]=0;DONT DO THIS
 		if( (o =strstr(conv->norm_area.buffer,"Content")) && strstr(conv->norm_area.buffer,"text/x-"))
 			printf("%s\n",o);*/
 		/* final cli_readline from conv->norm_area */
 		return cli_readline(NULL, &conv->norm_area, maxlen);
 	}
 }