@@ -72,7 +72,6 @@

 #include "lzma_iface.h"

 #include "egg.h"

-#include "msdoc.h"

 #include "entconv.h"

 #ifndef WCHAR

@@ -619,9 +618,9 @@ static cl_error_t egg_parse_comment_header(const uint8_t* index, size_t size, ex

     if (extraField->bit_flag & COMMENT_HEADER_FLAGS_MULTIBYTE_CODEPAGE_INSTEAD_OF_UTF8) {

         /*

          * Unlike with filenames, the multibyte string codepage (or "locale") is not present in comment headers.

-         * Try conversion with codepage 65001.

+         * Try conversion with CODEPAGE_UTF8.

          */

-        if (CL_SUCCESS != cli_codepage_to_utf8((char*)index, size, 65001, &comment_utf8, &comment_utf8_size)) {

+        if (CL_SUCCESS != cli_codepage_to_utf8((char*)index, size, CODEPAGE_UTF8, &comment_utf8, &comment_utf8_size)) {

             cli_dbgmsg("egg_parse_comment_header: failed to convert codepage \"0\" to UTF-8\n");

             comment_utf8 = cli_genfname(NULL);

         }

@@ -1163,7 +1162,7 @@ static cl_error_t egg_parse_file_extra_field(egg_handle* handle, egg_file* eggFi

                  * - 949 (Korean Unified Code)

                  * - 932 (Japanese Shift-JIS) */

                 if (0 == codepage) {

-                    if (CL_SUCCESS != cli_codepage_to_utf8((char*)index, name_size, 65001, &name_utf8, &name_utf8_size)) {

+                    if (CL_SUCCESS != cli_codepage_to_utf8((char*)index, name_size, CODEPAGE_UTF8, &name_utf8, &name_utf8_size)) {

                         cli_dbgmsg("egg_parse_file_extra_field: failed to convert codepage \"0\" to UTF-8\n");

                         name_utf8 = cli_genfname(NULL);

                     }

@@ -780,7 +780,7 @@ cl_error_t cli_codepage_to_utf8(char* in, size_t in_size, uint16_t codepage, cha

     size_t out_utf8_size = 0;

 #if defined(HAVE_ICONV)

-    iconv_t conv = (iconv_t) -1;

+    iconv_t conv = (iconv_t)-1;

 #elif defined(WIN32)

     LPWSTR lpWideCharStr = NULL;

     int cchWideChar      = 0;

@@ -796,8 +796,8 @@ cl_error_t cli_codepage_to_utf8(char* in, size_t in_size, uint16_t codepage, cha

     *out_size = 0;

     switch (codepage) {

-        case 20127:   /* US-ASCII (7-bit) */

-        case 65001: { /* Unicode (UTF-8) */

+        case CODEPAGE_US_7BIT_ASCII: /* US-ASCII (7-bit) */

+        case CODEPAGE_UTF8: {        /* Unicode (UTF-8) */

             char* track;

             int byte_count, sigbit_count;

@@ -811,7 +811,7 @@ cl_error_t cli_codepage_to_utf8(char* in, size_t in_size, uint16_t codepage, cha

             memcpy(out_utf8, in, in_size);

             track = out_utf8 + in_size - 1;

-            if ((codepage == 65001) && (*track & 0x80)) {

+            if ((codepage == CODEPAGE_UTF8) && (*track & 0x80)) {

                 /*

                  * UTF-8 with a most significant bit.

                  */

@@ -847,7 +847,7 @@ cl_error_t cli_codepage_to_utf8(char* in, size_t in_size, uint16_t codepage, cha

              * Do conversion using native Win32 APIs.

              */

-            if (1200 != codepage) { /* not already UTF16-LE (Windows Unicode) */

+            if (CODEPAGE_UTF16_LE != codepage) { /* not already UTF16-LE (Windows Unicode) */

                 /*

                  * First, Convert from codepage -> UCS-2 LE with MultiByteToWideChar(codepage)

                  */

@@ -942,7 +942,7 @@ cl_error_t cli_codepage_to_utf8(char* in, size_t in_size, uint16_t codepage, cha

                 }

             }

-            if (NULL == encoding){

+            if (NULL == encoding) {

                 cli_dbgmsg("cli_codepage_to_utf8: Invalid codepage parameter passed in.\n");

                 goto done;

             }

@@ -953,7 +953,7 @@ cl_error_t cli_codepage_to_utf8(char* in, size_t in_size, uint16_t codepage, cha

                 size_t iconvRet = -1;

                 size_t outbytesleft = 0;

-                char* out_utf8_tmp = NULL;

+                char* out_utf8_tmp   = NULL;

                 char* out_utf8_index = NULL;

                 /* Charset to UTF-8 should never exceed in_size * 6;

@@ -978,8 +978,8 @@ cl_error_t cli_codepage_to_utf8(char* in, size_t in_size, uint16_t codepage, cha

                 iconvRet = iconv(conv, &inbuf, &inbufsize, &out_utf8_index, &outbytesleft);

                 iconv_close(conv);

-                conv = (iconv_t) -1;

-                if ((size_t)-1 == iconvRet){

+                conv = (iconv_t)-1;

+                if ((size_t)-1 == iconvRet) {

                     switch (errno) {

                         case E2BIG:

                             cli_warnmsg("cli_codepage_to_utf8: iconv error: There is not sufficient room at *outbuf.\n");

@@ -1036,7 +1036,7 @@ done:

 #endif

 #if defined(HAVE_ICONV)

-    if (conv != (iconv_t) -1) {

+    if (conv != (iconv_t)-1) {

         iconv_close(conv);

     }

 #endif

@@ -1050,9 +1050,9 @@ done:

     return status;

 }

-char *cli_utf16toascii(const char *str, unsigned int length)

+char* cli_utf16toascii(const char* str, unsigned int length)

 {

-    char *decoded;

+    char* decoded;

     unsigned int i, j;

     if (length < 2) {

@@ -1074,14 +1074,14 @@ char *cli_utf16toascii(const char *str, unsigned int length)

     return decoded;

 }

-char *cli_utf16_to_utf8(const char *utf16, size_t length, encoding_t type)

+char* cli_utf16_to_utf8(const char* utf16, size_t length, encoding_t type)

 {

     /* utf8 -

      * 4 bytes for utf16 high+low surrogate (4 bytes input)

      * 3 bytes for utf16 otherwise (2 bytes input) */

     size_t i, j;

     size_t needed = length * 3 / 2 + 2;

-    char *s2;

+    char* s2;

     if (length < 2)

         return cli_strdup("");

@@ -1146,7 +1146,7 @@ char *cli_utf16_to_utf8(const char *utf16, size_t length, encoding_t type)

     return s2;

 }

-int cli_isutf8(const char *buf, unsigned int len)

+int cli_isutf8(const char* buf, unsigned int len)

 {

     unsigned int i, j;

@@ -34,24 +34,193 @@

 #include "hashtab.h"

 #include "htmlnorm.h"

-#define UCS4_1234 "UCS-4BE"

-#define UCS4_4321 "UCS-4LE"

-#define UCS4_2143 "UCS4"

-#define UCS4_3412 "UCS-4"

-#define UTF16_BE "UTF-16BE"

-#define UTF16_LE "UTF-16LE"

-#define UTF8 "UTF-8"

-#define UNDECIDED_32_1234 UCS4_1234

-#define UNDECIDED_32_4321 UCS4_4321

-#define UNDECIDED_32_2143 UCS4_2143

-#define UNDECIDED_32_3412 UCS4_3412

-#define UNDECIDED_16_BE UTF16_BE

-#define UNDECIDED_16_LE UTF16_LE

-#define UNDECIDED_8 "ISO-8859-1"

-

-#define CODEPAGE_ISO8859_1 28591

-#define CODEPAGE_UTF16_LE  1200

-#define CODEPAGE_UTF16_BE  1201

+// clang-format off

+#define UCS4_1234           "UCS-4BE"

+#define UCS4_4321           "UCS-4LE"

+#define UCS4_2143           "UCS4"

+#define UCS4_3412           "UCS-4"

+#define UTF16_BE            "UTF-16BE"

+#define UTF16_LE            "UTF-16LE"

+#define UTF8                "UTF-8"

+#define UNDECIDED_32_1234   UCS4_1234

+#define UNDECIDED_32_4321   UCS4_4321

+#define UNDECIDED_32_2143   UCS4_2143

+#define UNDECIDED_32_3412   UCS4_3412

+#define UNDECIDED_16_BE     UTF16_BE

+#define UNDECIDED_16_LE     UTF16_LE

+#define UNDECIDED_8         "ISO-8859-1"

+

+#define CODEPAGE_JAPANESE_SHIFT_JIS 932     /* Japanese Shift-JIS */

+#define CODEPAGE_KOREAN_UNIFIED     949     /* Korean Unified Code */

+#define CODEPAGE_UTF16_LE           1200    /* UTF16 Little Endian */

+#define CODEPAGE_UTF16_BE           1201    /* UTF16 Big Endian */

+#define CODEPAGE_US_7BIT_ASCII      20127   /* US-ASCII (7-bit)  */

+#define CODEPAGE_ISO8859_1          28591   /* ISO 8859-1 Latin 1; Western European (ISO) */

+#define CODEPAGE_UTF8               65001   /* UTF-8 */

+// clang-format on

+

+/* string conversion */

+struct codepage_entry {

+    uint16_t codepage;

+    const char *encoding;

+};

+

+#define NUMCODEPAGES (sizeof(codepage_entries) / sizeof(struct codepage_entry))

+/* MAINTAIN - the array in codepage value sorted order */

+static const struct codepage_entry codepage_entries[] = {

+    {37, "IBM037"},           /* IBM EBCDIC US-Canada */

+    {437, "IBM437"},          /* OEM United States */

+    {500, "IBM500"},          /* IBM EBCDIC International */

+    {708, "ASMO-708"},        /* Arabic (ASMO 708) */

+    {709, NULL},              /* Arabic (ASMO-449+, BCON V4) */

+    {710, NULL},              /* Arabic - Transparent Arabic */

+    {720, NULL},              /* Arabic (Transparent ASMO); Arabic (DOS) */

+    {737, NULL},              /* OEM Greek (formerly 437G); Greek (DOS) */

+    {775, "IBM775"},          /* OEM Baltic; Baltic (DOS) */

+    {850, "IBM850"},          /* OEM Multilingual Latin 1; Western European (DOS) */

+    {852, "IBM852"},          /* OEM Latin 2; Central European (DOS) */

+    {855, "IBM855"},          /* OEM Cyrillic (primarily Russian) */

+    {857, "IBM857"},          /* OEM Turkish; Turkish (DOS) */

+    {858, NULL},              /* OEM Multilingual Latin 1 + Euro symbol */

+    {860, "IBM860"},          /* OEM Portuguese; Portuguese (DOS) */

+    {861, "IBM861"},          /* OEM Icelandic; Icelandic (DOS) */

+    {862, NULL},              /* OEM Hebrew; Hebrew (DOS) */

+    {863, "IBM863"},          /* OEM French Canadian; French Canadian (DOS) */

+    {864, "IBM864"},          /* OEM Arabic; Arabic (864) */

+    {865, "IBM865"},          /* OEM Nordic; Nordic (DOS) */

+    {866, "CP866"},           /* OEM Russian; Cyrillic (DOS) */

+    {869, "IBM869"},          /* OEM Modern Greek; Greek, Modern (DOS) */

+    {870, "IBM870"},          /* IBM EBCDIC Multilingual/ROECE (Latin 2); IBM EBCDIC Multilingual Latin 2 */

+    {874, "WINDOWS-874"},     /* ANSI/OEM Thai (ISO 8859-11); Thai (Windows) */

+    {875, "CP875"},           /* IBM EBCDIC Greek Modern */

+    {932, "SHIFT_JIS"},       /* ANSI/OEM Japanese; Japanese (Shift-JIS) */

+    {936, "GB2312"},          /* ANSI/OEM Simplified Chinese (PRC, Singapore); Chinese Simplified (GB2312) */

+    {949, "CP949"},           /* ANSI/OEM Korean (Unified Hangul Code) */

+    {950, "BIG5"},            /* ANSI/OEM Traditional Chinese (Taiwan; Hong Kong SAR, PRC); Chinese Traditional (Big5) */

+    {1026, "IBM1026"},        /* IBM EBCDIC Turkish (Latin 5) */

+    {1047, NULL},             /* IBM EBCDIC Latin 1/Open System */

+    {1140, NULL},             /* IBM EBCDIC US-Canada (037 + Euro symbol); IBM EBCDIC (US-Canada-Euro) */

+    {1141, NULL},             /* IBM EBCDIC Germany (20273 + Euro symbol); IBM EBCDIC (Germany-Euro) */

+    {1142, NULL},             /* IBM EBCDIC Denmark-Norway (20277 + Euro symbol); IBM EBCDIC (Denmark-Norway-Euro) */

+    {1143, NULL},             /* IBM EBCDIC Finland-Sweden (20278 + Euro symbol); IBM EBCDIC (Finland-Sweden-Euro) */

+    {1144, NULL},             /* IBM EBCDIC Italy (20280 + Euro symbol); IBM EBCDIC (Italy-Euro) */

+    {1145, NULL},             /* IBM EBCDIC Latin America-Spain (20284 + Euro symbol); IBM EBCDIC (Spain-Euro) */

+    {1146, NULL},             /* IBM EBCDIC United Kingdom (20285 + Euro symbol); IBM EBCDIC (UK-Euro) */

+    {1147, NULL},             /* IBM EBCDIC France (20297 + Euro symbol); IBM EBCDIC (France-Euro) */

+    {1148, NULL},             /* IBM EBCDIC International (500 + Euro symbol); IBM EBCDIC (International-Euro) */

+    {1149, NULL},             /* IBM EBCDIC Icelandic (20871 + Euro symbol); IBM EBCDIC (Icelandic-Euro) */

+    {1200, "UTF-16LE"},       /* Unicode UTF-16, little endian byte order (BMP of ISO 10646); available only to managed applications */

+    {1201, "UTF-16BE"},       /* Unicode UTF-16, big endian byte order; available only to managed applications */

+    {1250, "WINDOWS-1250"},   /* ANSI Central European; Central European (Windows) */

+    {1251, "WINDOWS-1251"},   /* ANSI Cyrillic; Cyrillic (Windows) */

+    {1252, "WINDOWS-1252"},   /* ANSI Latin 1; Western European (Windows) */

+    {1253, "WINDOWS-1253"},   /* ANSI Greek; Greek (Windows) */

+    {1254, "WINDOWS-1254"},   /* ANSI Turkish; Turkish (Windows) */

+    {1255, "WINDOWS-1255"},   /* ANSI Hebrew; Hebrew (Windows) */

+    {1256, "WINDOWS-1256"},   /* ANSI Arabic; Arabic (Windows) */

+    {1257, "WINDOWS-1257"},   /* ANSI Baltic; Baltic (Windows) */

+    {1258, "WINDOWS-1258"},   /* ANSI/OEM Vietnamese; Vietnamese (Windows) */

+    {1361, "JOHAB"},          /* Korean (Johab) */

+    {10000, "MACINTOSH"},     /* MAC Roman; Western European (Mac) */

+    {10001, NULL},            /* Japanese (Mac) */

+    {10002, NULL},            /* MAC Traditional Chinese (Big5); Chinese Traditional (Mac) */

+    {10003, NULL},            /* Korean (Mac) */

+    {10004, NULL},            /* Arabic (Mac) */

+    {10005, NULL},            /* Hebrew (Mac) */

+    {10006, NULL},            /* Greek (Mac) */

+    {10007, NULL},            /* Cyrillic (Mac) */

+    {10008, NULL},            /* MAC Simplified Chinese (GB 2312); Chinese Simplified (Mac) */

+    {10010, NULL},            /* Romanian (Mac) */

+    {10017, NULL},            /* Ukrainian (Mac) */

+    {10021, NULL},            /* Thai (Mac) */

+    {10029, NULL},            /* MAC Latin 2; Central European (Mac) */

+    {10079, NULL},            /* Icelandic (Mac) */

+    {10081, NULL},            /* Turkish (Mac) */

+    {10082, NULL},            /* Croatian (Mac) */

+    {12000, "UTF-32LE"},      /* Unicode UTF-32, little endian byte order; available only to managed applications */

+    {12001, "UTF-32BE"},      /* Unicode UTF-32, big endian byte order; available only to managed applications */

+    {20000, NULL},            /* CNS Taiwan; Chinese Traditional (CNS) */

+    {20001, NULL},            /* TCA Taiwan */

+    {20002, NULL},            /* Eten Taiwan; Chinese Traditional (Eten) */

+    {20003, NULL},            /* IBM5550 Taiwan */

+    {20004, NULL},            /* TeleText Taiwan */

+    {20005, NULL},            /* Wang Taiwan */

+    {20105, NULL},            /* IA5 (IRV International Alphabet No. 5, 7-bit); Western European (IA5) */

+    {20106, NULL},            /* IA5 German (7-bit) */

+    {20107, NULL},            /* IA5 Swedish (7-bit) */

+    {20108, NULL},            /* IA5 Norwegian (7-bit) */

+    {20127, "US-ASCII"},      /* US-ASCII (7-bit) */

+    {20261, NULL},            /* T.61 */

+    {20269, NULL},            /* ISO 6937 Non-Spacing Accent */

+    {20273, "IBM273"},        /* IBM EBCDIC Germany */

+    {20277, "IBM277"},        /* IBM EBCDIC Denmark-Norway */

+    {20278, "IBM278"},        /* IBM EBCDIC Finland-Sweden */

+    {20280, "IBM280"},        /* IBM EBCDIC Italy */

+    {20284, "IBM284"},        /* IBM EBCDIC Latin America-Spain */

+    {20285, "IBM285"},        /* IBM EBCDIC United Kingdom */

+    {20290, "IBM290"},        /* IBM EBCDIC Japanese Katakana Extended */

+    {20297, "IBM297"},        /* IBM EBCDIC France */

+    {20420, "IBM420"},        /* IBM EBCDIC Arabic */

+    {20423, "IBM423"},        /* IBM EBCDIC Greek */

+    {20424, "IBM424"},        /* IBM EBCDIC Hebrew */

+    {20833, NULL},            /* IBM EBCDIC Korean Extended */

+    {20838, NULL},            /* IBM EBCDIC Thai */

+    {20866, "KOI8-R"},        /* Russian (KOI8-R); Cyrillic (KOI8-R) */

+    {20871, "IBM871"},        /* IBM EBCDIC Icelandic */

+    {20880, "IBM880"},        /* IBM EBCDIC Cyrillic Russian */

+    {20905, "IBM905"},        /* IBM EBCDIC Turkish */

+    {20924, NULL},            /* IBM EBCDIC Latin 1/Open System (1047 + Euro symbol) */

+    {20932, "EUC-JP"},        /* Japanese (JIS 0208-1990 and 0212-1990) */

+    {20936, NULL},            /* Simplified Chinese (GB2312); Chinese Simplified (GB2312-80) */

+    {20949, NULL},            /* Korean Wansung */

+    {21025, "CP1025"},        /* IBM EBCDIC Cyrillic Serbian-Bulgarian */

+    {21027, NULL},            /* (deprecated) */

+    {21866, "KOI8-U"},        /* Ukrainian (KOI8-U); Cyrillic (KOI8-U) */

+    {28591, "ISO-8859-1"},    /* ISO 8859-1 Latin 1; Western European (ISO) */

+    {28592, "ISO-8859-2"},    /* ISO 8859-2 Central European; Central European (ISO) */

+    {28593, "ISO-8859-3"},    /* ISO 8859-3 Latin 3 */

+    {28594, "ISO-8859-4"},    /* ISO 8859-4 Baltic */

+    {28595, "ISO-8859-5"},    /* ISO 8859-5 Cyrillic */

+    {28596, "ISO-8859-6"},    /* ISO 8859-6 Arabic */

+    {28597, "ISO-8859-7"},    /* ISO 8859-7 Greek */

+    {28598, "ISO-8859-8"},    /* ISO 8859-8 Hebrew; Hebrew (ISO-Visual) */

+    {28599, "ISO-8859-9"},    /* ISO 8859-9 Turkish */

+    {28603, "ISO-8859-13"},   /* ISO 8859-13 Estonian */

+    {28605, "ISO-8859-15"},   /* ISO 8859-15 Latin 9 */

+    {29001, NULL},            /* Europa 3 */

+    {38598, NULL},            /* ISO 8859-8 Hebrew; Hebrew (ISO-Logical) */

+    {50220, "ISO-2022-JP"},   /* ISO 2022 Japanese with no halfwidth Katakana; Japanese (JIS) (guess) */

+    {50221, "ISO-2022-JP-2"}, /* ISO 2022 Japanese with halfwidth Katakana; Japanese (JIS-Allow 1 byte Kana) (guess) */

+    {50222, "ISO-2022-JP-3"}, /* ISO 2022 Japanese JIS X 0201-1989; Japanese (JIS-Allow 1 byte Kana - SO/SI) (guess) */

+    {50225, "ISO-2022-KR"},   /* ISO 2022 Korean */

+    {50227, NULL},            /* ISO 2022 Simplified Chinese; Chinese Simplified (ISO 2022) */

+    {50229, NULL},            /* ISO 2022 Traditional Chinese */

+    {50930, NULL},            /* EBCDIC Japanese (Katakana) Extended */

+    {50931, NULL},            /* EBCDIC US-Canada and Japanese */

+    {50933, NULL},            /* EBCDIC Korean Extended and Korean */

+    {50935, NULL},            /* EBCDIC Simplified Chinese Extended and Simplified Chinese */

+    {50936, NULL},            /* EBCDIC Simplified Chinese */

+    {50937, NULL},            /* EBCDIC US-Canada and Traditional Chinese */

+    {50939, NULL},            /* EBCDIC Japanese (Latin) Extended and Japanese */

+    {51932, "EUC-JP"},        /* EUC Japanese */

+    {51936, "EUC-CN"},        /* EUC Simplified Chinese; Chinese Simplified (EUC) */

+    {51949, "EUC-KR"},        /* EUC Korean */

+    {51950, NULL},            /* EUC Traditional Chinese */

+    {52936, NULL},            /* HZ-GB2312 Simplified Chinese; Chinese Simplified (HZ) */

+    {54936, "GB18030"},       /* Windows XP and later: GB18030 Simplified Chinese (4 byte); Chinese Simplified (GB18030) */

+    {57002, NULL},            /* ISCII Devanagari */

+    {57003, NULL},            /* ISCII Bengali */

+    {57004, NULL},            /* ISCII Tamil */

+    {57005, NULL},            /* ISCII Telugu */

+    {57006, NULL},            /* ISCII Assamese */

+    {57007, NULL},            /* ISCII Oriya */

+    {57008, NULL},            /* ISCII Kannada */

+    {57009, NULL},            /* ISCII Malayalam */

+    {57010, NULL},            /* ISCII Gujarati */

+    {57011, NULL},            /* ISCII Punjabi */

+    {65000, "UTF-7"},         /* Unicode (UTF-7) */

+    {65001, "UTF-8"}          /* Unicode (UTF-8) */

+};

 #define MAX_ENTITY_SIZE 22

@@ -90,10 +259,10 @@ int encoding_normalize_toascii(const m_area_t* in_m_area, const char* initial_en

  */

 cl_error_t cli_codepage_to_utf8(char* in, size_t in_size, uint16_t codepage, char** out, size_t* out_size);

-char *cli_utf16toascii(const char *str, unsigned int length);

+char* cli_utf16toascii(const char* str, unsigned int length);

-char *cli_utf16_to_utf8(const char *utf16, size_t length, encoding_t type);

+char* cli_utf16_to_utf8(const char* utf16, size_t length, encoding_t type);

-int cli_isutf8(const char *buf, unsigned int len);

+int cli_isutf8(const char* buf, unsigned int len);

 #endif

@@ -261,6 +261,7 @@ CLAMAV_PRIVATE {

     cli_gentemp_with_prefix;

     cli_basename;

     cli_realpath;

+    cli_codepage_to_utf8;

     __cli_strcasestr;

     __cli_strndup;

@@ -47,6 +47,7 @@

 #include "scanners.h"

 #include "fmap.h"

 #include "json_api.h"

+#include "entconv.h"

 #if HAVE_JSON

 static char *

@@ -64,7 +65,7 @@ ole2_convert_utf(summary_ctx_t *sctx, char *begin, size_t sz, const char *encodi

     UNUSEDPARAM(encoding);

 #endif

     /* applies in the both case */

-    if (sctx->codepage == 20127 || sctx->codepage == 65001) {

+    if (sctx->codepage == 20127 || sctx->codepage == CODEPAGE_UTF8) {

         char *track;

         size_t bcnt, scnt;

@@ -74,7 +75,7 @@ ole2_convert_utf(summary_ctx_t *sctx, char *begin, size_t sz, const char *encodi

         memcpy(outbuf, begin, sz);

         track = outbuf + sz - 1;

-        if ((sctx->codepage == 65001) && (*track & 0x80)) { /* UTF-8 with a most significant bit */

+        if ((sctx->codepage == CODEPAGE_UTF8) && (*track & 0x80)) { /* UTF-8 with a most significant bit */

             /* locate the start of the last character */

             for (bcnt = 1; (track != outbuf); track--, bcnt++) {

                 if (((uint8_t)*track & 0xC0) != 0x80)

@@ -27,169 +27,6 @@

 #include "others.h"

 #include "uniq.h"

-/* string conversion */

-struct codepage_entry {

-    uint16_t codepage;

-    const char *encoding;

-};

-

-#define NUMCODEPAGES (sizeof(codepage_entries) / sizeof(struct codepage_entry))

-/* MAINTAIN - the array in codepage value sorted order */

-static const struct codepage_entry codepage_entries[] = {

-    {37, "IBM037"},           /* IBM EBCDIC US-Canada */

-    {437, "IBM437"},          /* OEM United States */

-    {500, "IBM500"},          /* IBM EBCDIC International */

-    {708, "ASMO-708"},        /* Arabic (ASMO 708) */

-    {709, NULL},              /* Arabic (ASMO-449+, BCON V4) */

-    {710, NULL},              /* Arabic - Transparent Arabic */

-    {720, NULL},              /* Arabic (Transparent ASMO); Arabic (DOS) */

-    {737, NULL},              /* OEM Greek (formerly 437G); Greek (DOS) */

-    {775, "IBM775"},          /* OEM Baltic; Baltic (DOS) */

-    {850, "IBM850"},          /* OEM Multilingual Latin 1; Western European (DOS) */

-    {852, "IBM852"},          /* OEM Latin 2; Central European (DOS) */

-    {855, "IBM855"},          /* OEM Cyrillic (primarily Russian) */

-    {857, "IBM857"},          /* OEM Turkish; Turkish (DOS) */

-    {858, NULL},              /* OEM Multilingual Latin 1 + Euro symbol */

-    {860, "IBM860"},          /* OEM Portuguese; Portuguese (DOS) */

-    {861, "IBM861"},          /* OEM Icelandic; Icelandic (DOS) */

-    {862, NULL},              /* OEM Hebrew; Hebrew (DOS) */

-    {863, "IBM863"},          /* OEM French Canadian; French Canadian (DOS) */

-    {864, "IBM864"},          /* OEM Arabic; Arabic (864) */

-    {865, "IBM865"},          /* OEM Nordic; Nordic (DOS) */

-    {866, "CP866"},           /* OEM Russian; Cyrillic (DOS) */

-    {869, "IBM869"},          /* OEM Modern Greek; Greek, Modern (DOS) */

-    {870, "IBM870"},          /* IBM EBCDIC Multilingual/ROECE (Latin 2); IBM EBCDIC Multilingual Latin 2 */

-    {874, "WINDOWS-874"},     /* ANSI/OEM Thai (ISO 8859-11); Thai (Windows) */

-    {875, "CP875"},           /* IBM EBCDIC Greek Modern */

-    {932, "SHIFT_JIS"},       /* ANSI/OEM Japanese; Japanese (Shift-JIS) */

-    {936, "GB2312"},          /* ANSI/OEM Simplified Chinese (PRC, Singapore); Chinese Simplified (GB2312) */

-    {949, "CP949"},           /* ANSI/OEM Korean (Unified Hangul Code) */

-    {950, "BIG5"},            /* ANSI/OEM Traditional Chinese (Taiwan; Hong Kong SAR, PRC); Chinese Traditional (Big5) */

-    {1026, "IBM1026"},        /* IBM EBCDIC Turkish (Latin 5) */

-    {1047, NULL},             /* IBM EBCDIC Latin 1/Open System */

-    {1140, NULL},             /* IBM EBCDIC US-Canada (037 + Euro symbol); IBM EBCDIC (US-Canada-Euro) */

-    {1141, NULL},             /* IBM EBCDIC Germany (20273 + Euro symbol); IBM EBCDIC (Germany-Euro) */

-    {1142, NULL},             /* IBM EBCDIC Denmark-Norway (20277 + Euro symbol); IBM EBCDIC (Denmark-Norway-Euro) */

-    {1143, NULL},             /* IBM EBCDIC Finland-Sweden (20278 + Euro symbol); IBM EBCDIC (Finland-Sweden-Euro) */

-    {1144, NULL},             /* IBM EBCDIC Italy (20280 + Euro symbol); IBM EBCDIC (Italy-Euro) */

-    {1145, NULL},             /* IBM EBCDIC Latin America-Spain (20284 + Euro symbol); IBM EBCDIC (Spain-Euro) */

-    {1146, NULL},             /* IBM EBCDIC United Kingdom (20285 + Euro symbol); IBM EBCDIC (UK-Euro) */

-    {1147, NULL},             /* IBM EBCDIC France (20297 + Euro symbol); IBM EBCDIC (France-Euro) */

-    {1148, NULL},             /* IBM EBCDIC International (500 + Euro symbol); IBM EBCDIC (International-Euro) */

-    {1149, NULL},             /* IBM EBCDIC Icelandic (20871 + Euro symbol); IBM EBCDIC (Icelandic-Euro) */

-    {1200, "UTF-16LE"},       /* Unicode UTF-16, little endian byte order (BMP of ISO 10646); available only to managed applications */

-    {1201, "UTF-16BE"},       /* Unicode UTF-16, big endian byte order; available only to managed applications */

-    {1250, "WINDOWS-1250"},   /* ANSI Central European; Central European (Windows) */

-    {1251, "WINDOWS-1251"},   /* ANSI Cyrillic; Cyrillic (Windows) */

-    {1252, "WINDOWS-1252"},   /* ANSI Latin 1; Western European (Windows) */

-    {1253, "WINDOWS-1253"},   /* ANSI Greek; Greek (Windows) */

-    {1254, "WINDOWS-1254"},   /* ANSI Turkish; Turkish (Windows) */

-    {1255, "WINDOWS-1255"},   /* ANSI Hebrew; Hebrew (Windows) */

-    {1256, "WINDOWS-1256"},   /* ANSI Arabic; Arabic (Windows) */

-    {1257, "WINDOWS-1257"},   /* ANSI Baltic; Baltic (Windows) */

-    {1258, "WINDOWS-1258"},   /* ANSI/OEM Vietnamese; Vietnamese (Windows) */

-    {1361, "JOHAB"},          /* Korean (Johab) */

-    {10000, "MACINTOSH"},     /* MAC Roman; Western European (Mac) */

-    {10001, NULL},            /* Japanese (Mac) */

-    {10002, NULL},            /* MAC Traditional Chinese (Big5); Chinese Traditional (Mac) */

-    {10003, NULL},            /* Korean (Mac) */

-    {10004, NULL},            /* Arabic (Mac) */

-    {10005, NULL},            /* Hebrew (Mac) */

-    {10006, NULL},            /* Greek (Mac) */

-    {10007, NULL},            /* Cyrillic (Mac) */

-    {10008, NULL},            /* MAC Simplified Chinese (GB 2312); Chinese Simplified (Mac) */

-    {10010, NULL},            /* Romanian (Mac) */

-    {10017, NULL},            /* Ukrainian (Mac) */

-    {10021, NULL},            /* Thai (Mac) */

-    {10029, NULL},            /* MAC Latin 2; Central European (Mac) */

-    {10079, NULL},            /* Icelandic (Mac) */

-    {10081, NULL},            /* Turkish (Mac) */

-    {10082, NULL},            /* Croatian (Mac) */

-    {12000, "UTF-32LE"},      /* Unicode UTF-32, little endian byte order; available only to managed applications */

-    {12001, "UTF-32BE"},      /* Unicode UTF-32, big endian byte order; available only to managed applications */

-    {20000, NULL},            /* CNS Taiwan; Chinese Traditional (CNS) */

-    {20001, NULL},            /* TCA Taiwan */

-    {20002, NULL},            /* Eten Taiwan; Chinese Traditional (Eten) */

-    {20003, NULL},            /* IBM5550 Taiwan */

-    {20004, NULL},            /* TeleText Taiwan */

-    {20005, NULL},            /* Wang Taiwan */

-    {20105, NULL},            /* IA5 (IRV International Alphabet No. 5, 7-bit); Western European (IA5) */

-    {20106, NULL},            /* IA5 German (7-bit) */

-    {20107, NULL},            /* IA5 Swedish (7-bit) */

-    {20108, NULL},            /* IA5 Norwegian (7-bit) */

-    {20127, "US-ASCII"},      /* US-ASCII (7-bit) */

-    {20261, NULL},            /* T.61 */

-    {20269, NULL},            /* ISO 6937 Non-Spacing Accent */

-    {20273, "IBM273"},        /* IBM EBCDIC Germany */

-    {20277, "IBM277"},        /* IBM EBCDIC Denmark-Norway */

-    {20278, "IBM278"},        /* IBM EBCDIC Finland-Sweden */

-    {20280, "IBM280"},        /* IBM EBCDIC Italy */

-    {20284, "IBM284"},        /* IBM EBCDIC Latin America-Spain */

-    {20285, "IBM285"},        /* IBM EBCDIC United Kingdom */

-    {20290, "IBM290"},        /* IBM EBCDIC Japanese Katakana Extended */

-    {20297, "IBM297"},        /* IBM EBCDIC France */

-    {20420, "IBM420"},        /* IBM EBCDIC Arabic */

-    {20423, "IBM423"},        /* IBM EBCDIC Greek */

-    {20424, "IBM424"},        /* IBM EBCDIC Hebrew */

-    {20833, NULL},            /* IBM EBCDIC Korean Extended */

-    {20838, NULL},            /* IBM EBCDIC Thai */

-    {20866, "KOI8-R"},        /* Russian (KOI8-R); Cyrillic (KOI8-R) */

-    {20871, "IBM871"},        /* IBM EBCDIC Icelandic */

-    {20880, "IBM880"},        /* IBM EBCDIC Cyrillic Russian */

-    {20905, "IBM905"},        /* IBM EBCDIC Turkish */

-    {20924, NULL},            /* IBM EBCDIC Latin 1/Open System (1047 + Euro symbol) */

-    {20932, "EUC-JP"},        /* Japanese (JIS 0208-1990 and 0212-1990) */

-    {20936, NULL},            /* Simplified Chinese (GB2312); Chinese Simplified (GB2312-80) */

-    {20949, NULL},            /* Korean Wansung */

-    {21025, "CP1025"},        /* IBM EBCDIC Cyrillic Serbian-Bulgarian */

-    {21027, NULL},            /* (deprecated) */

-    {21866, "KOI8-U"},        /* Ukrainian (KOI8-U); Cyrillic (KOI8-U) */

-    {28591, "ISO-8859-1"},    /* ISO 8859-1 Latin 1; Western European (ISO) */

-    {28592, "ISO-8859-2"},    /* ISO 8859-2 Central European; Central European (ISO) */

-    {28593, "ISO-8859-3"},    /* ISO 8859-3 Latin 3 */

-    {28594, "ISO-8859-4"},    /* ISO 8859-4 Baltic */

-    {28595, "ISO-8859-5"},    /* ISO 8859-5 Cyrillic */

-    {28596, "ISO-8859-6"},    /* ISO 8859-6 Arabic */

-    {28597, "ISO-8859-7"},    /* ISO 8859-7 Greek */

-    {28598, "ISO-8859-8"},    /* ISO 8859-8 Hebrew; Hebrew (ISO-Visual) */

-    {28599, "ISO-8859-9"},    /* ISO 8859-9 Turkish */

-    {28603, "ISO-8859-13"},   /* ISO 8859-13 Estonian */

-    {28605, "ISO-8859-15"},   /* ISO 8859-15 Latin 9 */

-    {29001, NULL},            /* Europa 3 */

-    {38598, NULL},            /* ISO 8859-8 Hebrew; Hebrew (ISO-Logical) */

-    {50220, "ISO-2022-JP"},   /* ISO 2022 Japanese with no halfwidth Katakana; Japanese (JIS) (guess) */

-    {50221, "ISO-2022-JP-2"}, /* ISO 2022 Japanese with halfwidth Katakana; Japanese (JIS-Allow 1 byte Kana) (guess) */

-    {50222, "ISO-2022-JP-3"}, /* ISO 2022 Japanese JIS X 0201-1989; Japanese (JIS-Allow 1 byte Kana - SO/SI) (guess) */

-    {50225, "ISO-2022-KR"},   /* ISO 2022 Korean */

-    {50227, NULL},            /* ISO 2022 Simplified Chinese; Chinese Simplified (ISO 2022) */

-    {50229, NULL},            /* ISO 2022 Traditional Chinese */

-    {50930, NULL},            /* EBCDIC Japanese (Katakana) Extended */

-    {50931, NULL},            /* EBCDIC US-Canada and Japanese */

-    {50933, NULL},            /* EBCDIC Korean Extended and Korean */

-    {50935, NULL},            /* EBCDIC Simplified Chinese Extended and Simplified Chinese */

-    {50936, NULL},            /* EBCDIC Simplified Chinese */

-    {50937, NULL},            /* EBCDIC US-Canada and Traditional Chinese */

-    {50939, NULL},            /* EBCDIC Japanese (Latin) Extended and Japanese */

-    {51932, "EUC-JP"},        /* EUC Japanese */

-    {51936, "EUC-CN"},        /* EUC Simplified Chinese; Chinese Simplified (EUC) */

-    {51949, "EUC-KR"},        /* EUC Korean */

-    {51950, NULL},            /* EUC Traditional Chinese */

-    {52936, NULL},            /* HZ-GB2312 Simplified Chinese; Chinese Simplified (HZ) */

-    {54936, "GB18030"},       /* Windows XP and later: GB18030 Simplified Chinese (4 byte); Chinese Simplified (GB18030) */

-    {57002, NULL},            /* ISCII Devanagari */

-    {57003, NULL},            /* ISCII Bengali */

-    {57004, NULL},            /* ISCII Tamil */

-    {57005, NULL},            /* ISCII Telugu */

-    {57006, NULL},            /* ISCII Assamese */

-    {57007, NULL},            /* ISCII Oriya */

-    {57008, NULL},            /* ISCII Kannada */

-    {57009, NULL},            /* ISCII Malayalam */

-    {57010, NULL},            /* ISCII Gujarati */

-    {57011, NULL},            /* ISCII Punjabi */

-    {65000, "UTF-7"},         /* Unicode (UTF-7) */

-    {65001, "UTF-8"}          /* Unicode (UTF-8) */

-};

-

 #if HAVE_JSON

 #define PROPCNTLIMIT 25

@@ -24,6 +24,7 @@

 #include "../libclamav/dsig.h"

 #include "../libclamav/fpu.h"

 #include "../platform.h"

+#include "../libclamav/entconv.h"

 #include "checks.h"

 static int fpu_words = FPU_ENDIAN_INITME;

@@ -983,6 +984,54 @@ START_TEST(test_sanitize_path)

 }

 END_TEST

+START_TEST(test_cli_codepage_to_utf8)

+{

+    cl_error_t ret;

+    char *utf8       = NULL;

+    size_t utf8_size = 0;

+

+    ret = cli_codepage_to_utf8("\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD", 10, CODEPAGE_JAPANESE_SHIFT_JIS, &utf8, &utf8_size);

+    ck_assert_msg(CL_SUCCESS == ret, "test_cli_codepage_to_utf8: Failed to convert CODEPAGE_JAPANESE_SHIFT_JIS to UTF8: ret != SUCCESS!");

+    ck_assert_msg(NULL != utf8, "sanitize_path: Failed to convert CODEPAGE_JAPANESE_SHIFT_JIS to UTF8: utf8 pointer is NULL!");

+    ck_assert_msg(0 == strcmp(utf8, "こんにちは"), "sanitize_path: '%s' doesn't match '%s'", utf8, "こんにちは");

+

+    if (NULL != utf8) {

+        free(utf8);

+        utf8 = NULL;

+    }

+

+    ret = cli_codepage_to_utf8("\x00\x48\x00\x65\x00\x6c\x00\x6c\x00\x6f\x00\x20\x00\x77\x00\x6f\x00\x72\x00\x6c\x00\x64\x00\x21\x00\x00", 26, CODEPAGE_UTF16_BE, &utf8, &utf8_size);

+    ck_assert_msg(CL_SUCCESS == ret, "test_cli_codepage_to_utf8: Failed to convert CODEPAGE_UTF16_LE to UTF8: ret != SUCCESS!");

+    ck_assert_msg(NULL != utf8, "sanitize_path: Failed to convert CODEPAGE_UTF16_LE to UTF8: utf8 pointer is NULL!");

+    ck_assert_msg(0 == strcmp(utf8, "Hello world!"), "sanitize_path: '%s' doesn't match '%s'", utf8, "Hello world!");

+

+    if (NULL != utf8) {

+        free(utf8);

+        utf8 = NULL;

+    }

+

+    ret = cli_codepage_to_utf8("\x00\x48\x00\x65\x00\x6c\x00\x6c\x00\x6f\x00\x20\x00\x77\x00\x6f\x00\x72\x00\x6c\x00\x64\x00\x21", 24, CODEPAGE_UTF16_BE, &utf8, &utf8_size);

+    ck_assert_msg(CL_SUCCESS == ret, "test_cli_codepage_to_utf8: Failed to convert CODEPAGE_UTF16_BE to UTF8: ret != SUCCESS!");

+    ck_assert_msg(NULL != utf8, "sanitize_path: Failed to convert CODEPAGE_UTF16_BE to UTF8: utf8 pointer is NULL!");

+    ck_assert_msg(0 == strcmp(utf8, "Hello world!"), "sanitize_path: '%s' doesn't match '%s'", utf8, "Hello world!");

+

+    if (NULL != utf8) {

+        free(utf8);

+        utf8 = NULL;

+    }

+

+    ret = cli_codepage_to_utf8("\x48\x00\x65\x00\x6c\x00\x6c\x00\x6f\x00\x20\x00\x77\x00\x6f\x00\x72\x00\x6c\x00\x64\x00\x21\x00\x00\x00", 26, CODEPAGE_UTF16_LE, &utf8, &utf8_size);

+    ck_assert_msg(CL_SUCCESS == ret, "test_cli_codepage_to_utf8: Failed to convert CODEPAGE_UTF16_LE to UTF8: ret != SUCCESS!");

+    ck_assert_msg(NULL != utf8, "sanitize_path: Failed to convert CODEPAGE_UTF16_LE to UTF8: utf8 pointer is NULL!");

+    ck_assert_msg(0 == strcmp(utf8, "Hello world!"), "sanitize_path: '%s' doesn't match '%s'", utf8, "Hello world!");

+

+    if (NULL != utf8) {

+        free(utf8);

+        utf8 = NULL;

+    }

+}

+END_TEST

+

 static Suite *test_cli_suite(void)

 {

     Suite *s               = suite_create("cli");

@@ -1002,6 +1051,7 @@ static Suite *test_cli_suite(void)

     suite_add_tcase(s, tc_cli_assorted);

     tcase_add_test(tc_cli_assorted, test_sanitize_path);

+    tcase_add_test(tc_cli_assorted, test_cli_codepage_to_utf8);

     return s;

 }