[ROSAPPS] Add initial version txt2nls file converter (DBCS codepages not supported yet)

svn path=/trunk/; revision=72700

rosapps/applications/devutils/CMakeLists.txt

@@ -7,4 +7,5 @@ add_subdirectory(nls2txt)
 add_subdirectory(shimdbg)
 add_subdirectory(symdump)
 add_subdirectory(syscalldump)
+add_subdirectory(txt2nls)
 add_subdirectory(vgafontedit)

rosapps/applications/devutils/txt2nls/CMakeLists.txt

@@ -0,0 +1,12 @@
+
+list(APPEND SOURCE
+    main.c
+    txt.c
+    nls.c
+    precomp.h)
+
+add_executable(txt2nls ${SOURCE} txt2nls.rc)
+add_pch(txt2nls precomp.h SOURCE)
+set_module_type(txt2nls win32cui)
+add_importlibs(txt2nls msvcrt kernel32)
+add_cd_file(TARGET txt2nls DESTINATION reactos/system32 FOR all)

rosapps/applications/devutils/txt2nls/main.c

@@ -0,0 +1,19 @@
+/*
+ * PROJECT:     ReactOS TXT to NLS Converter
+ * LICENSE:     GNU General Public License Version 2.0 or any later version
+ * FILE:        devutils/txt2nls/main.c
+ * COPYRIGHT:   Copyright 2016 Dmitry Chapyshev <dmitry@reactos.org>
+ */
+
+#include "precomp.h"
+
+int main(int argc, char* argv[])
+{
+    if (argc != 3)
+        return 1;
+
+    if (!nls_from_txt(argv[1], argv[2]))
+        return 1;
+
+    return 0;
+}

rosapps/applications/devutils/txt2nls/nls.c

@@ -0,0 +1,231 @@
+/*
+ * PROJECT:     ReactOS TXT to NLS Converter
+ * LICENSE:     GNU General Public License Version 2.0 or any later version
+ * FILE:        devutils/txt2nls/nls.c
+ * COPYRIGHT:   Copyright 2016 Dmitry Chapyshev <dmitry@reactos.org>
+ */
+
+#include "precomp.h"
+
+#define _NLS_DEBUG_PRINT
+
+#ifdef _NLS_DEBUG_PRINT
+
+static void
+nls_print_header(NLS_FILE_HEADER *header)
+{
+    uint32_t i;
+
+    printf("HEADER:\n");
+    printf("CodePage: %u\n", header->CodePage);
+    printf("Character size: %u\n", header->MaximumCharacterSize);
+    printf("Default char: 0x%02X\n", header->DefaultChar);
+    printf("Default unicode char: 0x%04X\n", header->UniDefaultChar);
+    printf("Trans default char: 0x%02X\n", header->TransUniDefaultChar);
+    printf("Trans default unicode char: 0x%04X\n", header->TransUniDefaultChar);
+
+    for (i = 0; i < MAXIMUM_LEADBYTES; i++)
+    {
+        printf("LeadByte[%u] = 0x%02X\n", i, header->LeadByte[i]);
+    }
+
+    printf("\n");
+}
+
+static void
+nls_print_mb_table(uint16_t *mb_table, uint16_t uni_default_char)
+{
+    uint32_t ch;
+
+    printf("MBTABLE:\n");
+
+    for (ch = 0; ch <= 0xFF; ch++)
+    {
+        if (mb_table[ch] != uni_default_char)
+        {
+            printf("0x%02X 0x%04X\n", (unsigned int)ch, (unsigned int)mb_table[ch]);
+        }
+    }
+
+    printf("\n");
+}
+
+static void
+nls_print_wc_table(uint16_t *wc_table, uint16_t default_char, int is_dbcs)
+{
+    uint32_t ch;
+
+    printf("WCTABLE:\n");
+
+    for (ch = 0; ch <= 0xFFFF; ch++)
+    {
+        /* DBCS code page */
+        if (is_dbcs)
+        {
+            uint16_t *table = (uint16_t*)wc_table;
+
+            if (table[ch] != default_char)
+                printf("0x%04X 0x%04X\n", (unsigned int)ch, (unsigned int)table[ch]);
+        }
+        /* SBCS code page */
+        else
+        {
+            uint8_t *table = (uint8_t*)wc_table;
+
+            if (table[ch] != default_char)
+                printf("0x%04X 0x%02X\n", (unsigned int)ch, (unsigned int)table[ch]);
+        }
+    }
+
+    printf("\n");
+}
+
+static void
+nls_print_glyph_table(uint16_t *glyph_table, uint16_t uni_default_char)
+{
+    uint32_t ch;
+
+    printf("GLYPHTABLE:\n");
+
+    for (ch = 0; ch <= 0xFF; ch++)
+    {
+        if (glyph_table[ch] != uni_default_char)
+        {
+            printf("0x%02X 0x%04X\n", (unsigned int)ch, (unsigned int)glyph_table[ch]);
+        }
+    }
+
+    printf("\n");
+}
+
+#endif /* _NLS_DEBUG_PRINT */
+
+int
+nls_from_txt(const char *txt_file_path, const char *nls_file_path)
+{
+    NLS_FILE_HEADER header;
+    FILE *file = NULL;
+    uint16_t *mb_table = NULL;
+    uint16_t *wc_table = NULL;
+    uint16_t *glyph_table = NULL;
+    uint16_t number_of_lb_ranges;
+    uint16_t size;
+    int is_dbcs;
+    int res = 0;
+
+    memset(&header, 0, sizeof(header));
+
+    if (!txt_get_header(txt_file_path, &header))
+        goto Cleanup;
+
+    is_dbcs = (header.MaximumCharacterSize == 2) ? 1 : 0;
+
+    mb_table = txt_get_mb_table(txt_file_path, header.UniDefaultChar);
+    if (!mb_table)
+        goto Cleanup;
+
+    wc_table = txt_get_wc_table(txt_file_path, header.DefaultChar, is_dbcs);
+    if (!wc_table)
+        goto Cleanup;
+
+    /* GLYPHTABLE optionally. We do not leave if it is absent */
+    glyph_table = txt_get_glyph_table(txt_file_path, header.UniDefaultChar);
+
+#ifdef _NLS_DEBUG_PRINT
+    nls_print_header(&header);
+    nls_print_mb_table(mb_table, header.UniDefaultChar);
+    if (glyph_table)
+        nls_print_glyph_table(glyph_table, header.UniDefaultChar);
+    nls_print_wc_table(wc_table, header.DefaultChar, is_dbcs);
+#endif /* _NLS_DEBUG_PRINT */
+
+    /* Create binary file with write access */
+    file = fopen(nls_file_path, "wb");
+    if (!file)
+    {
+        printf("Unable to create NLS file.\n");
+        goto Cleanup;
+    }
+
+    /* Write NLS file header */
+    if (fwrite(&header, 1, sizeof(header), file) != sizeof(header))
+    {
+        printf("Unable to write NLS file.\n");
+        goto Cleanup;
+    }
+
+    size = (256 * sizeof(uint16_t)) + /* Primary CP to Unicode table */
+           sizeof(uint16_t) + /* optional OEM glyph table size in words */
+           (glyph_table ? (256 * sizeof(uint16_t)) : 0) + /* OEM glyph table size in words * sizeof(uint16_t) */
+           sizeof(uint16_t) + /* Number of DBCS LeadByte ranges */
+           0 + /* offsets of lead byte sub tables */
+           0 + /* LeadByte sub tables */
+           sizeof(uint16_t); /* Unknown flag */
+
+    size /= sizeof(uint16_t);
+
+    if (fwrite(&size, 1, sizeof(size), file) != sizeof(size))
+    {
+        printf("Unable to write NLS file.\n");
+        goto Cleanup;
+    }
+
+    /* Write multibyte table */
+    if (fwrite(mb_table, 1, (256 * sizeof(uint16_t)), file) != (256 * sizeof(uint16_t)))
+    {
+        printf("Unable to write NLS file.\n");
+        goto Cleanup;
+    }
+
+    /* OEM glyph table size in words */
+    size = (glyph_table ? (256 * sizeof(uint16_t)) : 0);
+
+    if (fwrite(&size, 1, sizeof(size), file) != sizeof(size))
+    {
+        printf("Unable to write NLS file.\n");
+        goto Cleanup;
+    }
+
+    if (glyph_table)
+    {
+        /* Write OEM glyph table */
+        if (fwrite(glyph_table, 1, (256 * sizeof(uint16_t)), file) != (256 * sizeof(uint16_t)))
+        {
+            printf("Unable to write NLS file.\n");
+            goto Cleanup;
+        }
+    }
+
+    /* Number of DBCS LeadByte ranges */
+    number_of_lb_ranges = 0;
+    if (fwrite(&number_of_lb_ranges, 1, sizeof(number_of_lb_ranges), file) != sizeof(number_of_lb_ranges))
+    {
+        printf("Unable to write NLS file.\n");
+        goto Cleanup;
+    }
+
+    /* Unknown flag */
+    size = 0;
+    if (fwrite(&size, 1, sizeof(size), file) != sizeof(size))
+    {
+        printf("Unable to write NLS file.\n");
+        goto Cleanup;
+    }
+
+    /* Write wide char table */
+    if (fwrite(wc_table, 1, (65535 * header.MaximumCharacterSize), file) != (65535 * header.MaximumCharacterSize))
+    {
+        printf("Unable to write NLS file.\n");
+        goto Cleanup;
+    }
+
+    res = 1;
+
+Cleanup:
+    if (file) fclose(file);
+    free(mb_table);
+    free(wc_table);
+    free(glyph_table);
+
+    return res;
+}

rosapps/applications/devutils/txt2nls/precomp.h

@@ -0,0 +1,49 @@
+/*
+ * PROJECT:     ReactOS TXT to NLS Converter
+ * LICENSE:     GNU General Public License Version 2.0 or any later version
+ * FILE:        devutils/txt2nls/precomp.h
+ * COPYRIGHT:   Copyright 2016 Dmitry Chapyshev <dmitry@reactos.org>
+ */
+
+#ifndef __PRECOMP_H
+#define __PRECOMP_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <ctype.h>
+#include <memory.h>
+
+#define MAXIMUM_LEADBYTES   12
+
+typedef struct
+{
+    uint16_t HeaderSize;
+    uint16_t CodePage;
+    uint16_t MaximumCharacterSize;
+    uint16_t DefaultChar;
+    uint16_t UniDefaultChar;
+    uint16_t TransDefaultChar;
+    uint16_t TransUniDefaultChar;
+    uint8_t LeadByte[MAXIMUM_LEADBYTES];
+} NLS_FILE_HEADER;
+
+/* nls.c */
+int
+nls_from_txt(const char *txt_file_path, const char *nls_file_path);
+
+/* bestfit.c */
+int
+txt_get_header(const char *file_path, NLS_FILE_HEADER *header);
+
+uint16_t*
+txt_get_mb_table(const char *file_path, uint16_t uni_default_char);
+
+uint16_t*
+txt_get_wc_table(const char *file_path, uint16_t default_char, int is_dbcs);
+
+uint16_t*
+txt_get_glyph_table(const char *file_path, uint16_t uni_default_char);
+
+#endif

rosapps/applications/devutils/txt2nls/txt.c

@@ -0,0 +1,495 @@
+/*
+ * PROJECT:     ReactOS TXT to NLS Converter
+ * LICENSE:     GNU General Public License Version 2.0 or any later version
+ * FILE:        devutils/txt2nls/txt.c
+ * COPYRIGHT:   Copyright 2016 Dmitry Chapyshev <dmitry@reactos.org>
+ */
+
+#include "precomp.h"
+
+int
+txt_get_header(const char *file_path, NLS_FILE_HEADER *header)
+{
+    FILE *file;
+    char *p;
+    char buf[256];
+    uint32_t line = 0;
+    int res = 0;
+    int found;
+    uint32_t val;
+
+    file = fopen(file_path, "r");
+    if (!file)
+    {
+        printf("Unable to read TXT file.\n");
+        return 0;
+    }
+
+    /* Find CODEPAGE entry */
+    found = 0;
+    while (fgets(buf, sizeof(buf), file))
+    {
+        ++line;
+
+        p = strstr(buf, "CODEPAGE");
+        if (p)
+        {
+            /* Length of CODEPAGE string is 8 chars */
+            p += 8;
+
+            /* Skip all spaces after CODEPAGE */
+            while (isspace(*p)) ++p;
+
+            /* Convert string to uint32_t */
+            val = strtoul(p, &p, 10);
+
+            /* Validate codepage value */
+            if (val > 0xFFFF)
+            {
+                printf("Wrong codepage: %u (line: %u)\n", val, line);
+                goto Cleanup;
+            }
+
+            header->CodePage = (uint16_t)val;
+
+            found = 1;
+            break;
+        }
+    }
+
+    if (!found)
+    {
+        printf("CODEPAGE not found.\n");
+        goto Cleanup;
+    }
+
+    /* Find CPINFO entry */
+    found = 0;
+    while (fgets(buf, sizeof(buf), file))
+    {
+        ++line;
+
+        p = strstr(buf, "CPINFO");
+        if (p)
+        {
+            /* Length of CPINFO string is 6 chars */
+            p += 6;
+
+            /* Skip all spaces after CPINFO */
+            while (isspace(*p)) ++p;
+
+            /* Convert string to uint32_t */
+            val = strtoul(p, &p, 10);
+
+            /* Validate value */
+            if (val != 1 && val != 2)
+            {
+                printf("Wrong character size: %u (line: %u)\n", val, line);
+                goto Cleanup;
+            }
+
+            header->MaximumCharacterSize = (uint16_t)val;
+
+            /* Skip all spaces after character size */
+            while (isspace(*p)) ++p;
+
+            /* Convert string to uint32_t */
+            val = strtoul(p, &p, 16);
+            header->DefaultChar = (uint16_t)val;
+            header->TransDefaultChar = (uint16_t)val;
+
+            /* Skip all spaces after default char */
+            while (isspace(*p)) ++p;
+
+            /* Convert string to uint32_t */
+            val = strtoul(p, &p, 16);
+            header->UniDefaultChar = (uint16_t)val;
+            header->TransUniDefaultChar = (uint16_t)val;
+
+            found = 1;
+            break;
+        }
+    }
+
+    if (!found)
+    {
+        printf("CPINFO not found.\n");
+        goto Cleanup;
+    }
+
+    header->HeaderSize = sizeof(NLS_FILE_HEADER) / sizeof(uint16_t);
+
+    res = 1;
+
+Cleanup:
+    fclose(file);
+
+    return res;
+}
+
+uint16_t*
+txt_get_mb_table(const char *file_path, uint16_t uni_default_char)
+{
+    uint16_t *table;
+    char buf[256];
+    char *p;
+    uint32_t count = 0;
+    uint32_t index;
+    uint32_t line = 0;
+    int found;
+    int res = 0;
+    FILE *file;
+
+    table = malloc(0xFF * sizeof(uint16_t));
+    if (!table)
+    {
+        printf("Memory allocation failure\n");
+        return NULL;
+    }
+
+    /* Set default value for all table items */
+    for (index = 0; index <= 0xFF; index++)
+        table[index] = uni_default_char;
+
+    file = fopen(file_path, "r");
+    if (!file)
+    {
+        printf("Unable to read TXT file.\n");
+        goto Cleanup;
+    }
+
+    /* Find MBTABLE entry */
+    found = 0;
+    while (fgets(buf, sizeof(buf), file))
+    {
+        ++line;
+
+        p = strstr(buf, "MBTABLE");
+        if (p)
+        {
+            p += 7;
+
+            /* Skip spaces */
+            while (isspace(*p)) ++p;
+
+            count = strtoul(p, &p, 10);
+            if (count == 0 || count > 256)
+            {
+                printf("Wrong MBTABLE size: %u (line: %u)\n", count, line);
+                goto Cleanup;
+            }
+
+            found = 1;
+            break;
+        }
+    }
+
+    if (!found)
+    {
+        printf("MBTABLE not found.\n");
+        goto Cleanup;
+    }
+
+    /* Parse next line */
+    while (fgets(buf, sizeof(buf), file) && count)
+    {
+        uint32_t cp_char;
+        uint32_t uni_char;
+
+        ++line;
+
+        p = buf;
+
+        /* Skip spaces */
+        while (isspace(*p)) ++p;
+
+        if (!*p || p[0] == ';')
+            continue;
+
+        cp_char = strtoul(p, &p, 16);
+        if (cp_char > 0xFF)
+        {
+            printf("Wrong char value: %u (line: %u)\n", cp_char, line);
+            goto Cleanup;
+        }
+
+        /* Skip spaces */
+        while (isspace(*p)) ++p;
+
+        uni_char = strtoul(p, &p, 16);
+        if (uni_char > 0xFFFF)
+        {
+            printf("Wrong unicode char value: %u (line: %u)\n", uni_char, line);
+            goto Cleanup;
+        }
+
+        table[cp_char] = uni_char;
+        --count;
+    }
+
+    res = 1;
+
+Cleanup:
+    if (!res)
+    {
+        free(table);
+        table = NULL;
+    }
+
+    fclose(file);
+
+    return table;
+}
+
+uint16_t*
+txt_get_wc_table(const char *file_path, uint16_t default_char, int is_dbcs)
+{
+    char buf[256];
+    char *p;
+    uint16_t *table;
+    uint32_t index;
+    uint32_t count = 0;
+    uint32_t line = 0;
+    int res = 0;
+    int found;
+    FILE *file;
+
+    table = malloc(0xFFFF * (is_dbcs ? sizeof(uint16_t) : sizeof(uint8_t)));
+    if (!table)
+    {
+        printf("Memory allocation failure\n");
+        return NULL;
+    }
+
+    /* Set default value for all table items */
+    for (index = 0; index <= 0xFFFF; index++)
+    {
+        /* DBCS code page */
+        if (is_dbcs)
+        {
+            uint16_t *tmp = (uint16_t*)table;
+            tmp[index] = default_char;
+        }
+        /* SBCS code page */
+        else
+        {
+            uint8_t *tmp = (uint8_t*)table;
+            tmp[index] = default_char;
+        }
+    }
+
+    file = fopen(file_path, "r");
+    if (!file)
+    {
+        printf("Unable to read TXT file.\n");
+        goto Cleanup;
+    }
+
+    /* Find WCTABLE entry */
+    found = 0;
+    while (fgets(buf, sizeof(buf), file))
+    {
+        ++line;
+
+        p = strstr(buf, "WCTABLE");
+        if (p)
+        {
+            p += 7;
+
+            /* Skip spaces */
+            while (isspace(*p)) ++p;
+
+            count = strtoul(p, &p, 10);
+            if (count == 0 || count > 65536)
+            {
+                printf("Wrong WCTABLE size: %u (line: %u)\n", count, line);
+                goto Cleanup;
+            }
+
+            found = 1;
+            break;
+        }
+    }
+
+    if (!found)
+    {
+        printf("WCTABLE not found.\n");
+        goto Cleanup;
+    }
+
+    /* Parse next line */
+    while (fgets(buf, sizeof(buf), file) && count)
+    {
+        uint32_t cp_char;
+        uint32_t uni_char;
+
+        ++line;
+
+        p = buf;
+
+        /* Skip spaces */
+        while (isspace(*p)) ++p;
+
+        if (!*p || p[0] == ';')
+            continue;
+
+        uni_char = strtoul(p, &p, 16);
+        if (uni_char > 0xFFFF)
+        {
+            printf("Wrong unicode char value: %u (line: %u)\n", uni_char, line);
+            goto Cleanup;
+        }
+
+        /* Skip spaces */
+        while (isspace(*p)) ++p;
+
+        cp_char = strtoul(p, &p, 16);
+        if ((is_dbcs && cp_char > 0xFFFF) || (!is_dbcs && cp_char > 0xFF))
+        {
+            printf("Wrong char value: %u (line: %u)\n", cp_char, line);
+            goto Cleanup;
+        }
+
+        /* DBCS code page */
+        if (is_dbcs)
+        {
+            uint16_t *tmp = (uint16_t*)table;
+            tmp[uni_char] = cp_char;
+        }
+        /* SBCS code page */
+        else
+        {
+            uint8_t *tmp = (uint8_t*)table;
+            tmp[uni_char] = cp_char;
+        }
+
+        --count;
+    }
+
+    res = 1;
+
+Cleanup:
+    if (!res)
+    {
+        free(table);
+        table = NULL;
+    }
+
+    fclose(file);
+
+    return table;
+}
+
+uint16_t*
+txt_get_glyph_table(const char *file_path, uint16_t uni_default_char)
+{
+    uint16_t *table;
+    char buf[256];
+    char *p;
+    uint32_t count = 0;
+    uint32_t index;
+    uint32_t line = 0;
+    int found;
+    int res = 0;
+    FILE *file;
+
+    table = malloc(0xFF * sizeof(uint16_t));
+    if (!table)
+    {
+        printf("Memory allocation failure\n");
+        return NULL;
+    }
+
+    /* Set default value for all table items */
+    for (index = 0; index <= 0xFF; index++)
+        table[index] = uni_default_char;
+
+    file = fopen(file_path, "r");
+    if (!file)
+    {
+        printf("Unable to read TXT file.\n");
+        goto Cleanup;
+    }
+
+    /* Find GLYPHTABLE entry */
+    found = 0;
+    while (fgets(buf, sizeof(buf), file))
+    {
+        ++line;
+
+        p = strstr(buf, "GLYPHTABLE");
+        if (p)
+        {
+            p += 10;
+
+            /* Skip spaces */
+            while (isspace(*p)) ++p;
+
+            count = strtoul(p, &p, 10);
+            if (count == 0 || count > 256)
+            {
+                printf("Wrong GLYPHTABLE size: %u (line: %u)\n", count, line);
+                goto Cleanup;
+            }
+
+            found = 1;
+            break;
+        }
+    }
+
+    if (!found)
+    {
+        printf("GLYPHTABLE not found.\n");
+        goto Cleanup;
+    }
+
+    /* Parse next line */
+    while (fgets(buf, sizeof(buf), file) && count)
+    {
+        uint32_t cp_char;
+        uint32_t uni_char;
+
+        ++line;
+
+        p = buf;
+
+        /* Skip spaces */
+        while (isspace(*p)) ++p;
+
+        if (!*p || p[0] == ';')
+            continue;
+
+        cp_char = strtoul(p, &p, 16);
+        if (cp_char > 0xFF)
+        {
+            printf("Wrong char value: %u (line: %u)\n", cp_char, line);
+            goto Cleanup;
+        }
+
+        /* Skip spaces */
+        while (isspace(*p)) ++p;
+
+        uni_char = strtoul(p, &p, 16);
+        if (uni_char > 0xFFFF)
+        {
+            printf("Wrong unicode char value: %u (line: %u)\n", uni_char, line);
+            goto Cleanup;
+        }
+
+        table[cp_char] = uni_char;
+        --count;
+    }
+
+    res = 1;
+
+Cleanup:
+    if (!res)
+    {
+        free(table);
+        table = NULL;
+    }
+
+    fclose(file);
+
+    return table;
+}

rosapps/applications/devutils/txt2nls/txt2nls.rc

@@ -0,0 +1,18 @@
+/*
+ * PROJECT:     ReactOS TXT to NLS Converter
+ * LICENSE:     GNU General Public License Version 2.0 or any later version
+ * FILE:        devutils/txt2nls/txt2nls.rc
+ * COPYRIGHT:   Copyright 2016 Dmitry Chapyshev <dmitry@reactos.org>
+ */
+
+#include <windows.h>
+#include <commctrl.h>
+
+LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL
+
+#define REACTOS_STR_FILE_DESCRIPTION    "ReactOS TXT to NLS Converter"
+#define REACTOS_STR_INTERNAL_NAME       "txt2nls"
+#define REACTOS_STR_ORIGINAL_FILENAME   "txt2nls.exe"
+#include <reactos/version.rc>
+
+#include <reactos/manifest_exe.rc>