[APPHELP] Implement automatic stringtable generation when writing an Sdb database. CORE-10367

svn path=/trunk/; revision=71614

reactos/dll/appcompat/apphelp/CMakeLists.txt

@@ -5,9 +5,10 @@ list(APPEND SOURCE
     apphelp.c
     layer.c
     sdbapi.c
-    sdbread.c
-    sdbwrite.c
     sdbfileattr.c
+    sdbread.c
+    sdbstringtable.c
+    sdbwrite.c
     apphelp.spec
     apphelp.h
     ${CMAKE_CURRENT_BINARY_DIR}/apphelp_stubs.c)

reactos/dll/appcompat/apphelp/apphelp.h

@@ -50,26 +50,11 @@ typedef struct tagATTRINFO {
     };
 } ATTRINFO, *PATTRINFO;
 
-typedef enum _SHIM_LOG_LEVEL {
-    SHIM_ERR = 1,
-    SHIM_WARN = 2,
-    SHIM_INFO = 3,
-}SHIM_LOG_LEVEL;
-
 /* apphelp.c */
-BOOL WINAPIV ShimDbgPrint(SHIM_LOG_LEVEL Level, PCSTR FunctionName, PCSTR Format, ...);
-extern ULONG g_ShimDebugLevel;
-
-#define SHIM_ERR(fmt, ...)  do { if (g_ShimDebugLevel) ShimDbgPrint(SHIM_ERR, __FUNCTION__, fmt, ##__VA_ARGS__ ); } while (0)
-#define SHIM_WARN(fmt, ...)  do { if (g_ShimDebugLevel) ShimDbgPrint(SHIM_WARN, __FUNCTION__, fmt, ##__VA_ARGS__ ); } while (0)
-#define SHIM_INFO(fmt, ...)  do { if (g_ShimDebugLevel) ShimDbgPrint(SHIM_INFO, __FUNCTION__, fmt, ##__VA_ARGS__ ); } while (0)
-
 
 #include "sdbpapi.h"
 
 PWSTR SdbpStrDup(LPCWSTR string);
-BOOL WINAPI SdbpCheckTagType(TAG tag, WORD type);
-BOOL WINAPI SdbpCheckTagIDType(PDB db, TAGID tagid, WORD type);
 
 PDB WINAPI SdbOpenDatabase(LPCWSTR path, PATH_TYPE type);
 void WINAPI SdbCloseDatabase(PDB);

reactos/dll/appcompat/apphelp/sdbapi.c

@@ -23,6 +23,7 @@
 #include "ntndk.h"
 #include "strsafe.h"
 #include "apphelp.h"
+#include "sdbstringtable.h"
 
 #include "wine/unicode.h"
 
@@ -228,9 +229,6 @@ PDB WINAPI SdbpCreate(LPCWSTR path, PATH_TYPE type, BOOL write)
 
     InitializeObjectAttributes(&attr, &str, OBJ_CASE_INSENSITIVE, NULL, NULL);
 
-    //Status = NtCreateFile(&db->file, FILE_GENERIC_READ | SYNCHRONIZE,
-    //                      &attr, &io, NULL, FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ,
-    //                      FILE_OPEN, FILE_NON_DIRECTORY_FILE | FILE_SYNCHRONOUS_IO_NONALERT, NULL, 0);
     Status = NtCreateFile(&db->file, (write ? FILE_GENERIC_WRITE : FILE_GENERIC_READ )| SYNCHRONIZE,
                           &attr, &io, NULL, FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ,
                           write ? FILE_SUPERSEDE : FILE_OPEN, FILE_NON_DIRECTORY_FILE | FILE_SYNCHRONOUS_IO_NONALERT, NULL, 0);
@@ -454,6 +452,10 @@ void WINAPI SdbCloseDatabase(PDB db)
 
     if (db->file)
         NtClose(db->file);
+    if (db->string_buffer)
+        SdbCloseDatabase(db->string_buffer);
+    if (db->string_lookup)
+        SdbpTableDestroy(&db->string_lookup);
     SdbFree(db->data);
     SdbFree(db);
 }

reactos/dll/appcompat/apphelp/sdbpapi.h

@@ -68,6 +68,22 @@ void WINAPI SdbpFlush(PDB db);
 DWORD SdbpStrlen(PCWSTR string);
 DWORD SdbpStrsize(PCWSTR string);
 
+BOOL WINAPI SdbpCheckTagType(TAG tag, WORD type);
+BOOL WINAPI SdbpCheckTagIDType(PDB db, TAGID tagid, WORD type);
+
+
+typedef enum _SHIM_LOG_LEVEL {
+    SHIM_ERR = 1,
+    SHIM_WARN = 2,
+    SHIM_INFO = 3,
+} SHIM_LOG_LEVEL;
+
+BOOL WINAPIV ShimDbgPrint(SHIM_LOG_LEVEL Level, PCSTR FunctionName, PCSTR Format, ...);
+extern ULONG g_ShimDebugLevel;
+
+#define SHIM_ERR(fmt, ...)  do { if (g_ShimDebugLevel) ShimDbgPrint(SHIM_ERR, __FUNCTION__, fmt, ##__VA_ARGS__ ); } while (0)
+#define SHIM_WARN(fmt, ...)  do { if (g_ShimDebugLevel) ShimDbgPrint(SHIM_WARN, __FUNCTION__, fmt, ##__VA_ARGS__ ); } while (0)
+#define SHIM_INFO(fmt, ...)  do { if (g_ShimDebugLevel) ShimDbgPrint(SHIM_INFO, __FUNCTION__, fmt, ##__VA_ARGS__ ); } while (0)
 
 #ifdef __cplusplus
 } // extern "C"

reactos/dll/appcompat/apphelp/sdbstringtable.c

@@ -0,0 +1,157 @@
+/*
+ * Copyright 2016 Mark Jansen
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+ */
+
+#if !defined(SDBWRITE_HOSTTOOL)
+#define WIN32_NO_STATUS
+#include "windows.h"
+#include "sdbtypes.h"
+#include "sdbpapi.h"
+#else /* !defined(SDBWRITE_HOSTTOOL) */
+#include <typedefs.h>
+#include "sdbtypes.h"
+#include "sdbpapi.h"
+#endif /* !defined(SDBWRITE_HOSTTOOL) */
+
+#include "sdbstringtable.h"
+
+#define DEFAULT_TABLE_SIZE    0x100
+
+typedef struct SdbHashEntry
+{
+    struct SdbHashEntry* Next;
+    TAGID Tagid;
+    WCHAR Name[1];
+} SdbHashEntry;
+
+struct SdbStringHashTable
+{
+    DWORD Size;
+    struct SdbHashEntry** Entries;
+};
+
+
+static struct SdbStringHashTable* HashCreate(void)
+{
+    struct SdbStringHashTable* tab = SdbAlloc(sizeof(*tab));
+    if (!tab)
+    {
+        SHIM_ERR("Failed to allocate 8 bytes.\r\n");
+        return tab;
+    }
+    tab->Size = DEFAULT_TABLE_SIZE;
+    tab->Entries = SdbAlloc(tab->Size * sizeof(*tab->Entries));
+    return tab;
+}
+
+
+void SdbpTableDestroy(struct SdbStringHashTable** pTable)
+{
+    struct SdbStringHashTable* table = *pTable;
+    struct SdbHashEntry* entry, *next;
+    DWORD n, depth = 0, once = 1;
+
+    *pTable = NULL;
+    for (n = 0; n < table->Size; ++n)
+    {
+        depth = 0;
+        entry = next = table->Entries[n];
+        while (entry)
+        {
+            next = entry->Next;
+            SdbFree(entry);
+            entry = next;
+            depth++;
+        }
+        if (once && depth > 3)
+        {
+            // warn
+            once = 0;
+        }
+    }
+    SdbFree(table->Entries);
+    SdbFree(table);
+}
+
+/* Based on RtlHashUnicodeString */
+static DWORD StringHash(const WCHAR* str)
+{
+    DWORD hash = 0;
+    for (; *str; str++)
+    {
+        hash = ((65599 * hash) + (ULONG)(*str));
+    }
+    return hash;
+}
+
+static struct SdbHashEntry** TableFindPtr(struct SdbStringHashTable* table, const WCHAR* str)
+{
+    DWORD hash = StringHash(str);
+    struct SdbHashEntry** entry = &table->Entries[hash % table->Size];
+    while (*entry)
+    {
+        if (!wcscmp((*entry)->Name, str))
+            return entry;
+        entry = &(*entry)->Next;
+    }
+    return entry;
+}
+
+static BOOL HashAddString(struct SdbStringHashTable* table, struct SdbHashEntry** position, const WCHAR* str, TAGID tagid)
+{
+    struct SdbHashEntry* entry;
+    SIZE_T size;
+
+    if (!position)
+        position = TableFindPtr(table, str);
+
+    size = offsetof(struct SdbHashEntry, Name[SdbpStrlen(str) + 2]);
+    entry = (*position) = SdbAlloc(size);
+    if (!entry)
+    {
+        SHIM_ERR("Failed to allocate %u bytes.", size);
+        return FALSE;
+    }
+    entry->Tagid = tagid;
+    wcscpy(entry->Name, str);
+    return TRUE;
+}
+
+
+BOOL SdbpAddStringToTable(struct SdbStringHashTable** table, const WCHAR* str, TAGID* tagid)
+{
+    struct SdbHashEntry** entry;
+
+    if (!*table)
+    {
+        *table = HashCreate();
+        if (!*table)
+        {
+            SHIM_ERR("Error creating hash table\n");
+            return FALSE;
+        }
+    }
+
+    entry = TableFindPtr(*table, str);
+    if (*entry)
+    {
+        *tagid = (*entry)->Tagid;
+        return FALSE;
+    }
+    return HashAddString(*table, entry, str, *tagid);
+}
+

reactos/dll/appcompat/apphelp/sdbstringtable.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2016 Mark Jansen
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+ */
+
+#ifndef SDBSTRINGTABLE_H
+#define SDBSTRINGTABLE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/**
+ * Destroy the hashtable and release all resources.
+ *
+ * @param [in]  table       Pointer to table pointer, will be cleared after use
+ *
+ */
+void SdbpTableDestroy(struct SdbStringHashTable* * table);
+
+/**
+ * Find an entry in the stringtable, or allocate it when an entry could not be found.
+ * - When the string specified does not yet exist, a new entry will be added to the table,
+ *   and the pTagid specified will be associated with this string.
+ * - When the string specified does already exist,
+ *   the TAGID associated with this string will be returned in pTagid.
+ * 
+ *
+ * @param [in]  table       Pointer to table pointer, will be allocated when needed.
+ * @param [in]  str         The string to search for
+ * @param [in,out] pTagid   
+ *                          the data written (in bytes)
+ *
+ * @return  TRUE if the string was added to the table, FALSE if it already existed
+ */
+BOOL SdbpAddStringToTable(struct SdbStringHashTable* * table, const WCHAR* str, TAGID* pTagid);
+
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // SDBSTRINGTABLE_H

reactos/dll/appcompat/apphelp/sdbtypes.h

@@ -39,6 +39,8 @@ typedef struct _DB {
     TAGID stringtable;
     DWORD write_iter;
     GUID database_id;
+    struct SdbStringHashTable* string_lookup;
+    struct _DB* string_buffer;
 } DB, *PDB;
 
 typedef enum _PATH_TYPE {

reactos/dll/appcompat/apphelp/sdbwrite.c

@@ -1,6 +1,6 @@
 /*
- * Copyright 2011 André Hentschel
- * Copyright 2013 Mislav Blaževic
+ * Copyright 2011 André Hentschel
+ * Copyright 2013 Mislav Blažević
  * Copyright 2015,2016 Mark Jansen
  *
  * This library is free software; you can redistribute it and/or
@@ -19,28 +19,29 @@
  */
 
 #if !defined(SDBWRITE_HOSTTOOL)
-
 #define WIN32_NO_STATUS
 #include "windows.h"
 #include "ntndk.h"
-#include "apphelp.h"
-#include "wine/unicode.h"
-
 #else
-
 #include <typedefs.h>
 #include <guiddef.h>
+#endif
 
 #include "sdbtypes.h"
 #include "sdbpapi.h"
 #include "sdbtagid.h"
+#include "sdbstringtable.h"
 
-#endif
 
+/* Local functions */
 BOOL WINAPI SdbWriteStringRefTag(PDB db, TAG tag, TAGID tagid);
+BOOL WINAPI SdbWriteStringTag(PDB db, TAG tag, LPCWSTR string);
 TAGID WINAPI SdbBeginWriteListTag(PDB db, TAG tag);
 BOOL WINAPI SdbEndWriteListTag(PDB db, TAGID tagid);
 
+/* sdbapi.c */
+void WINAPI SdbCloseDatabase(PDB);
+
 
 static void WINAPI SdbpWrite(PDB db, const void* data, DWORD size)
 {
@@ -57,19 +58,35 @@ static void WINAPI SdbpWrite(PDB db, const void* data, DWORD size)
 
 static BOOL WINAPI SdbpGetOrAddStringRef(PDB db, LPCWSTR string, TAGID* tagid)
 {
-    /* TODO:
-    - Insert or find in stringtable
-    - return TAGID
-    */
+    PDB buf = db->string_buffer;
+    if (db->string_buffer == NULL)
+    {
+        db->string_buffer = buf = SdbpAlloc(sizeof(DB));
+        if (buf == NULL)
+            return FALSE;
+        buf->size = 128;
+        buf->data = SdbAlloc(buf->size);
+        if (buf->data == NULL)
+            return FALSE;
+    }
 
-    return FALSE;
+   *tagid = buf->write_iter + sizeof(TAG) + sizeof(DWORD);
+   if (SdbpAddStringToTable(&db->string_lookup, string, tagid))
+       return SdbWriteStringTag(buf, TAG_STRINGTABLE_ITEM, string);
+
+    return db->string_lookup != NULL;
 }
 
-static void WINAPI SdbpWriteStringtable(PDB db)
+static BOOL WINAPI SdbpWriteStringtable(PDB db)
 {
-    TAGID table = SdbBeginWriteListTag(db, TAG_STRINGTABLE);
-    /* TODO: Write out all strings*/
-    SdbEndWriteListTag(db, table);
+    TAGID table;
+    PDB buf = db->string_buffer;
+    if (buf == NULL || db->string_lookup == NULL)
+        return FALSE;
+
+    table = SdbBeginWriteListTag(db, TAG_STRINGTABLE);
+    SdbpWrite(db, buf->data, buf->write_iter);
+    return SdbEndWriteListTag(db, table);
 }
 
 /**

reactos/dll/appcompat/hacking.txt

@@ -0,0 +1,23 @@
+
+Set the environment variable 'SHIM_DEBUG_LEVEL' to '4' before loading apphelp.dll for debug info related to loading / matching modules.
+The environment variable 'SHIMENG_DEBUG_LEVEL' is related to the hooking code.
+
+
+When there is not enough debug output, force memory allocations to fail:
+
+pSdbCreateDatabase
+    [Err ][SdbpCreateFile      ] Failed to convert DOS path "TEST1.SDB"
+    [Err ][SdbCreateDatabase   ] Failed to create the database.
+    [Err ][SdbpWriteBufferedData] Failed to allocate 65548 bytes.
+
+pSdbWriteStringTag
+    [Err ][SdbpAddStringToTable] Error Gettting temp path 0x8
+    [Err ][SdbpCreateFile      ] Failed to convert DOS path "C:\Users\MAE67~1.JAN\AppData\Local\Temp\SDBAB16.tmp"
+    [Err ][SdbpAddStringToTable] Error copying string table temp filename
+    [Err ][HashCreate          ] Failed to allocate 8 bytes.
+    [Err ][SdbpAddStringToTable] Error creating hash table
+    [Err ][HashAddString       ] Failed to allocate 22 bytes.
+
+pSdbCloseDatabaseWrite
+    [Err ][SdbCloseDatabase    ] Failed to close the file.
+    [Err ][SdbpDeleteFile      ] Failed to convert DOS path "C:\Users\MAE67~1.JAN\AppData\Local\Temp\SDBAB2D.tmp"