modified nls/3rdparty/icu4ros.rbuild

- Build fixes

added      nls/3rdparty/stubs.cpp
 - Stub out unused ICU dependencies

added      nls/dll/idndl
added      nls/dll/idndl/idndl.cpp
added      nls/dll/idndl/idndl.def
added      nls/dll/idndl/idndl.rbuild
 - Implemented idndl.dll forwarder

added      nls/dll/idndl_redist
added      nls/dll/idndl_redist/data
added      nls/dll/idndl_redist/data/genidldndata.cpp
added      nls/dll/idndl_redist/data/idldata.cpp
added      nls/dll/idndl_redist/idndl.cpp
added      nls/dll/idndl_redist/idndl.def
added      nls/dll/idndl_redist/idndl_redist.rbuild
added      nls/dll/idndl_redist/stubs.cpp
 - Implemented stand-alone idndl.dll
 - Implemented DownlevelGetLocaleScripts, DownlevelGetStringScripts, DownlevelVerifyScripts

added      nls/dll/normaliz
added      nls/dll/normaliz/dummy.c
added      nls/dll/normaliz/normaliz.def
added      nls/dll/normaliz/normaliz.rbuild
 - Implemented forwarder normaliz.dll

added      nls/dll/normaliz_redist/data
added      nls/dll/normaliz_redist/data/icudt38.c
added      nls/dll/normaliz_redist/data/icudt38.lst
modified   nls/dll/normaliz_redist/normaliz.cpp
modified   nls/dll/normaliz_redist/normaliz.def
modified   nls/dll/normaliz_redist/normaliz_redist.rbuild
 - Implemented stand-alone normaliz.dll

added      nls/lib/idna
added      nls/lib/idna/idna.cpp
added      nls/lib/idna/idna.rbuild
added      nls/lib/idna/nameprep.cpp
 - Implemented IdnToAscii, IdnToUnicode, IdnToNameprepUnicode

modified   nls/lib/normalize/normalize.cpp
 - Implemented NormalizeString, IsNormalizedString

added      nls/lib/scripts
added      nls/lib/scripts/scripts.cpp
added      nls/lib/scripts/scripts.h
added      nls/lib/scripts/scripts.rbuild
 - Implemented GetStringScripts, VerifyScripts

added      nls/media
 - Added directory for future data files

modified   nls/nls.rbuild
 - Build all NLS modules

added      nls/tests
added      nls/tests/normalization
added      nls/tests/normalization/normalizationTest.c
added      nls/tests/normalization/normalizationTest.h
added      nls/tests/normalization/normalizationTest.l
added      nls/tests/normalization/normalizationTest.rbuild
added      nls/tests/normalization/normalizationTest.sln
added      nls/tests/normalization/normalizationTest.tab.c
added      nls/tests/normalization/normalizationTest.tab.h
added      nls/tests/normalization/NormalizationTest.txt
added      nls/tests/normalization/normalizationTest.vcproj
added      nls/tests/normalization/normalizationTest.y
added      nls/tests/normalization/normalizationTest.yy.c
 - Test harness for NormalizeString based on official Unicode 5.0 test suite. Our implementation passes all tests. Will become an actual regression test suite in the future.

svn path=/trunk/; revision=34402

reactos/nls/3rdparty/icu4ros.rbuild

@@ -3,14 +3,19 @@
 <module name="icu4ros" type="staticlibrary" allowwarnings="true">
 	<define name="U_STATIC_IMPLEMENTATION" />
 	<define name="U_HAVE_INTTYPES_H" />
+	<define name="UCONFIG_NO_FILE_IO">1</define>
+	<define name="ICU_NO_USER_DATA_OVERRIDE">1</define>
 	<compilerflag>-fno-exceptions</compilerflag>
 	<compilerflag>-fno-rtti</compilerflag>
 	<include base="icu4ros">icu/source/common</include>
+	<include base="icu4ros">icu/source/i18n</include>
+	<file>stubs.cpp</file>
 	<directory name="icu">
 	<directory name="source">
 	<directory name="common">
 		<file>bmpset.cpp</file>
 		<file>uhash_us.cpp</file>
+		<file>uidna.cpp</file>
 		<file>uiter.cpp</file>
 		<file>unifilt.cpp</file>
 		<file>unifunct.cpp</file>
@@ -20,15 +25,24 @@
 		<file>unorm.cpp</file>
 		<file>uobject.cpp</file>
 		<file>uset.cpp</file>
+		<file>usprep.cpp</file>
 		<file>util.cpp</file>
 		<file>uvector.cpp</file>
 		<file>cmemory.c</file>
 		<file>cstring.c</file>
+		<file>locmap.c</file>
+		<file>punycode.c</file>
 		<file>ucln_cmn.c</file>
+		<file>ucmndata.c</file>
+		<file>ucol_swp.c</file>
+		<file>ubidi_props.c</file>
 		<file>udata.c</file>
+		<file>udatamem.c</file>
 		<file>udataswp.c</file>
 		<file>uhash.c</file>
+		<file>uinit.c</file>
 		<file>uinvchar.c</file>
+		<file>umapfile.c</file>
 		<file>umath.c</file>
 		<file>umutex.c</file>
 		<file>ustring.c</file>

reactos/nls/3rdparty/stubs.cpp

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "unicode/utypes.h"
+#include "unicode/putil.h"
+#include "unicode/utrace.h"
+#include "ucnv_io.h"
+
+// TODO: do a better job of ensuring these internal functions really can be stubbed away
+
+/* Unused dependencies of udata.c */
+U_CAPI UBool U_EXPORT2
+uprv_pathIsAbsolute(const char *) {
+    return TRUE;
+}
+
+U_CAPI const char * U_EXPORT2
+u_getDataDirectory() {
+    return "";
+}
+
+/* Unused dependencies of uinit.c */
+U_CFUNC UBool
+utrace_cleanup() {
+    return TRUE;
+}
+
+U_CFUNC uint16_t
+ucnv_io_countKnownConverters(UErrorCode *) {
+    return 0;
+}

reactos/nls/dll/idndl/idndl.cpp

@@ -0,0 +1,25 @@
+#define WIN32_LEAN_AND_MEAN
+#define STRICT
+
+#define WINVER 0x0600
+
+#include <windows.h>
+
+extern "C"
+{
+#include <idndl.h>
+}
+
+int
+WINAPI
+DownlevelGetLocaleScripts
+(
+	LPCWSTR lpLocaleName,
+	LPWSTR lpScripts,
+	int cchScripts
+)
+{
+	return GetLocaleInfoEx(lpLocaleName, LOCALE_SSCRIPTS, lpScripts, cchScripts);
+}
+
+// EOF

reactos/nls/dll/idndl/idndl.def

@@ -0,0 +1,6 @@
+LIBRARY	"idndl"
+
+EXPORTS
+	DownlevelGetLocaleScripts    = DownlevelGetLocaleScripts@12 @1
+	DownlevelGetStringScripts@20 = kernel32.GetStringScripts    @2
+	DownlevelVerifyScripts@20    = kernel32.VerifyScripts       @3

reactos/nls/dll/idndl/idndl.rbuild

@@ -0,0 +1,10 @@
+<?xml version="1.0"?>
+<!DOCTYPE module SYSTEM "../../../tools/rbuild/project.dtd">
+<module name="idndl" type="win32dll" installname="idndl.dll">
+	<library>kernel32</library>
+	<compilerflag>-fno-exceptions</compilerflag>
+	<compilerflag>-fno-rtti</compilerflag>
+	<linkerflag>-Wl,--entry,0</linkerflag>
+	<file>idndl.cpp</file>
+	<importlibrary definition="idndl.def" />
+</module>

reactos/nls/dll/idndl_redist/data/genidldndata.cpp

@@ -0,0 +1,322 @@
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <unicode/putil.h>
+#include <unicode/uchar.h>
+#include <unicode/uloc.h>
+#include <unicode/ures.h>
+#include <unicode/uscript.h>
+
+#include <assert.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <functional>
+#include <map>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+struct script_name
+{
+private:
+	void normalize()
+	{
+		chars[0] = toupper(chars[0]);
+		chars[1] = tolower(chars[1]);
+		chars[2] = tolower(chars[2]);
+		chars[3] = tolower(chars[3]);
+	}
+
+public:
+	char chars[4];
+
+	bool operator<(const script_name& Y) const { return strncmp(chars, Y.chars, 4) < 0; }
+	bool operator>(const script_name& Y) const { return strncmp(chars, Y.chars, 4) > 0; }
+	bool operator==(const script_name& Y) const { return strncmp(chars, Y.chars, 4) == 0; }
+	bool operator!=(const script_name& Y) const { return strncmp(chars, Y.chars, 4) != 0; }
+	bool operator<=(const script_name& Y) const { return strncmp(chars, Y.chars, 4) <= 0; }
+	bool operator>=(const script_name& Y) const { return strncmp(chars, Y.chars, 4) >= 0; }
+
+	script_name(): chars() { }
+	script_name(const script_name& Y) { memcpy(chars, Y.chars, sizeof(chars)); }
+
+	const script_name& operator=(const script_name& Y)
+	{
+		memcpy(chars, Y.chars, sizeof(chars));
+		return *this;
+	}
+
+	explicit script_name(const char * pChars)
+	{
+		assert(pChars);
+		assert(strlen(pChars) == 4);
+		chars[0] = pChars[0];
+		chars[1] = pChars[1];
+		chars[2] = pChars[2];
+		chars[3] = pChars[3];
+		normalize();
+	}
+
+	explicit script_name(const UChar * pChars)
+	{
+		assert(pChars);
+		assert(u_strlen(pChars) == 4);
+		u_UCharsToChars(pChars, chars, 4);
+		normalize();
+	}
+};
+
+struct lessLocaleId: public std::binary_function<std::string, std::string, bool>
+{
+	result_type operator()(const first_argument_type& x, const second_argument_type& y) const
+	{
+		return stricmp(x.c_str(), y.c_str()) < 0;
+	}
+};
+
+std::string convertLocale(const char * locale)
+{
+	std::string s(locale); // FIXME!!!
+	return s;
+}
+
+bool validId(const std::string& id)
+{
+	std::string::const_iterator p = id.begin();
+
+	if(p == id.end() || !u_isIDStart(*p))
+		return false;
+
+	++ p;
+
+	for(; p != id.end(); ++ p)
+		if(!u_isIDPart(*p))
+			return false;
+
+	return true;
+}
+
+std::string getLocaleLiteral(const std::string& locale)
+{
+	std::string lit;
+	lit += '"';
+	lit += locale; // FIXME!!! escapes
+	lit += '"';
+	return lit;
+}
+
+std::string getScriptLiteral(const script_name& s)
+{
+	std::string lit;
+	lit += '"';
+	lit += s.chars[0];
+	lit += s.chars[1];
+	lit += s.chars[2];
+	lit += s.chars[3];
+	lit += '"';
+	return lit;
+}
+
+std::string getScriptId(const script_name& s)
+{
+	std::string id("IDNDL_Script_");
+	id += s.chars[0];
+	id += s.chars[1];
+	id += s.chars[2];
+	id += s.chars[3];
+	assert(validId(id));
+	return id;
+}
+
+std::string getScriptSetId(const std::set<script_name>& s)
+{
+	std::string id("IDNDL_ScriptSet_");
+
+	for(std::set<script_name>::const_iterator p = s.begin(); p != s.end(); ++ p)
+	{
+		id += p->chars[0];
+		id += p->chars[1];
+		id += p->chars[2];
+		id += p->chars[3];
+	}
+
+	assert(validId(id));
+	return id;
+}
+
+int main()
+{
+	UErrorCode status = U_ZERO_ERROR;
+
+	/* Locale -> scripts table */
+	int32_t localeCount = uloc_countAvailable();
+
+	typedef std::map<std::string, std::set<script_name>, lessLocaleId> LocalesScripts;
+	LocalesScripts localesScripts;
+
+	for(int32_t i = 0; i < localeCount; ++ i)
+	{
+		const char * locale = uloc_getAvailable(i);
+		UResourceBundle * localeRes = ures_open(NULL, locale, &status);
+
+		if(U_SUCCESS(status))
+		{
+			UErrorCode localStatus = U_ZERO_ERROR;
+			UResourceBundle * scriptsRes = ures_getByKey(localeRes, "LocaleScript", NULL, &status);
+
+			if(U_SUCCESS(status))
+			{
+				std::set<script_name> localeScripts;
+
+				while(ures_hasNext(scriptsRes))
+				{
+					int32_t scriptLen = 0;
+					const UChar * script = ures_getNextString(scriptsRes, &scriptLen, NULL, &localStatus);
+
+					if(U_SUCCESS(localStatus))
+						localeScripts.insert(script_name(script));
+					else
+					{
+						fprintf(stderr, "warning: failed reading scripts for locale %s: %s\n", locale, u_errorName(localStatus));
+						break;
+					}
+				}
+
+				if(localeScripts.size())
+					localesScripts[convertLocale(locale)].insert(localeScripts.begin(), localeScripts.end());
+
+				ures_close(scriptsRes);
+			}
+			else
+				fprintf(stderr, "warning: failed reading scripts for locale %s: %s\n", locale, u_errorName(localStatus));
+
+			ures_close(localeRes);
+		}
+		else
+			break;
+	}
+
+	if(!U_SUCCESS(status))
+	{
+		fprintf(stderr, "error: failed enumerating locale scripts: %s\n", u_errorName(status));
+		return 1;
+	}
+
+	typedef std::set<std::set<script_name> > UniqueScriptSets;
+	UniqueScriptSets uniqueScriptSets;
+
+	for(LocalesScripts::const_iterator p = localesScripts.begin(); p != localesScripts.end(); ++ p)
+		uniqueScriptSets.insert(p->second);
+
+	typedef std::map<std::string, UniqueScriptSets::const_iterator> LocalesScriptsFolded;
+	LocalesScriptsFolded localesScriptsFolded;
+
+	for(LocalesScripts::const_iterator p = localesScripts.begin(); p != localesScripts.end(); ++ p)
+		localesScriptsFolded.insert(std::make_pair(p->first, uniqueScriptSets.find(p->second)));
+
+	// Unique script sets
+	printf("struct %s { wchar_t const * scripts; int length; };\n", "IDNDL_ScriptSet");
+
+	for(UniqueScriptSets::const_iterator p = uniqueScriptSets.begin(); p != uniqueScriptSets.end(); ++ p)
+	{
+		printf("static const %s %s = {", "IDNDL_ScriptSet", getScriptSetId(*p).c_str());
+
+		for(std::set<script_name>::const_iterator pScript = p->begin(); pScript != p->end(); ++ pScript)
+			printf(" L%s L\";\"", getScriptLiteral(*pScript).c_str());
+
+		printf(", %d };\n", static_cast<int>(p->size() * (4 + 1) + 1));
+	}
+
+	// Sorted table of locale ids
+	printf("static wchar_t const * const %s [] = {\n", "IDNDL_Locales");
+
+	for(LocalesScriptsFolded::const_iterator p = localesScriptsFolded.begin(); p != localesScriptsFolded.end(); ++ p)
+		printf("L%s,\n", getLocaleLiteral(p->first).c_str());
+
+	printf("};\n");
+
+	// Locale id index -> script set
+	printf("static %s const * const %s [] = {\n", "IDNDL_ScriptSet", "IDNDL_ScriptSets");
+
+	for(LocalesScriptsFolded::const_iterator p = localesScriptsFolded.begin(); p != localesScriptsFolded.end(); ++ p)
+		printf("&%s,\n", getScriptSetId(*p->second).c_str());
+
+	printf("};\n");
+
+	/* Codepoint -> script table */
+	// Script code -> script name table
+	printf("static const wchar_t * const %s[] = {\n", "IDNDL_ScriptNames");
+
+	for(int script = 0; script < USCRIPT_CODE_LIMIT; ++ script)
+		printf("L%s,\n", getScriptLiteral(script_name(uscript_getShortName(static_cast<UScriptCode>(script)))).c_str());
+
+	printf("};\n");
+
+	// Codepoint range -> script code
+	printf("struct %s { int lbound; int ubound; int code; };\n", "IDNDL_CharRangeScript");
+	printf("static const %s %s[] = {\n", "IDNDL_CharRangeScript", "IDNDL_CharRangeScripts");
+
+	int lbound = UCHAR_MIN_VALUE;
+	UScriptCode lastScript = uscript_getScript(UCHAR_MIN_VALUE, &status);
+
+	if(!U_SUCCESS(status) || lastScript < 0)
+		lastScript = USCRIPT_UNKNOWN;
+
+	for(UChar32 c = UCHAR_MIN_VALUE + 1; c <= UCHAR_MAX_VALUE; ++ c)
+	{
+		UScriptCode script = uscript_getScript(c, &status);
+
+		if(!U_SUCCESS(status) || script < 0)
+			script = USCRIPT_UNKNOWN;
+
+		assert(script >= 0 && script < USCRIPT_CODE_LIMIT);
+
+		if(script != lastScript)
+		{
+			if(lastScript != USCRIPT_UNKNOWN)
+				printf("{ %d, %d, %d },\n", lbound, c - 1, static_cast<int>(lastScript));
+
+			lbound = c;
+			lastScript = script;
+		}
+	}
+
+	if(lastScript != USCRIPT_UNKNOWN)
+		printf("{ %d, %d, %d },\n", lbound, UCHAR_MAX_VALUE, static_cast<int>(lastScript));
+
+	printf("};\n");
+}
+
+// EOF

reactos/nls/dll/idndl_redist/idndl.cpp

@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <unicode/uchar.h>
+#include <unicode/uclean.h>
+#include <unicode/umachine.h>
+#include <unicode/uscript.h>
+
+#define WIN32_LEAN_AND_MEAN
+#define STRICT
+
+#define WINVER 0x0600
+
+#include <windows.h>
+
+extern "C"
+{
+#include <idndl.h>
+}
+
+#include "scripts.h"
+#include "data/idldata.cpp"
+
+int
+WINAPI
+DownlevelGetLocaleScripts
+(
+	LPCWSTR lpLocaleName,
+	LPWSTR lpScripts,
+	int cchScripts
+)
+{
+	void * lpFoundLocale = bsearch
+	(
+		lpLocaleName,
+		IDNDL_Locales,
+		//~ ARRAYSIZE(IDNDL_Locales),
+		sizeof(IDNDL_Locales) / sizeof(IDNDL_Locales[0]),
+		sizeof(IDNDL_Locales[0]),
+		(int (*)(const void *, const void *))_stricmp
+	);
+
+	if(lpFoundLocale == NULL)
+	{
+		SetLastError(ERROR_INVALID_PARAMETER);
+		return 0;
+	}
+
+	const IDNDL_ScriptSet * pScriptSet = IDNDL_ScriptSets[static_cast<const wchar_t **>(lpFoundLocale) - IDNDL_Locales];
+
+	if(pScriptSet->length > cchScripts)
+		SetLastError(ERROR_INSUFFICIENT_BUFFER);
+	else
+		memcpy(lpScripts, pScriptSet->scripts, pScriptSet->length * sizeof(WCHAR));
+
+	return pScriptSet->length;
+}
+
+static int IDNDL_CompareCharRange(const void * x, const void * y)
+{
+	const IDNDL_CharRangeScript * pRangeX = static_cast<const IDNDL_CharRangeScript *>(x);
+	const IDNDL_CharRangeScript * pRangeY = static_cast<const IDNDL_CharRangeScript *>(y);
+
+	assert(pRangeX->lbound <= pRangeX->ubound);
+	assert(pRangeY->lbound <= pRangeY->ubound);
+
+	int cmp;
+
+	cmp = pRangeX->ubound - pRangeY->lbound;
+
+	if(cmp < 0)
+		return cmp;
+
+	cmp = pRangeX->lbound - pRangeY->ubound;
+
+	if(cmp > 0)
+		return cmp;
+
+	assert((pRangeX->lbound >= pRangeY->lbound && pRangeX->ubound <= pRangeY->ubound) || (pRangeY->lbound >= pRangeX->lbound && pRangeY->ubound <= pRangeX->ubound));
+	return 0;
+}
+
+extern "C" bool SCRIPTS_GetCharScriptCode(UChar32 c, int32_t * code)
+{
+	assert(c >= UCHAR_MIN_VALUE && c <= UCHAR_MAX_VALUE);
+
+	IDNDL_CharRangeScript character;
+	character.lbound = c;
+	character.ubound = c;
+
+	void * pRange = bsearch
+	(
+		&character,
+		IDNDL_CharRangeScripts,
+		//~ ARRAYSIZE(IDNDL_CharRangeScripts),
+		sizeof(IDNDL_CharRangeScripts) / sizeof(IDNDL_CharRangeScripts[0]),
+		sizeof(IDNDL_CharRangeScripts[0]),
+		&IDNDL_CompareCharRange
+	);
+
+	if(pRange == NULL)
+		*code = USCRIPT_UNKNOWN;
+	else
+		*code = static_cast<IDNDL_CharRangeScript *>(pRange)->code;
+
+	return true;
+}
+
+extern "C" bool SCRIPTS_GetScriptCode(const SCRIPTS_Script * pScript, int32_t * code)
+{
+	void * ppScript = bsearch
+	(
+		pScript,
+		IDNDL_ScriptNames,
+		//~ ARRAYSIZE(IDNDL_ScriptNames),
+		sizeof(IDNDL_ScriptNames) / sizeof(IDNDL_ScriptNames[0]),
+		sizeof(IDNDL_ScriptNames[0]),
+		(int (*)(const void *, const void *))_stricmp
+	);
+
+	bool retval;
+
+	retval = !!ppScript;
+
+	if(!retval)
+	{
+		SetLastError(ERROR_INVALID_PARAMETER);
+		return retval;
+	}
+
+	*code = static_cast<int32_t>(static_cast<const wchar_t **>(ppScript) - IDNDL_ScriptNames);
+	return retval;
+}
+
+extern "C" void SCRIPTS_GetScriptName(int32_t code, SCRIPTS_Script * pScript)
+{
+	//~ assert(code >= 0 && static_cast<uint32_t>(code) < ARRAYSIZE(IDNDL_ScriptNames));
+	assert(code >= 0 && static_cast<uint32_t>(code) < (sizeof(IDNDL_ScriptNames) / sizeof(IDNDL_ScriptNames[0])));
+	memcpy(pScript->ScriptName, IDNDL_ScriptNames[code], sizeof(pScript->ScriptName));
+}
+
+// EOF

reactos/nls/dll/idndl_redist/idndl.def

@@ -0,0 +1,35 @@
+; Copyright (c) 2008, KJK::Hyperion
+; All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions are met:
+;
+; - Redistributions of source code must retain the above copyright notice,
+;   this list of conditions and the following disclaimer.
+;
+; - Redistributions in binary form must reproduce the above copyright notice,
+;   this list of conditions and the following disclaimer in the documentation
+;   and/or other materials provided with the distribution.
+;
+; - Neither the name of the ReactOS Foundation nor the names of its
+;   contributors may be used to endorse or promote products derived from this
+;   software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+; ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+; LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+; CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+; SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+; INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+; CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+; ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+; POSSIBILITY OF SUCH DAMAGE.
+
+LIBRARY	"idndl"
+
+EXPORTS
+	DownlevelGetLocaleScripts = DownlevelGetLocaleScripts@12 @1
+	DownlevelGetStringScripts = GetStringScripts@20          @2
+	DownlevelVerifyScripts    = VerifyScripts@20             @3

reactos/nls/dll/idndl_redist/idndl_redist.rbuild

@@ -0,0 +1,15 @@
+<?xml version="1.0"?>
+<!DOCTYPE module SYSTEM "../../../tools/rbuild/project.dtd">
+<module name="idndl_redist" type="win32dll" installname="idndl.dll">
+	<library>kernel32</library>
+	<library>scripts</library>
+	<library>libcntpr</library>
+	<compilerflag>-fno-exceptions</compilerflag>
+	<compilerflag>-fno-rtti</compilerflag>
+	<linkerflag>--entry=0</linkerflag>
+	<include base="icu4ros">icu/source/common</include>
+	<include base="scripts">.</include>
+	<importlibrary definition="idndl.def" />
+	<file>idndl.cpp</file>
+	<file>stubs.cpp</file>
+</module>

reactos/nls/dll/idndl_redist/stubs.cpp

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+// FIXME: move stubs elsewhere
+
+#include <stdlib.h>
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+void free(void * memory)
+{
+	HeapFree(GetProcessHeap(), 0, memory);
+}
+
+void * malloc(size_t size)
+{
+	return HeapAlloc(GetProcessHeap(), 0, size);
+}
+
+void * realloc(void * memory, size_t size)
+{
+	return HeapReAlloc(GetProcessHeap(), 0, memory, size);
+}
+
+void operator delete(void * memory)
+{
+	free(memory);
+}
+
+extern "C" int __cdecl _purecall()
+{
+	FatalAppExitW(0, L"pure virtual call");
+	FatalExit(0);
+	return 0;
+}
+
+extern "C" void __cxa_pure_virtual() { _purecall(); }
+
+extern "C" void _assert()
+{
+	FatalAppExitW(0, L"assertion failed");
+	FatalExit(0);
+}
+
+// EOF

reactos/nls/dll/normaliz/dummy.c

@@ -0,0 +1 @@
+/* Leave empty */

reactos/nls/dll/normaliz/normaliz.def

@@ -0,0 +1,37 @@
+; Copyright (c) 2008, KJK::Hyperion
+; All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions are met:
+;
+; - Redistributions of source code must retain the above copyright notice,
+;   this list of conditions and the following disclaimer.
+;
+; - Redistributions in binary form must reproduce the above copyright notice,
+;   this list of conditions and the following disclaimer in the documentation
+;   and/or other materials provided with the distribution.
+;
+; - Neither the name of the ReactOS Foundation nor the names of its
+;   contributors may be used to endorse or promote products derived from this
+;   software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+; ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+; LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+; CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+; SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+; INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+; CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+; ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+; POSSIBILITY OF SUCH DAMAGE.
+
+LIBRARY	"normaliz"
+
+EXPORTS
+	IdnToAscii@20           = kernel32.IdnToAscii           @1
+	IdnToNameprepUnicode@20 = kernel32.IdnToNameprepUnicode @2
+	IdnToUnicode@20         = kernel32.IdnToUnicode         @3
+	IsNormalizedString@12   = kernel32.IsNormalizedString   @4
+	NormalizeString@20      = kernel32.NormalizeString      @5

reactos/nls/dll/normaliz/normaliz.rbuild

@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<!DOCTYPE module SYSTEM "../../../tools/rbuild/project.dtd">
+<module name="normaliz" type="win32dll">
+	<library>kernel32</library>
+	<importlibrary definition="normaliz.def" />
+	<file>dummy.c</file>
+</module>

reactos/nls/dll/normaliz_redist/data/icudt38.lst

@@ -0,0 +1 @@
+uidna.spp

reactos/nls/dll/normaliz_redist/normaliz.cpp

@@ -1,7 +1,39 @@
-// FIXME: clean up this mess
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+// FIXME: move stubs elsewhere
 
-#ifndef _DEBUG
 #include <stdlib.h>
+#include <unicode/uclean.h>
 
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
@@ -35,6 +67,21 @@ extern "C" int __cdecl _purecall()
 
 extern "C" void __cxa_pure_virtual() { _purecall(); }
 
-#endif
+extern "C" void _assert()
+{
+	FatalAppExitW(0, L"assertion failed");
+	FatalExit(0);
+}
+
+extern "C" BOOL WINAPI DllMain(HINSTANCE hinstDll, DWORD fdwReason, LPVOID lpvReserved)
+{
+	if(fdwReason == DLL_PROCESS_ATTACH)
+		DisableThreadLibraryCalls(hinstDll);
+
+	if(fdwReason == DLL_PROCESS_DETACH && lpvReserved == NULL)
+		u_cleanup();
+
+	return TRUE;
+}
 
 // EOF

reactos/nls/dll/normaliz_redist/normaliz.def

@@ -1,5 +1,37 @@
+; Copyright (c) 2008, KJK::Hyperion
+; All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions are met:
+;
+; - Redistributions of source code must retain the above copyright notice,
+;   this list of conditions and the following disclaimer.
+;
+; - Redistributions in binary form must reproduce the above copyright notice,
+;   this list of conditions and the following disclaimer in the documentation
+;   and/or other materials provided with the distribution.
+;
+; - Neither the name of the ReactOS Foundation nor the names of its
+;   contributors may be used to endorse or promote products derived from this
+;   software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+; ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+; LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+; CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+; SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+; INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+; CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+; ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+; POSSIBILITY OF SUCH DAMAGE.
+
 LIBRARY	"normaliz"
 
 EXPORTS
-	NormalizeString=NormalizeString@20
+	IdnToAscii           = IdnToAscii@20           @1
-	IsNormalizedString=IsNormalizedString@12
+	IdnToNameprepUnicode = IdnToNameprepUnicode@20 @2
+	IdnToUnicode         = IdnToUnicode@20         @3
+	IsNormalizedString   = IsNormalizedString@12   @4
+	NormalizeString      = NormalizeString@20      @5

reactos/nls/dll/normaliz_redist/normaliz_redist.rbuild

@@ -1,11 +1,19 @@
 <?xml version="1.0"?>
 <!DOCTYPE module SYSTEM "../../../tools/rbuild/project.dtd">
+<module name="normaliz_redist_data" type="staticlibrary">
+	<include base="icu4ros">icu/source/common</include>
+	<directory name="data"><file>icudt38.c</file></directory>
+</module>
 <module name="normaliz_redist" type="win32dll" installname="normaliz.dll">
 	<library>normalize</library>
+	<library>idna</library>
 	<library>icu4ros</library>
+	<library>normaliz_redist_data</library>
+	<library>libcntpr</library>
 	<library>kernel32</library>
 	<compilerflag>-fno-exceptions</compilerflag>
 	<compilerflag>-fno-rtti</compilerflag>
+	<include base="icu4ros">icu/source/common</include>
 	<importlibrary definition="normaliz.def" />
 	<file>normaliz.cpp</file>
 </module>

reactos/nls/lib/idna/idna.cpp

@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#define WIN32_LEAN_AND_MEAN
+#define STRICT
+
+#define WINVER 0x0600
+
+#include <windows.h>
+#include <winnls.h>
+
+#include <unicode/uidna.h>
+
+static
+int32_t
+IDNA_FlagsToICU
+(
+	DWORD dwFlags
+)
+{
+	C_ASSERT(IDN_ALLOW_UNASSIGNED == UIDNA_ALLOW_UNASSIGNED);
+	C_ASSERT(IDN_USE_STD3_ASCII_RULES == UIDNA_USE_STD3_RULES);
+	return dwFlags;
+}
+
+static
+DWORD
+NLS_ErrorFromICU
+(
+	UErrorCode ErrorCode
+)
+{
+	// TODO
+	return ERROR_GEN_FAILURE;
+}
+
+int
+WINAPI
+IdnToAscii
+(
+	DWORD dwFlags,
+	LPCWSTR lpUnicodeCharStr,
+	int cchUnicodeChar,
+	LPWSTR lpASCIICharStr,
+	int cchASCIIChar
+)
+{
+	UErrorCode status;
+	int32_t retval = uidna_IDNToASCII
+	(
+		lpUnicodeCharStr,
+		cchUnicodeChar,
+		lpASCIICharStr,
+		cchASCIIChar,
+		IDNA_FlagsToICU(dwFlags),
+		NULL,
+		&status
+	);
+
+	if(U_FAILURE(status))
+		SetLastError(NLS_ErrorFromICU(status));
+
+	return retval;
+}
+
+int
+WINAPI
+IdnToUnicode
+(
+	DWORD dwFlags,
+	LPCWSTR lpASCIICharStr,
+	int cchASCIIChar,
+	LPWSTR lpUnicodeCharStr,
+	int cchUnicodeChar
+)
+{
+	UErrorCode status;
+	int32_t retval = uidna_IDNToUnicode
+	(
+		lpASCIICharStr,
+		cchASCIIChar,
+		lpUnicodeCharStr,
+		cchUnicodeChar,
+		IDNA_FlagsToICU(dwFlags),
+		NULL,
+		&status
+	);
+
+	if(U_FAILURE(status))
+		SetLastError(NLS_ErrorFromICU(status));
+
+	return retval;
+}
+
+// EOF

reactos/nls/lib/idna/idna.rbuild

@@ -0,0 +1,10 @@
+<?xml version="1.0"?>
+<!DOCTYPE module SYSTEM "../../../tools/rbuild/project.dtd">
+<module name="idna" type="staticlibrary">
+	<library>icu4ros</library>
+	<compilerflag>-fno-exceptions</compilerflag>
+	<compilerflag>-fno-rtti</compilerflag>
+	<include base="icu4ros">icu/source/common</include>
+	<file>idna.cpp</file>
+	<file>nameprep.cpp</file>
+</module>

reactos/nls/lib/idna/nameprep.cpp

@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#define WIN32_LEAN_AND_MEAN
+#define STRICT
+
+#define WINVER 0x0600
+
+#include <windows.h>
+#include <winnls.h>
+
+#include <unicode/usprep.h>
+
+static
+int32_t
+NAMEPREP_FlagsToICU
+(
+	DWORD dwFlags
+)
+{
+	C_ASSERT(IDN_ALLOW_UNASSIGNED == USPREP_ALLOW_UNASSIGNED);
+	return dwFlags;
+}
+
+static
+DWORD
+NLS_ErrorFromICU
+(
+	UErrorCode ErrorCode
+)
+{
+	// TODO
+	return ERROR_GEN_FAILURE;
+}
+
+int
+WINAPI
+IdnToNameprepUnicode
+(
+	DWORD dwFlags,
+	LPCWSTR lpUnicodeCharStr,
+	int cchUnicodeChar,
+	LPWSTR lpNameprepCharStr,
+	int cchNameprepChar
+)
+{
+	UErrorCode status;
+
+	UStringPrepProfile * nameprep = usprep_open(NULL, "uidna", &status);
+
+	int32_t retval = usprep_prepare
+	(
+		nameprep,
+		lpUnicodeCharStr,
+		cchUnicodeChar,
+		lpNameprepCharStr,
+		cchNameprepChar,
+		NAMEPREP_FlagsToICU(dwFlags),
+		NULL,
+		&status
+	);
+
+	usprep_close(nameprep);
+
+	if(U_FAILURE(status))
+		SetLastError(NLS_ErrorFromICU(status));
+
+	return retval;
+}
+
+// EOF

reactos/nls/lib/normalize/normalize.cpp

@@ -1,3 +1,37 @@
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <unicode/unorm.h>
+
 #define WIN32_LEAN_AND_MEAN
 #define STRICT
 
@@ -6,8 +40,6 @@
 #include <windows.h>
 #include <winnls.h>
 
-#include <unicode/unorm.h>
-
 static
 UNormalizationMode
 NORMALIZE_ModeToICU
@@ -65,6 +97,9 @@ NormalizeString
 	if(U_FAILURE(ErrorCode))
 		SetLastError(NLS_ErrorFromICU(ErrorCode));
 
+	if(cwDstLength == 0 || ErrorCode == U_BUFFER_OVERFLOW_ERROR)
+		retval += 1;
+
 	return retval;
 }
 

reactos/nls/lib/scripts/scripts.cpp

@@ -0,0 +1,288 @@
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <assert.h>
+#include <stdlib.h>
+
+#include <unicode/uchar.h>
+#include <unicode/utf.h>
+#include <unicode/uscript.h>
+
+#define WIN32_LEAN_AND_MEAN
+#define STRICT
+
+#define WINVER 0x0600
+
+#include <windows.h>
+#include <winnls.h>
+
+#include "scripts.h"
+
+struct SCRIPTS_ScriptsSet
+{
+private:
+	uint32_t m_bits[USCRIPT_CODE_LIMIT / 32 + !!(USCRIPT_CODE_LIMIT % 32)];
+
+public:
+	SCRIPTS_ScriptsSet(): m_bits() { }
+
+	void set(int index)
+	{
+		assert(index >= 0 && index < USCRIPT_CODE_LIMIT);
+		m_bits[index / 32] |= 1 << (index % 32);
+	}
+
+	void reset(int index)
+	{
+		assert(index >= 0 && index < USCRIPT_CODE_LIMIT);
+		m_bits[index / 32] &= ~(1 << (index % 32));
+	}
+
+	bool get(int index) const
+	{
+		assert(index >= 0 && index < USCRIPT_CODE_LIMIT);
+		return ((m_bits[index / 32] >> (index % 32)) % 2) != 0;
+	}
+
+	bool processCharacter(UChar32 c)
+	{
+		int32_t code;
+		bool retval = SCRIPTS_GetCharScriptCode(c, &code);
+
+		if(retval)
+			set(code);
+
+		return retval;
+	}
+
+	bool processScript(const SCRIPTS_Script * pScript)
+	{
+		assert(pScript);
+
+		bool retval;
+
+		retval = pScript->Separator == L';';
+
+		if(!retval)
+		{
+			SetLastError(ERROR_INVALID_PARAMETER);
+			return retval;
+		}
+
+		int32_t script;
+		retval = SCRIPTS_GetScriptCode(pScript, &script);
+
+		if(retval)
+			set(script);
+
+		return retval;
+	}
+
+	bool processScripts(LPCWSTR lpScripts, int cchScripts)
+	{
+		bool retval;
+		const SCRIPTS_Script * pScripts = reinterpret_cast<const SCRIPTS_Script *>(lpScripts);
+
+		if(cchScripts < 0)
+			cchScripts = wcslen(lpScripts);
+
+		div_t divmod = div(cchScripts, (sizeof(SCRIPTS_Script) / sizeof(WCHAR)));
+		retval = !divmod.rem;
+
+		if(!retval)
+		{
+			SetLastError(ERROR_INVALID_PARAMETER);
+			return retval;
+		}
+
+		for(int iScript = 0, ccScripts = divmod.quot; retval && iScript < ccScripts; ++ iScript)
+			retval = processScript(pScripts + iScript);
+
+		return retval;
+	}
+
+	bool operator<=(const SCRIPTS_ScriptsSet& Y) const
+	{
+		bool retval;
+
+		//~ for(int i = 0; i < ARRAYSIZE(m_bits); ++ i)
+		for(unsigned i = 0; i < (sizeof(m_bits) / sizeof(m_bits[0])); ++ i)
+		{
+			retval = (~m_bits[i] | Y.m_bits[i]) != 0;
+
+			if(!retval)
+				return retval;
+		}
+
+		return retval;
+	}
+};
+
+int
+WINAPI
+GetStringScripts
+(
+	DWORD dwFlags,
+	LPCWSTR lpString,
+	int cchString,
+	LPWSTR lpScripts,
+	int cchScripts
+)
+{
+	if((dwFlags | GSS_ALLOW_INHERITED_COMMON) != GSS_ALLOW_INHERITED_COMMON)
+	{
+		SetLastError(ERROR_INVALID_FLAGS);
+		return 0;
+	}
+
+	if((lpString == NULL) != (cchString == 0) || (cchScripts > 0 && lpScripts == NULL) || cchScripts < 0)
+	{
+		SetLastError(ERROR_INVALID_PARAMETER);
+		return 0;
+	}
+
+	SCRIPTS_ScriptsSet StringScripts;
+
+	if(cchString < 0)
+	{
+		for(int i = 0; lpString[i];)
+		{
+			UChar32 c = lpString[i ++];
+
+			if(U16_IS_LEAD(c) && U16_IS_TRAIL(lpString[i]))
+			{
+				assert(lpString[i]);
+				c = U16_GET_SUPPLEMENTARY(c, lpString[i ++]);
+			}
+
+			if(!StringScripts.processCharacter(c))
+				return 0;
+		}
+	}
+	else
+	{
+		for(int i = 0; i < cchString;)
+		{
+			UChar32 c;
+			U16_NEXT(lpString, i, cchString, c);
+
+			if(!StringScripts.processCharacter(c))
+				return 0;
+		}
+	}
+
+	if(!(dwFlags & GSS_ALLOW_INHERITED_COMMON))
+	{
+		StringScripts.reset(USCRIPT_COMMON);
+		StringScripts.reset(USCRIPT_INHERITED);
+	}
+
+	if(cchScripts)
+	{
+		int ccScripts = 0;
+		int ccScriptsCapacity = (cchScripts - 1) / (sizeof(SCRIPTS_Script) / sizeof(WCHAR));
+		SCRIPTS_Script * pScripts = reinterpret_cast<SCRIPTS_Script *>(lpScripts);
+
+		for(int iScript = 0; iScript < USCRIPT_CODE_LIMIT; ++ iScript)
+		{
+			if(StringScripts.get(iScript))
+			{
+				if(ccScripts == ccScriptsCapacity)
+				{
+					SetLastError(ERROR_INSUFFICIENT_BUFFER);
+					return 0;
+				}
+
+				SCRIPTS_GetScriptName(iScript, &pScripts[ccScripts]);
+				pScripts[ccScripts].Separator = L';';
+			}
+		}
+
+		qsort(pScripts, ccScripts, sizeof(SCRIPTS_Script), &SCRIPTS_Script::Compare);
+
+		cchScripts = ccScripts * (sizeof(SCRIPTS_Script) / sizeof(WCHAR));
+
+		if(cchScripts)
+			lpScripts[cchScripts - 1] = 0;
+	}
+	else
+	{
+		for(int iScript = 0; iScript < USCRIPT_CODE_LIMIT; ++ iScript)
+			cchScripts += !!StringScripts.get(iScript);
+
+		cchScripts *= (sizeof(SCRIPTS_Script) / sizeof(WCHAR));
+		cchScripts += 1;
+		assert(cchScripts);
+	}
+
+	return cchScripts;
+}
+
+BOOL
+WINAPI
+VerifyScripts
+(
+	DWORD dwFlags,
+	LPCWSTR lpLocaleScripts,
+	int cchLocaleScripts,
+	LPCWSTR lpTestScripts,
+	int cchTestScripts
+)
+{
+	if((dwFlags | VS_ALLOW_LATIN) != VS_ALLOW_LATIN)
+	{
+		SetLastError(ERROR_INVALID_FLAGS);
+		return FALSE;
+	}
+
+	if((lpLocaleScripts == NULL) != (cchLocaleScripts == 0) || (lpTestScripts == NULL) != (cchTestScripts == 0) )
+	{
+		SetLastError(ERROR_INVALID_PARAMETER);
+		return FALSE;
+	}
+
+	SCRIPTS_ScriptsSet LocaleScripts;
+	SCRIPTS_ScriptsSet TestScripts;
+
+	if(!LocaleScripts.processScripts(lpLocaleScripts, cchLocaleScripts))
+		return FALSE;
+
+	if(!TestScripts.processScripts(lpTestScripts, cchTestScripts))
+		return FALSE;
+
+	if(dwFlags & VS_ALLOW_LATIN)
+		LocaleScripts.set(USCRIPT_LATIN);
+
+	return !!(TestScripts <= LocaleScripts);
+}
+
+// EOF

reactos/nls/lib/scripts/scripts.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef REACTOS_SCRIPTS_H_
+#define REACTOS_SCRIPTS_H_
+
+#include <assert.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <string.h>
+#include <wchar.h>
+
+struct SCRIPTS_Script
+{
+	WCHAR ScriptName[4];
+	WCHAR Separator;
+
+	static int Compare(const SCRIPTS_Script * x, const SCRIPTS_Script * y)
+	{
+		for(unsigned i = 0; i < 4; ++ i)
+		{
+			int comparison = (int)x->ScriptName[i] - (int)y->ScriptName[i];
+
+			if(comparison)
+				return comparison;
+		}
+
+		return 0;
+	}
+
+	static int Compare(const void * x, const void * y)
+	{
+		return Compare(static_cast<const SCRIPTS_Script *>(x), static_cast<const SCRIPTS_Script *>(y));
+	}
+};
+
+C_ASSERT(sizeof(SCRIPTS_Script) == 5 * sizeof(WCHAR));
+
+extern "C" bool SCRIPTS_GetCharScriptCode(UChar32 c, int32_t * code);
+extern "C" bool SCRIPTS_GetScriptCode(const SCRIPTS_Script * pScript, int32_t * code);
+extern "C" void SCRIPTS_GetScriptName(int32_t code, SCRIPTS_Script * pScript);
+
+#endif
+
+/* EOF */

reactos/nls/lib/scripts/scripts.rbuild

@@ -0,0 +1,9 @@
+<?xml version="1.0"?>
+<!DOCTYPE module SYSTEM "../../../tools/rbuild/project.dtd">
+<module name="scripts" type="staticlibrary">
+	<library>icu4ros</library>
+	<compilerflag>-fno-exceptions</compilerflag>
+	<compilerflag>-fno-rtti</compilerflag>
+	<include base="icu4ros">icu/source/common</include>
+	<file>scripts.cpp</file>
+</module>

reactos/nls/nls.rbuild

@@ -4,10 +4,19 @@
 	<directory name="3rdparty"><xi:include href="3rdparty/icu4ros.rbuild" /></directory>
 
 	<directory name="lib">
+		<directory name="idna"><xi:include href="lib/idna/idna.rbuild" /></directory>
 		<directory name="normalize"><xi:include href="lib/normalize/normalize.rbuild" /></directory>
+		<directory name="scripts"><xi:include href="lib/scripts/scripts.rbuild" /></directory>
 	</directory>
 
 	<directory name="dll">
+		<directory name="idndl"><xi:include href="dll/idndl/idndl.rbuild" /></directory>
+		<directory name="idndl_redist"><xi:include href="dll/idndl_redist/idndl_redist.rbuild" /></directory>
+		<directory name="normaliz"><xi:include href="dll/normaliz/normaliz.rbuild" /></directory>
 		<directory name="normaliz_redist"><xi:include href="dll/normaliz_redist/normaliz_redist.rbuild" /></directory>
 	</directory>
+
+	<directory name="tests">
+		<directory name="normalization"><xi:include href="tests/normalization/normalizationTest.rbuild" /></directory>
+	</directory>
 </group>

reactos/nls/tests/normalization/normalizationTest.c

@@ -0,0 +1,391 @@
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <wchar.h>
+
+#include <unicode/uchar.h>
+#include <unicode/utf.h>
+
+#define WINVER 0x0600
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+#include "normalizationTest.h"
+
+static int (WINAPI * pfnNormalizeString)(NORM_FORM, LPCWSTR, int, LPWSTR, int);
+
+char * FormatString(const struct TestSuite * test, const WCHAR * str)
+{
+	int length;
+	int codepoints = 0;
+	int formattedLength = 0;
+	char * formattedPos;
+	int formattedRemaining;
+	char * formattedString;
+	int i;
+
+	if(str == NULL)
+		return NULL;
+
+	length = wcslen(str);
+
+	for(i = 0; i < length; )
+	{
+		UChar32 c;
+		U16_NEXT(str, i, length, c);
+
+		++ codepoints;
+
+		assert(c <= 0x10ffff);
+
+		if(c < 0x10000)
+			formattedLength += sizeof("U+ffff") - 1;
+		else if(c < 0x100000)
+			formattedLength += sizeof("U+fffff") - 1;
+		else
+			formattedLength += sizeof("U+10ffff") - 1;
+	}
+
+	if(codepoints)
+		formattedLength += (sizeof(" ") - 1) * (codepoints - 1);
+
+	formattedLength += sizeof("");
+
+	formattedString = malloc(formattedLength);
+
+	if(formattedString == NULL)
+	{
+		fprintf(stderr, "ERROR: %s:%d: @Part%d: out of memory\n", test->filename, test->line, test->part);
+		return NULL;
+	}
+
+	formattedPos = formattedString;
+	formattedRemaining = formattedLength;
+
+	for(i = 0; i < length; )
+	{
+		int unitLength;
+		UChar32 c;
+
+		U16_NEXT(str, i, length, c);
+
+		assert(formattedRemaining == (formattedLength - (formattedPos - formattedString)));
+
+		if(c < 0x10000)
+		{
+			unitLength = _snprintf(formattedPos, formattedRemaining, "U+%04x", c);
+			assert(unitLength == (sizeof("U+ffff") - 1));
+		}
+		else if(c < 0x100000)
+		{
+			unitLength = _snprintf(formattedPos, formattedRemaining, "U+%05x", c);
+			assert(unitLength == (sizeof("U+fffff") - 1));
+		}
+		else
+		{
+			assert(c <= 0x10ffff);
+			unitLength = _snprintf(formattedPos, formattedRemaining, "U+%06x", c);
+			assert(unitLength == (sizeof("U+10ffff") - 1));
+		}
+
+		assert(unitLength >= 0);
+
+		if(unitLength < 0)
+		{
+			fprintf
+			(
+				stderr,
+				"ERROR: %s:%d: @Part%d: couldn't format codepoint %x\n",
+				test->filename, test->line, test->part,
+				c
+			);
+
+			free(formattedString);
+			return NULL;
+		}
+
+		formattedPos += unitLength;
+		formattedRemaining -= unitLength;
+
+		assert(formattedRemaining >= 0);
+
+		if(i < length)
+		{
+			*formattedPos = ' ';
+			formattedPos += sizeof(" ") - 1;
+			formattedRemaining -= sizeof(" ") - 1;
+			assert(formattedRemaining > 0);
+		}
+	}
+
+	formattedString[formattedLength - 1] = 0;
+	return formattedString;
+}
+
+static WCHAR * Normalize(const struct TestSuite * test, const WCHAR * str, NORM_FORM NormForm)
+{
+	LPWSTR lpDstString = NULL;
+	int cwDstLength = 0;
+	int cwRet;
+
+	for(;;)
+	{
+		if(cwDstLength)
+		{
+			lpDstString = malloc(cwDstLength * sizeof(WCHAR));
+
+			if(lpDstString == NULL)
+			{
+				fprintf(stderr, "ERROR: %s:%d: @Part%d: out of memory\n", test->filename, test->line, test->part);
+				return NULL;
+			}
+		}
+
+		cwRet = pfnNormalizeString(NormForm, str, -1, lpDstString, cwDstLength);
+
+		if(cwRet > 0)
+		{
+			if(cwDstLength)
+				break;
+			else
+			{
+				cwDstLength = cwRet;
+				continue;
+			}
+		}
+
+		free(lpDstString);
+		lpDstString = NULL;
+
+		switch(GetLastError())
+		{
+		case ERROR_SUCCESS:
+			/* Should never happen (length of the buffer must be at least 1) */
+			assert(0);
+			return NULL;
+
+		case ERROR_INSUFFICIENT_BUFFER:
+			break;
+
+		case ERROR_NO_UNICODE_TRANSLATION:
+			{
+				int i;
+				int invalid;
+				WCHAR invalidChar[U16_MAX_LENGTH + 1];
+				char * invalidCharFormatted;
+				UChar32 c;
+				size_t strLen;
+
+				strLen = wcslen(str);
+				invalid = -cwRet;
+
+				if(str[invalid] == 0 && invalid > 0 && U_IS_SURROGATE(str[invalid - 1]))
+					-- invalid;
+				else if(U_IS_TRAIL(str[invalid]) && invalid > 0 && U_IS_LEAD(str[invalid - 1]))
+					-- invalid;
+
+				i = invalid;
+				U16_NEXT(str, i, strLen, c);
+
+				i = 0;
+				U16_APPEND_UNSAFE(invalidChar, i, c);
+				invalidChar[i] = 0;
+
+				invalidCharFormatted = FormatString(test, invalidChar);
+
+				fprintf
+				(
+					stderr,
+					"ERROR: %s:%d: @Part%d: invalid Unicode character <%s>\n",
+					test->filename, test->line, test->part,
+					invalidCharFormatted
+				);
+
+				free(invalidCharFormatted);
+			}
+
+			return NULL;
+
+		default:
+			fprintf
+			(
+				stderr,
+				"ERROR: %s:%d: @Part%d: couldn't normalize: system error %lu\n",
+				test->filename, test->line, test->part,
+				GetLastError()
+			);
+
+			return NULL;
+		}
+
+		cwDstLength = -cwRet;
+	}
+
+	assert(lpDstString);
+	return lpDstString;
+}
+
+static void Test
+(
+	const struct TestSuite * test,
+	NORM_FORM NormForm,
+	const WCHAR * data,
+	const WCHAR * expected
+)
+{
+	const char * normForm = "???";
+	WCHAR * actual = NULL;
+	char * dataFormatted = NULL;
+	char * expectedFormatted = NULL;
+	char * actualFormatted = NULL;
+
+	assert(data);
+	assert(expected);
+
+	switch(NormForm)
+	{
+		case NormalizationC: normForm = "NFC"; break;
+		case NormalizationD: normForm = "NFD"; break;
+		case NormalizationKC: normForm = "NFKC"; break;
+		case NormalizationKD: normForm = "NFKD"; break;
+		default: assert(0); break;
+	}
+
+	dataFormatted = FormatString(test, data);
+	expectedFormatted = FormatString(test, expected);
+
+	actual = Normalize(test, data, NormForm);
+
+	if(!expected || !actual)
+	{
+		fprintf
+		(
+			stderr,
+			"SKIP: %s:%d @Part%d: %s(<%s>): earlier errors\n",
+			test->filename, test->line, test->part,
+			normForm,
+			dataFormatted
+		);
+	}
+	else
+	{
+		actualFormatted = FormatString(test, actual);
+
+		if(wcscmp(expected, actual))
+		{
+			fprintf
+			(
+				stderr,
+				"FAIL: %s:%d: @Part%d: %s(<%s>): expected <%s>, got <%s>\n",
+				test->filename, test->line, test->part,
+				normForm,
+				dataFormatted, expectedFormatted, actualFormatted
+			);
+		}
+		else
+		{
+			/*fprintf
+			(
+				stdout,
+				"PASS: %s:%d @Part%d: %s(<%s>): <%s>\n",
+				test->filename, test->line, test->part,
+				normForm,
+				dataFormatted, expectedFormatted
+			);*/
+		}
+	}
+
+	free(dataFormatted);
+	free(expectedFormatted);
+	free(actualFormatted);
+	free(actual);
+}
+
+void NORMTEST_testInvariant(const struct TestSuite * test, const WCHAR * c)
+{
+	Test(test, NormalizationC, c, c);
+	Test(test, NormalizationD, c, c);
+	Test(test, NormalizationKC, c, c);
+	Test(test, NormalizationKD, c, c);
+}
+
+void NORMTEST_test(const struct TestSuite * test, const WCHAR * c1, const WCHAR * c2, const WCHAR * c3, const WCHAR * c4, const WCHAR * c5)
+{
+	Test(test, NormalizationC, c1, c2);
+	Test(test, NormalizationC, c2, c2);
+	Test(test, NormalizationC, c3, c2);
+	Test(test, NormalizationC, c4, c4);
+	Test(test, NormalizationC, c5, c4);
+	Test(test, NormalizationD, c1, c3);
+	Test(test, NormalizationD, c2, c3);
+	Test(test, NormalizationD, c3, c3);
+	Test(test, NormalizationD, c4, c5);
+	Test(test, NormalizationD, c5, c5);
+	Test(test, NormalizationKC, c1, c4);
+	Test(test, NormalizationKC, c2, c4);
+	Test(test, NormalizationKC, c3, c4);
+	Test(test, NormalizationKC, c4, c4);
+	Test(test, NormalizationKC, c5, c4);
+	Test(test, NormalizationKD, c1, c5);
+	Test(test, NormalizationKD, c2, c5);
+	Test(test, NormalizationKD, c3, c5);
+	Test(test, NormalizationKD, c4, c5);
+	Test(test, NormalizationKD, c5, c5);
+}
+
+int main(int argc, const char * const * argv)
+{
+	struct TestSuite testSuite;
+	HMODULE hmNormaliz;
+
+	hmNormaliz = LoadLibrary(TEXT("normaliz")); /* TODO: error handling */
+	pfnNormalizeString = (void *)GetProcAddress(hmNormaliz, "NormalizeString");
+
+	testSuite.filename = "NormalizationTest.txt";
+	testSuite.part = -1;
+
+	if(argc > 1)
+		testSuite.filename = argv[1];
+
+	testSuite.file = fopen(testSuite.filename, "rb"); /* TODO: error handling */
+
+	NORMTEST_runTests(&testSuite);
+
+	fclose(testSuite.file);
+
+	return 0;
+}
+
+/* EOF */

reactos/nls/tests/normalization/normalizationTest.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+struct TestSuite
+{
+	const char * filename;
+	int line;
+	int part;
+
+	FILE * file;
+};
+
+extern void * NORMTEST_createScanner(struct TestSuite * test);
+extern void NORMTEST_deleteScanner(void * scanner);
+
+extern void NORMTEST_runTests(struct TestSuite * test);
+
+extern void NORMTEST_test(const struct TestSuite * test, const WCHAR * c1, const WCHAR * c2, const WCHAR * c3, const WCHAR * c4, const WCHAR * c5);
+extern void NORMTEST_testInvariant(const struct TestSuite * test, const WCHAR * c);
+
+/* EOF */

reactos/nls/tests/normalization/normalizationTest.l

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+%{
+#include <stdlib.h>
+#include <stdio.h>
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+#include "normalizationTest.h"
+
+#define YY_EXTRA_TYPE struct TestSuite *
+#define YY_DECL int yylex (YYSTYPE * pyylval, int * pyylloc, yyscan_t yyscanner)
+
+#include "normalizationTest.tab.h"
+%}
+
+%option batch
+%option reentrant
+%option nounistd
+%option nodefault
+
+%%
+
+"#"[^\r\n]*  /* Ignore comments */
+[\t\f\v ]+   /* Ignore whitespace */
+
+;            return TOKEN_COLUMN_SEPARATOR;
+
+\r\n         { ++ *pyylloc; return TOKEN_EOL; }
+\r           { ++ *pyylloc; return TOKEN_EOL; }
+\n           { ++ *pyylloc; return TOKEN_EOL; }
+
+@Part[0-9]+  {
+	pyylval->part = strtoul(yytext + 5, NULL, 10);
+	return TOKEN_PART_HEADER;
+}
+
+[0-9A-Fa-f]+ {
+	pyylval->codepoint = strtoul(yytext, NULL, 16);
+	return TOKEN_CODEPOINT;
+}
+
+.            {
+	struct TestSuite * test = yyextra;
+
+	test->line = *pyylloc;
+
+	fprintf
+	(
+		stderr,
+		"FATAL: %s:%d: @Part%d: invalid character \"%s\"\n",
+		test->filename, test->line, test->part,
+		yytext
+	);
+
+	yyterminate();
+}
+
+%%
+void * NORMTEST_createScanner(struct TestSuite * test)
+{
+	yyscan_t scanner;
+	(void)yyunput;
+	yylex_init(&scanner);
+	yyset_in(test->file, scanner);
+	yyset_extra(test, scanner);
+	return (void *)scanner;
+}
+
+void NORMTEST_deleteScanner(void * scanner)
+{
+	yylex_destroy((yyscan_t)scanner);
+}
+
+int yywrap(yyscan_t scanner) { return 1; }
+
+/* EOF */

reactos/nls/tests/normalization/normalizationTest.rbuild

@@ -0,0 +1,10 @@
+<?xml version="1.0"?>
+<module name="normalizationTest" type="win32cui">
+	<define name="U_HAVE_INTTYPES_H" />
+	<include base="icu4ros">icu/source/common</include>
+	<library>kernel32</library>
+	<library>normaliz</library>
+	<file>normalizationTest.c</file>
+	<file>normalizationTest.tab.c</file>
+	<file>normalizationTest.yy.c</file>
+</module>

reactos/nls/tests/normalization/normalizationTest.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 9.00
+# Visual Studio 2005
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "normalizationTest", "normalizationTest.vcproj", "{F316D362-6B02-469F-8C47-9E28E472DC58}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Win32 = Debug|Win32
+		Release|Win32 = Release|Win32
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{F316D362-6B02-469F-8C47-9E28E472DC58}.Debug|Win32.ActiveCfg = Debug|Win32
+		{F316D362-6B02-469F-8C47-9E28E472DC58}.Debug|Win32.Build.0 = Debug|Win32
+		{F316D362-6B02-469F-8C47-9E28E472DC58}.Release|Win32.ActiveCfg = Release|Win32
+		{F316D362-6B02-469F-8C47-9E28E472DC58}.Release|Win32.Build.0 = Release|Win32
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

reactos/nls/tests/normalization/normalizationTest.tab.h

@@ -0,0 +1,78 @@
+/* A Bison parser, made by GNU Bison 2.1.  */
+
+/* Skeleton parser for Yacc-like parsing with Bison,
+   Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor,
+   Boston, MA 02110-1301, USA.  */
+
+/* As a special exception, when this file is copied by Bison into a
+   Bison output file, you may use that output file without restriction.
+   This special exception was added by the Free Software Foundation
+   in version 1.24 of Bison.  */
+
+/* Tokens.  */
+#ifndef YYTOKENTYPE
+# define YYTOKENTYPE
+   /* Put the tokens into the symbol table, so that GDB and other debuggers
+      know about them.  */
+   enum yytokentype {
+     TOKEN_COLUMN_SEPARATOR = 258,
+     TOKEN_EOL = 259,
+     TOKEN_PART_HEADER = 260,
+     TOKEN_CODEPOINT = 261
+   };
+#endif
+/* Tokens.  */
+#define TOKEN_COLUMN_SEPARATOR 258
+#define TOKEN_EOL 259
+#define TOKEN_PART_HEADER 260
+#define TOKEN_CODEPOINT 261
+
+
+
+
+#if ! defined (YYSTYPE) && ! defined (YYSTYPE_IS_DECLARED)
+#line 87 "normalizationTest.y"
+typedef union YYSTYPE {
+	long part;
+	long codepoint;
+	WCHAR * str;
+} YYSTYPE;
+/* Line 1447 of yacc.c.  */
+#line 56 "normalizationTest.tab.h"
+# define yystype YYSTYPE /* obsolescent; will be withdrawn */
+# define YYSTYPE_IS_DECLARED 1
+# define YYSTYPE_IS_TRIVIAL 1
+#endif
+
+
+
+#if ! defined (YYLTYPE) && ! defined (YYLTYPE_IS_DECLARED)
+typedef struct YYLTYPE
+{
+  int first_line;
+  int first_column;
+  int last_line;
+  int last_column;
+} YYLTYPE;
+# define yyltype YYLTYPE /* obsolescent; will be withdrawn */
+# define YYLTYPE_IS_DECLARED 1
+# define YYLTYPE_IS_TRIVIAL 1
+#endif
+
+
+
+

reactos/nls/tests/normalization/normalizationTest.vcproj

@@ -0,0 +1,222 @@
+<?xml version="1.0" encoding="Windows-1252"?>
+<VisualStudioProject
+	ProjectType="Visual C++"
+	Version="8,00"
+	Name="normalizationTest"
+	ProjectGUID="{F316D362-6B02-469F-8C47-9E28E472DC58}"
+	RootNamespace="normalizationTest"
+	Keyword="Win32Proj"
+	>
+	<Platforms>
+		<Platform
+			Name="Win32"
+		/>
+	</Platforms>
+	<ToolFiles>
+	</ToolFiles>
+	<Configurations>
+		<Configuration
+			Name="Debug|Win32"
+			OutputDirectory="$(SolutionDir)$(ConfigurationName)"
+			IntermediateDirectory="$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="../../3rdparty/icu/source/common"
+				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				SmallerTypeCheck="true"
+				RuntimeLibrary="3"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="4"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				AdditionalDependencies="kernel32.lib $(NOINHERIT)"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCWebDeploymentTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|Win32"
+			OutputDirectory="$(SolutionDir)$(ConfigurationName)"
+			IntermediateDirectory="$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="1"
+			WholeProgramOptimization="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				AdditionalIncludeDirectories="../../3rdparty/icu/source/common"
+				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+				RuntimeLibrary="2"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				AdditionalDependencies="kernel32.lib $(NOINHERIT)"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCWebDeploymentTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+	</Configurations>
+	<References>
+	</References>
+	<Files>
+		<Filter
+			Name="Source Files"
+			Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
+			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
+			>
+			<File
+				RelativePath=".\normalizationTest.c"
+				>
+			</File>
+			<File
+				RelativePath=".\normalizationTest.tab.c"
+				>
+			</File>
+			<File
+				RelativePath=".\normalizationTest.yy.c"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Header Files"
+			Filter="h;hpp;hxx;hm;inl;inc;xsd"
+			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
+			>
+			<File
+				RelativePath=".\normalizationTest.h"
+				>
+			</File>
+			<File
+				RelativePath=".\normalizationTest.tab.h"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Resource Files"
+			Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav"
+			UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
+			>
+			<File
+				RelativePath=".\NormalizationTest.txt"
+				>
+			</File>
+		</Filter>
+	</Files>
+	<Globals>
+	</Globals>
+</VisualStudioProject>

reactos/nls/tests/normalization/normalizationTest.y

@@ -0,0 +1,189 @@
+/*
+ * Copyright (c) 2008, KJK::Hyperion
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ *  - Neither the name of the ReactOS Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+%{
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include <unicode/uchar.h>
+#include <unicode/utf.h>
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+#include "normalizationTest.h"
+
+static void part1TestInvariants(const struct TestSuite * test, UChar32 from, UChar32 to)
+{
+	for(; from < to; ++ from)
+	{
+		int i = 0;
+		WCHAR szInvariant[U16_MAX_LENGTH + 1];
+
+		if(U_IS_SURROGATE(from))
+			continue;
+
+		U16_APPEND_UNSAFE(szInvariant, i, from);
+		szInvariant[i] = 0;
+
+		NORMTEST_testInvariant(test, szInvariant);
+	}
+}
+
+struct parserState
+{
+	void * scanner;
+	UChar32 lastCodepoint;
+	struct TestSuite * test;
+};
+
+#define YYLTYPE int
+#define YYLLOC_DEFAULT(Current, Rhs, N) for(;;) { (Current) = YYRHSLOC (Rhs, (N)); break; }
+#define YY_LOCATION_PRINT(File, Loc) fprintf((File), "%d", (Loc))
+
+#define YYLEX_PARAM state->scanner
+
+#include "normalizationTest.tab.h"
+
+extern int yylex(void *, int *, void *);
+
+static void yyerror(const YYLTYPE *, struct parserState *, const char *);
+
+%}
+
+%pure-parser
+%defines
+
+%union {
+	long part;
+	long codepoint;
+	WCHAR * str;
+}
+
+%token             TOKEN_COLUMN_SEPARATOR
+%token             TOKEN_EOL
+%token <part>      TOKEN_PART_HEADER
+%token <codepoint> TOKEN_CODEPOINT
+
+%type <str> string
+%type <str> column
+
+%destructor { free($$); } string column
+
+%parse-param { struct parserState * state }
+
+%initial-action {
+	@$ = 1;
+}
+
+%%
+
+testSuite : eol parts
+
+parts :
+      | parts part
+
+part : part_header tests {
+	if(state->test->part == 1)
+		part1TestInvariants(state->test, state->lastCodepoint + 1, UCHAR_MAX_VALUE + 1);
+}
+
+part_header : TOKEN_PART_HEADER eol {
+	state->test->part = $1;
+	state->lastCodepoint = 1;
+}
+
+tests : test
+      | tests test
+
+test : column column column column column eol {
+	@$ = @1;
+	state->test->line = @$;
+
+	if(state->test->part == 1)
+	{
+		UChar32 codepoint;
+		U16_GET_UNSAFE($1, 0, codepoint);
+		assert(codepoint > state->lastCodepoint);
+		part1TestInvariants(state->test, state->lastCodepoint + 1, codepoint);
+		state->lastCodepoint = codepoint;
+	}
+
+	NORMTEST_test(state->test, $1, $2, $3, $4, $5);
+
+	free($1);
+	free($2);
+	free($3);
+	free($4);
+	free($5);
+}
+
+column : string TOKEN_COLUMN_SEPARATOR
+
+string :                        { $$ = NULL; }
+       | string TOKEN_CODEPOINT {
+	size_t cchString = 0;
+
+	if($1)
+		cchString = wcslen($1);
+
+	$$ = malloc((cchString + U16_LENGTH($2) + 1) * sizeof(WCHAR));
+
+	memcpy($$, $1, cchString * sizeof(WCHAR));
+	U16_APPEND_UNSAFE($$, cchString, $2);
+	$$[cchString] = 0;
+
+	free($1);
+}
+
+eol :
+	| eol TOKEN_EOL
+
+%%
+
+void NORMTEST_runTests(struct TestSuite * test)
+{
+	struct parserState parser;
+	parser.test = test;
+	parser.scanner = NORMTEST_createScanner(test);
+	yyparse(&parser);
+}
+
+static void yyerror(const YYLTYPE * loc, struct parserState * state, const char * message)
+{
+	struct TestSuite * test = state->test;
+	test->line = *loc;
+	fprintf(stderr, "FATAL: %s:%d: @Part%d: %s\n", test->filename, test->line, test->part, message);
+}
+
+/* EOF */