[ATL][ATL_APITEST] Add CString::Tokenize + testcase

modules/rostests/apitests/atl/CString.cpp

@@ -190,4 +190,7 @@ START_TEST(CString)
 
     test_bstrW();
     test_bstrA();
+
+    test_tokenizeW();
+    test_tokenizeA();
 }

modules/rostests/apitests/atl/CString.inl

@@ -439,3 +439,90 @@ TEST_NAMEX(bstr)
         ::SysFreeString(bstr);
     }
 }
+
+TEST_NAMEX(tokenize)
+{
+    CStringX str(_X("%#First Second#Third"));
+    const XCHAR* Tokens = _X("% #");
+    CStringX res;
+
+    int nCurPos = 0;
+
+    // All 'current' tokens are skipped
+    res = str.Tokenize(Tokens, nCurPos);
+    ok(res == _X("First"), "Expected str to be 'First', was: %s\n", dbgstrx(res));
+    ok_dec(nCurPos, 8);
+
+    res = str.Tokenize(Tokens, nCurPos);
+    ok(res == _X("Second"), "Expected str to be 'Second', was: %s\n", dbgstrx(res));
+    ok_dec(nCurPos, 15);
+
+    res = str.Tokenize(Tokens, nCurPos);
+    ok(res == _X("Third"), "Expected str to be 'Third', was: %s\n", dbgstrx(res));
+    ok_dec(nCurPos, 21);
+
+    // The final 'token' is empty, and nCurPos is set to -1
+    // (Calling with nCurPos=-1 will assert)
+    res = str.Tokenize(Tokens, nCurPos);
+    ok(res == _X(""), "Expected str to be '', was: %s\n", dbgstrx(res));
+    ok_dec(nCurPos, -1);
+
+    str =_X("StartWithToken#%#");
+    nCurPos = 0;
+
+    res = str.Tokenize(Tokens, nCurPos);
+    ok(res == _X("StartWithToken"), "Expected str to be 'StartWithToken', was: %s\n", dbgstrx(res));
+    ok_dec(nCurPos, 15);
+
+    // Ending with tokens acts as if there were no tokens at the end
+    res = str.Tokenize(Tokens, nCurPos);
+    ok(res == _X(""), "Expected str to be '', was: %s\n", dbgstrx(res));
+    ok_dec(nCurPos, -1);
+
+
+    str = _X("");
+    nCurPos = 0;
+
+    // Calling with an empty string returns 'no tokens'
+    res = str.Tokenize(Tokens, nCurPos);
+    ok(res == _X(""), "Expected str to be '', was: %s\n", dbgstrx(res));
+    ok_dec(nCurPos, -1);
+
+    str = _X("");
+    nCurPos = 20;
+
+    // Calling with a current position outside the strings acts the same as 'no tokens left'
+    res = str.Tokenize(Tokens, nCurPos);
+    ok(res == _X(""), "Expected str to be '', was: %s\n", dbgstrx(res));
+    ok_dec(nCurPos, -1);
+
+
+    str = _X("test");
+    nCurPos = 2;
+
+    // Calling with a NULL pszTokens returns a substring starting at 'nCurPos', but not updating nCurPos!
+    res = str.Tokenize(NULL, nCurPos);
+    ok(res == _X("st"), "Expected str to be 'st', was: %s\n", dbgstrx(res));
+    ok_dec(nCurPos, 2);
+
+
+    // Calling with an empty pszTokens behaves exactly the same
+    res = str.Tokenize(_X(""), nCurPos);
+    ok(res == _X("st"), "Expected str to be 'st', was: %s\n", dbgstrx(res));
+    ok_dec(nCurPos, 2);
+
+    nCurPos = 8;
+
+    // Calling with a NULL pszTokens and an nCurPos out of bounds returns 'no tokens left'
+    res = str.Tokenize(NULL, nCurPos);
+    ok(res == _X(""), "Expected str to be '', was: %s\n", dbgstrx(res));
+    ok_dec(nCurPos, -1);
+
+    nCurPos = 8;
+
+    // Calling with an empty pszTokens behaves exactly the same
+    res = str.Tokenize(_X(""), nCurPos);
+    ok(res == _X(""), "Expected str to be 'st', was: %s\n", dbgstrx(res));
+    ok(res == _X(""), "Expected str to be '', was: %s\n", dbgstrx(res));
+    ok_dec(nCurPos, -1);
+}

sdk/lib/atl/cstringt.h

@@ -140,6 +140,20 @@ public:
         return ::_wcsicmp(psz1, psz2);
     }
 
+    static int __cdecl StringSpanIncluding(
+        _In_z_ LPCWSTR pszBlock,
+        _In_z_ LPCWSTR pszSet)
+    {
+        return (int)::wcsspn(pszBlock, pszSet);
+    }
+
+    static int __cdecl StringSpanExcluding(
+        _In_z_ LPCWSTR pszBlock,
+        _In_z_ LPCWSTR pszSet)
+    {
+        return (int)::wcscspn(pszBlock, pszSet);
+    }
+
     static int __cdecl FormatV(
         _In_opt_z_ LPWSTR pszDest,
         _In_z_ LPCWSTR pszFormat,
@@ -289,6 +303,20 @@ public:
         return ::_stricmp(psz1, psz2);
     }
 
+    static int __cdecl StringSpanIncluding(
+        _In_z_ LPCSTR pszBlock,
+        _In_z_ LPCSTR pszSet)
+    {
+        return (int)::strspn(pszBlock, pszSet);
+    }
+
+    static int __cdecl StringSpanExcluding(
+        _In_z_ LPCSTR pszBlock,
+        _In_z_ LPCSTR pszSet)
+    {
+        return (int)::strcspn(pszBlock, pszSet);
+    }
+
     static int __cdecl FormatV(
         _In_opt_z_ LPSTR pszDest,
         _In_z_ LPCSTR pszFormat,
@@ -804,6 +832,42 @@ public:
     }
 
 
+    CStringT Tokenize(_In_z_ PCXSTR pszTokens, _Inout_ int& iStart) const
+    {
+        ATLASSERT(iStart >= 0);
+
+        if (iStart < 0)
+            AtlThrow(E_INVALIDARG);
+
+        if (!pszTokens || !pszTokens[0])
+        {
+            if (iStart < CThisSimpleString::GetLength())
+            {
+                return Mid(iStart);
+            }
+            iStart = -1;
+            return CStringT();
+        }
+
+        if (iStart < CThisSimpleString::GetLength())
+        {
+            int iRangeOffset = StringTraits::StringSpanIncluding(CThisSimpleString::GetString() + iStart, pszTokens);
+
+            if (iRangeOffset + iStart < CThisSimpleString::GetLength())
+            {
+                int iNewStart = iStart + iRangeOffset;
+                int nCount = StringTraits::StringSpanExcluding(CThisSimpleString::GetString() + iNewStart, pszTokens);
+
+                iStart = iNewStart + nCount + 1;
+
+                return Mid(iNewStart, nCount);
+            }
+        }
+
+        iStart = -1;
+        return CStringT();
+    }
+
     static PCXSTR DefaultTrimChars()
     {
         static XCHAR str[] = { ' ', '\t', '\r', '\n', 0 };