Intravision/reactos
1
0
Fork 0
mirror of https://github.com/reactos/reactos.git synced 2025-06-13 09:38:30 +00:00
Code Issues Projects Releases Packages Wiki Activity

Git conversion: Make reactos the root directory, move rosapps, rostests, wallpapers into modules, and delete rossubsys.

Browse source
This commit is contained in:
Colin Finck 2017-10-03 07:45:34 +00:00
parent b94e2d8ca0
commit c2c66aff7d
24198 changed files with 0 additions and 37285 deletions
Download patch file Download diff file Expand all files Collapse all files

312
modules/rosapps/applications/sysutils/utils/sdkparse/tokenize.cpp Normal file
View file

@ -0,0 +1,312 @@
// tokenize.cpp
#ifdef _MSC_VER
#pragma warning ( disable : 4786 )
#endif//_MSC_VER
#include <string>
#include <vector>
#include <conio.h>
#include "assert.h"
#include "tokenize.h"
#include "skip_ws.h"
using std::string;
using std::vector;
void tokenize ( const string& text, vector<string>& tokens )
{
tokens.resize ( 0 );
string s ( text );
char* p = &s[0];
while ( *p )
{
// skip whitespace
p = skip_ws ( p );
// check for literal string
if ( *p == '\"' )
{
// skip initial quote
char* end = p + 1;
for ( ;; )
{
if ( *end == '\\' )
{
end++;
switch ( *end )
{
case 'x':
case 'X':
ASSERT(0); // come back to this....
break;
case '0':
ASSERT(0);
break;
default:
end++;
break;
}
}
else if ( *end == '\"' )
{
end++;
break;
}
else
end++;
}
tokens.push_back ( string ( p, end-p ) );
p = end;
}
else if ( __iscsymf(*p) )
{
char* end = p + 1;
while ( __iscsym ( *end ) )
end++;
tokens.push_back ( string ( p, end-p ) );
p = end;
}
else if ( isdigit(*p) || *p == '.' )
{
char* end = p;
while ( isdigit(*end) )
end++;
bool f = false;
if ( *end == '.' )
{
end++;
while ( isdigit(*end) )
end++;
f = true;
}
if ( *end == 'f' || *end == 'F' )
end++;
else if ( !f && ( *end == 'l' || *end == 'L' ) )
end++;
tokens.push_back ( string ( p, end-p ) );
p = end;
}
else switch ( *p )
{
case '.':
tokens.push_back ( "." );
p++;
break;
case ',':
tokens.push_back ( "," );
p++;
break;
case '(':
tokens.push_back ( "(" );
p++;
break;
case ')':
tokens.push_back ( ")" );
p++;
break;
case '{':
tokens.push_back ( "{" );
p++;
break;
case '}':
tokens.push_back ( "}" );
p++;
break;
case '[':
tokens.push_back ( "[" );
p++;
break;
case ']':
tokens.push_back ( "]" );
p++;
break;
case ';':
tokens.push_back ( ";" );
p++;
break;
case '\\':
switch ( p[1] )
{
case '\n':
tokens.push_back ( string ( p, 2 ) );
p += 2;
break;
default:
ASSERT(0); // shouldn't hit here, I think
tokens.push_back ( "\\" );
p++;
break;
}
break;
case '|':
switch ( p[1] )
{
case '|':
tokens.push_back ( string ( p, 2 ) );
p += 2;
break;
default:
tokens.push_back ( "|" );
p++;
break;
}
break;
case '&':
switch ( p[1] )
{
case '&':
tokens.push_back ( string ( p, 2 ) );
p += 2;
break;
default:
tokens.push_back ( "&" );
p++;
break;
}
break;
case '<':
switch ( p[1] )
{
case '<':
if ( p[2] == '=' )
tokens.push_back ( string ( p, 3 ) ), p += 3;
else
tokens.push_back ( string ( p, 2 ) ), p += 2;
break;
case '=':
tokens.push_back ( string ( p, 2 ) );
p += 2;
break;
default:
tokens.push_back ( "<" );
p++;
break;
}
break;
case '>':
switch ( p[1] )
{
case '>':
if ( p[2] == '=' )
tokens.push_back ( string ( p, 3 ) ), p += 3;
else
tokens.push_back ( string ( p, 2 ) ), p += 2;
break;
case '=':
tokens.push_back ( string ( p, 2 ) );
p += 2;
break;
default:
tokens.push_back ( ">" );
p++;
break;
}
break;
case '!':
switch ( p[1] )
{
case '=':
tokens.push_back ( string ( p, 2 ) );
p += 2;
break;
default:
tokens.push_back ( "!" );
p++;
break;
}
break;
case '=':
switch ( p[1] )
{
case '=':
tokens.push_back ( string ( p, 2 ) );
p += 2;
break;
default:
tokens.push_back ( "=" );
p++;
break;
}
break;
case ':':
switch ( p[1] )
{
case ':':
tokens.push_back ( string ( p, 2 ) );
p += 2;
break;
default:
tokens.push_back ( ":" );
p++;
break;
}
break;
case '*':
switch ( p[1] )
{
case '=':
tokens.push_back ( string ( p, 2 ) );
p += 2;
break;
default:
tokens.push_back ( "*" );
p++;
break;
}
break;
case '/':
switch ( p[1] )
{
case '=':
tokens.push_back ( string ( p, 2 ) );
p += 2;
break;
default:
tokens.push_back ( "/" );
p++;
break;
}
break;
case '+':
switch ( p[1] )
{
case '+':
case '=':
tokens.push_back ( string ( p, 2 ) );
p += 2;
break;
default:
tokens.push_back ( "+" );
p++;
break;
}
break;
case '-':
switch ( p[1] )
{
case '-':
case '=':
tokens.push_back ( string ( p, 2 ) );
p += 2;
break;
default:
tokens.push_back ( "-" );
p++;
break;
}
break;
case '#':
while ( *p && *p != '\n' )
p++;
break;
case 0:
break;
default:
printf ( "choked on '%c' in tokenize() - press any key to continue\n", *p );
getch();
p++;
break;
}
}
}