Cleaned up bare key handling and started cleaning up identifier

validation
This commit is contained in:
moandji.ezana 2015-02-12 11:21:00 +02:00
parent 7292fe5468
commit 8a6ca61101
13 changed files with 419 additions and 154 deletions

View file

@ -2,28 +2,39 @@ package com.moandjiezana.toml;
class Identifier {
static final Identifier INVALID = new Identifier("");
static final Identifier INVALID = new Identifier("", null);
private static final String ALLOWED_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890_-.";
private static final String ALLOWED_CHARS_KEYS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890_-";
private final String name;
private final Type type;
Identifier(String name) {
this.name = name;
if (name.startsWith("[[")) {
this.type = Type.TABLE_ARRAY;
} else if (name.startsWith("[")) {
this.type = Type.TABLE;
} else {
this.type = Type.KEY;
}
}
boolean acceptsNext(char c) {
if (isKey()) {
return c == '=';
static Identifier from(String name, Context context) {
Type type;
boolean valid;
name = name.trim();
if (name.startsWith("[[")) {
type = Type.TABLE_ARRAY;
valid = isValidTableArray(name, context);
} else if (name.startsWith("[")) {
type = Type.TABLE;
valid = isValidTable(name, context);
} else {
type = Type.KEY;
valid = isValidKey(name, context);
}
return c == '\n' || c == '#';
if (!valid) {
return Identifier.INVALID;
}
return new Identifier(extractName(name), type);
}
private Identifier(String name, Type type) {
this.name = name;
this.type = type;
}
String getName() {
@ -42,19 +53,145 @@ class Identifier {
return type == Type.TABLE_ARRAY;
}
boolean isValid() {
if (isKey()) {
return Keys.getKey(name) != null;
}
if (isTable()) {
return Keys.getTableName(name) != null;
}
return Keys.getTableArrayName(name) != null;
}
private static enum Type {
KEY, TABLE, TABLE_ARRAY;
}
private static String extractName(String raw) {
boolean quoted = false;
StringBuilder sb = new StringBuilder();
for (int i = 0; i < raw.length(); i++) {
char c = raw.charAt(i);
if (c == '"' && (i == 0 || raw.charAt(i - 1) != '\\')) {
quoted = !quoted;
sb.append('"');
} else if (quoted || !Character.isWhitespace(c)) {
sb.append(c);
}
}
return sb.toString();
}
private static boolean isValidKey(String name, Context context) {
if (name.trim().isEmpty()) {
context.errors.invalidKey(name, context.line.get());
return false;
}
boolean quoted = false;
char[] chars = name.toCharArray();
for (int i = 0; i < chars.length; i++) {
char c = chars[i];
if (c == '"' && (i == 0 || chars[i - 1] != '\\')) {
if (!quoted && i > 0 && chars [i - 1] != '.') {
context.errors.invalidKey(name, context.line.get());
return false;
}
quoted = !quoted;
} else if (!quoted && (ALLOWED_CHARS_KEYS.indexOf(c) == -1)) {
context.errors.invalidKey(name, context.line.get());
return false;
}
}
return true;
}
private static boolean isValidTable(String name, Context context) {
if (!name.endsWith("]")) {
context.errors.invalidTable(name, context.line.get());
return false;
}
char[] chars = name.toCharArray();
boolean quoted = false;
boolean terminated = false;
int endIndex = -1;
boolean preKey = true;
boolean valid = true;
for (int i = 1; i < name.length() - 1; i++) {
char c = name.charAt(i);
if (c == '"' && chars[i - 1] != '\\') {
if (!quoted && i > 1 && chars [i - 1] != '.' && !Character.isWhitespace(chars[i - 1])) {
valid = false;
break;
}
quoted = !quoted;
} else if (!quoted && c == '.') {
preKey = true;
} else if (!quoted && Character.isWhitespace(c)) {
if (preKey && i > 1 && chars[i - 1] != '.' && !Character.isWhitespace(chars[i - 1])) {
valid = false;
break;
}
if (!preKey && chars.length > i + 1 && chars[i + 1] != '.' && chars[i + 1] != ']' && !Character.isWhitespace(chars[i + 1])) {
valid = false;
break;
}
} else if (!quoted && (ALLOWED_CHARS.indexOf(c) == -1)) {
valid = false;
break;
} else if (!quoted) {
preKey = false;
}
}
if (!valid) {
context.errors.invalidTable(name, context.line.get());
return false;
}
// return StringConverter.STRING_PARSER.replaceUnicodeCharacters(tableName);
return true;
}
private static boolean isValidTableArray(String line, Context context) {
if (!line.endsWith("]]") || line.substring(2, line.length() - 2).trim().isEmpty()) {
context.errors.invalidTableArray(line, context.line.get());
return false;
}
char[] chars = line.toCharArray();
boolean quoted = false;
boolean preKey = true;
boolean valid = true;
for (int i = 2; i < line.length() - 2; i++) {
char c = chars[i];
if (c == '"' && chars[i - 1] != '\\') {
if (!quoted && i > 1 && chars [i - 1] != '.' && !Character.isWhitespace(chars[i - 1])) {
valid = false;
}
quoted = !quoted;
} else if (!quoted && c == '.') {
preKey = true;
} else if (!quoted && Character.isWhitespace(c)) {
if (preKey && i > 2 && chars[i - 1] != '.' && !Character.isWhitespace(chars[i - 1])) {
valid = false;
}
if (!preKey && chars.length > i + 1 && chars[i + 1] != '.' && chars[i + 1] != ']' && !Character.isWhitespace(chars[i + 1])) {
valid = false;
break;
}
} else if (!quoted && (ALLOWED_CHARS.indexOf(c) == -1)) {
valid = false;
} else if (!valid) {
break;
} else {
preKey = false;
}
}
if (!valid) {
context.errors.invalidTableArray(line, context.line.get());
}
return valid;
}
}

View file

@ -6,28 +6,186 @@ class IdentifierConverter {
static final IdentifierConverter IDENTIFIER_CONVERTER = new IdentifierConverter();
Identifier convert(char[] chars, AtomicInteger index) {
private static final String ALLOWED_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890_-.";
Identifier convert(char[] chars, AtomicInteger index, Context context) {
boolean quoted = false;
StringBuilder name = new StringBuilder();
Identifier identifier = null;
boolean terminated = false;
boolean isKey = chars[index.get()] != '[';
boolean isTableArray = !isKey && chars.length > index.get() + 1 && chars[index.get() + 1] == '[';
boolean inComment = false;
for (int i = index.get(); i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (c == '"' && (i == 0 || chars[i - 1] != '\\')) {
quoted = !quoted;
name.append('"');
} else if (c == '\n' || (!quoted && (c == '#' || c == '='))) {
return new Identifier(name.toString().trim());
} else if (i == chars.length - 1 && identifier == null) {
} else if (c == '\n') {
break;
} else if (quoted) {
name.append(c);
return new Identifier(name.toString().trim());
} else {
} else if (c == '=' && isKey) {
terminated = true;
break;
} else if (c == ']' && !isKey) {
if (!isTableArray || chars.length > index.get() + 1 && chars[index.get() + 1] == ']') {
terminated = true;
name.append(']');
if (isTableArray) {
name.append(']');
}
}
} else if (terminated && c == '#') {
inComment = true;
} else if (terminated && !Character.isWhitespace(c) && !inComment) {
terminated = false;
break;
} else if (!terminated) {
name.append(c);
}
}
return identifier != null ? identifier : Identifier.INVALID;
if (!terminated) {
if (isKey) {
context.errors.unterminatedKey(name.toString(), context.line.get());
} else {
context.errors.invalidKey(name.toString(), context.line.get());
}
return Identifier.INVALID;
}
return Identifier.from(name.toString(), context);
}
private IdentifierConverter() {}
private static String getKey(String key) {
key = key.trim();
if (key.isEmpty()) {
return null;
}
boolean quoted = false;
char[] chars = key.toCharArray();
StringBuilder sb = new StringBuilder(key.length());
for (int i = 0; i < chars.length; i++) {
char c = chars[i];
if (c == '"' && (i == 0 || chars[i - 1] != '\\')) {
if (!quoted && i > 0 && chars [i - 1] != '.') {
return null;
}
quoted = !quoted;
} else if (!quoted && (ALLOWED_CHARS.indexOf(c) == -1)) {
return null;
}
sb.append(c);
}
return sb.toString();
}
private static String getTableArrayName(String line) {
StringBuilder sb = new StringBuilder();
char[] chars = line.toCharArray();
boolean quoted = false;
boolean terminated = false;
int endIndex = -1;
boolean preKey = true;
for (int i = 2; i < chars.length; i++) {
char c = chars[i];
if (c == '"' && chars[i - 1] != '\\') {
if (!quoted && i > 1 && chars [i - 1] != '.' && !Character.isWhitespace(chars[i - 1])) {
break;
}
quoted = !quoted;
} else if (!quoted && c == ']') {
if (chars.length > i + 1 && chars[i + 1] == ']') {
terminated = true;
endIndex = i + 1;
break;
}
} else if (!quoted && c == '.') {
preKey = true;
} else if (!quoted && Character.isWhitespace(c)) {
if (preKey && i > 2 && chars[i - 1] != '.' && !Character.isWhitespace(chars[i - 1])) {
break;
}
if (!preKey && chars.length > i + 1 && chars[i + 1] != '.' && chars[i + 1] != ']' && !Character.isWhitespace(chars[i + 1])) {
break;
}
continue;
} else if (!quoted && (ALLOWED_CHARS.indexOf(c) == -1)) {
break;
} else {
preKey = false;
}
sb.append(c);
}
if (!terminated || sb.length() == 0) {
return null;
}
String tableName = sb.insert(0, "[[").append("]]").toString();
return StringConverter.STRING_PARSER.replaceUnicodeCharacters(tableName);
}
private static String getTableName(String line) {
StringBuilder sb = new StringBuilder();
char[] chars = line.toCharArray();
boolean quoted = false;
boolean terminated = false;
int endIndex = -1;
boolean preKey = true;
for (int i = 1; i < chars.length; i++) {
char c = chars[i];
if (c == '"' && chars[i - 1] != '\\') {
if (!quoted && i > 1 && chars [i - 1] != '.' && !Character.isWhitespace(chars[i - 1])) {
break;
}
quoted = !quoted;
} else if (!quoted && c == ']') {
terminated = true;
endIndex = i;
break;
} else if (!quoted && c == '.') {
preKey = true;
} else if (!quoted && Character.isWhitespace(c)) {
if (preKey && i > 1 && chars[i - 1] != '.' && !Character.isWhitespace(chars[i - 1])) {
break;
}
if (!preKey && chars.length > i + 1 && chars[i + 1] != '.' && chars[i + 1] != ']' && !Character.isWhitespace(chars[i + 1])) {
break;
}
continue;
} else if (!quoted && (ALLOWED_CHARS.indexOf(c) == -1)) {
break;
} else if (!quoted) {
preKey = false;
}
sb.append(c);
}
if (!terminated) {
return null;
}
sb.insert(0, '[').append(']');
String tableName = sb.toString();
return StringConverter.STRING_PARSER.replaceUnicodeCharacters(tableName);
}
}

View file

@ -37,7 +37,7 @@ class InlineTableConverter implements ValueConverter {
} else if (quoted) {
currentKey.append(c);
} else if (inValue && !Character.isWhitespace(c)) {
Object converted = CONVERTERS.convert(s, sharedIndex, context.with(new Identifier(currentKey.toString())));
Object converted = CONVERTERS.convert(s, sharedIndex, context.with(Identifier.from(currentKey.toString(), context)));
if (converted instanceof Results.Errors) {
errors.add((Results.Errors) converted);
@ -49,7 +49,7 @@ class InlineTableConverter implements ValueConverter {
inValue = false;
} else if (c == '{') {
sharedIndex.incrementAndGet();
Object converted = convert(s, sharedIndex, context.with(new Identifier(currentKey.toString())));
Object converted = convert(s, sharedIndex, context.with(Identifier.from(currentKey.toString(), context)));
if (converted instanceof Results.Errors) {
errors.add((Results.Errors) converted);

View file

@ -1,6 +1,6 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import static com.moandjiezana.toml.ValueConverters.isComment;
import java.util.ArrayList;
import java.util.List;
@ -25,34 +25,6 @@ class Keys {
}
}
static String getKey(String key) {
key = key.trim();
if (key.isEmpty()) {
return null;
}
boolean quoted = false;
char[] chars = key.toCharArray();
StringBuilder sb = new StringBuilder(key.length());
for (int i = 0; i < chars.length; i++) {
char c = chars[i];
if (c == '"' && (i == 0 || chars[i - 1] != '\\')) {
if (!quoted && i > 0 && chars [i - 1] != '.') {
return null;
}
quoted = !quoted;
} else if (!quoted && (ALLOWED_CHARS.indexOf(c) == -1)) {
return null;
}
sb.append(c);
}
return sb.toString();
}
static Keys.Key[] split(String key) {
List<Key> splitKey = new ArrayList<Key>();

View file

@ -61,20 +61,12 @@ class Results {
}
void invalidTextAfterIdentifier(Identifier identifier, char text, int line) {
if (identifier.isKey() && text == '\n') {
sb.append("Key ")
.append(identifier.getName())
.append(" is not followed by an equals sign on line ")
.append(line)
.append('\n');
} else {
sb.append("Invalid text after key ")
.append(identifier.getName())
.append(" on line ")
.append(line)
.append(". Make sure to terminate the value or add a comment (#).")
.append('\n');
}
sb.append("Invalid text after key ")
.append(identifier.getName())
.append(" on line ")
.append(line)
.append(". Make sure to terminate the value or add a comment (#).")
.append('\n');
}
void invalidKey(String key, int line) {
@ -106,6 +98,14 @@ class Results {
.append('\n');
}
void unterminatedKey(String key, int line) {
sb.append("Key is not followed by an equals sign on line ")
.append(line)
.append(": ")
.append(key)
.append('\n');
}
void unterminated(String key, String value, int line) {
sb.append("Unterminated value on line ")
.append(line)
@ -200,7 +200,8 @@ class Results {
}
}
void startTables(String tableName) {
void startTables(Identifier id) {
String tableName = id.getName().substring(1, id.getName().length() - 1);
if (!tables.add(tableName)) {
errors.duplicateTable(tableName, -1);
}

View file

@ -26,22 +26,16 @@ class TomlParser {
if (c == '#' && !inComment) {
inComment = true;
} else if (!Character.isWhitespace(c) && !inComment && identifier == null) {
Identifier id = IDENTIFIER_CONVERTER.convert(chars, index);
Identifier id = IDENTIFIER_CONVERTER.convert(chars, index, new Context(null, line, results.errors));
if (id.isValid()) {
char next = chars[index.get()];
if (index.get() < chars.length -1 && !id.acceptsNext(next)) {
results.errors.invalidTextAfterIdentifier(id, next, line.get());
} else if (id.isKey()) {
if (id != Identifier.INVALID) {
if (id.isKey()) {
identifier = id;
} else if (id.isTable()) {
results.startTables(Keys.getTableName(id.getName()));
results.startTables(id);
} else if (id.isTableArray()) {
results.startTableArray(Keys.getTableArrayName(id.getName()));
}
inComment = next == '#';
} else {
results.errors.invalidIdentifier(id, line.get());
}
} else if (c == '\n') {
inComment = false;
@ -49,7 +43,7 @@ class TomlParser {
value = null;
line.incrementAndGet();
} else if (!inComment && identifier != null && identifier.isKey() && value == null && !Character.isWhitespace(c)) {
Object converted = ValueConverters.CONVERTERS.convert(tomlString, index, new Context(identifier, line));
Object converted = ValueConverters.CONVERTERS.convert(tomlString, index, new Context(identifier, line, results.errors));
value = converted;
if (converted instanceof Results.Errors) {

View file

@ -13,10 +13,43 @@ public class BareKeysTest {
assertEquals("a", toml.getString("a.b.c.key"));
}
@Test
public void should_support_underscores_in_key_names() throws Exception {
Toml toml = new Toml().parse("a_a = 1");
assertEquals(1, toml.getLong("a_a").intValue());
}
@Test
public void should_support_underscores_in_table_names() throws Exception {
Toml toml = new Toml().parse("[group_a]\na = 1");
assertEquals(1, toml.getLong("group_a.a").intValue());
}
@Test
public void should_support_numbers_in_key_names() throws Exception {
Toml toml = new Toml().parse("a1 = 1");
assertEquals(1, toml.getLong("a1").intValue());
}
@Test
public void should_support_numbers_in_table_names() throws Exception {
Toml toml = new Toml().parse("[group1]\na = 1");
assertEquals(1, toml.getLong("group1.a").intValue());
}
@Test(expected = IllegalStateException.class)
public void should_fail_when_characters_outside_accept_range_are_used_in_table_name() throws Exception {
new Toml().parse("[~]");
}
@Test(expected = IllegalStateException.class)
public void should_fail_when_dots_in_key_name() throws Exception {
new Toml().parse("a.b = 1");
}
@Test(expected = IllegalStateException.class)
public void should_fail_when_characters_outside_accept_range_are_used_in_key_name() throws Exception {

View file

@ -129,11 +129,16 @@ public class BurntSushiValidTest {
run("key-special-chars-modified");
}
@Test
@Test @Ignore
public void keys_with_dots() throws Exception {
run("keys-with-dots");
}
@Test
public void keys_with_dots_modified() throws Exception {
run("keys-with-dots-modified");
}
@Test
public void long_float() throws Exception {
run("long-float");

View file

@ -39,7 +39,7 @@ public class ErrorMessagesTest {
@Test
public void should_message_invalid_key() throws Exception {
e.expectMessage("Invalid key on line 1: k\"");
e.expectMessage("Key is not followed by an equals sign on line 1: k\" = 1");
new Toml().parse("k\" = 1");
}
@ -81,7 +81,7 @@ public class ErrorMessagesTest {
@Test
public void should_message_key_without_equals() throws Exception {
e.expectMessage("Key k is not followed by an equals sign on line 2");
e.expectMessage("Key is not followed by an equals sign on line 2: k");
new Toml().parse("\nk\n=3");
}

View file

@ -96,49 +96,6 @@ public class TomlTest {
assertEquals("value", toml.getString("key"));
}
@Test
public void should_support_numbers_in_key_names() throws Exception {
Toml toml = new Toml().parse("a1 = 1");
assertEquals(1, toml.getLong("a1").intValue());
}
@Test
public void should_support_numbers_in_table_names() throws Exception {
Toml toml = new Toml().parse("[group1]\na = 1");
assertEquals(1, toml.getLong("group1.a").intValue());
}
@Test
public void should_support_underscores_in_key_names() throws Exception {
Toml toml = new Toml().parse("a_a = 1");
assertEquals(1, toml.getLong("a_a").intValue());
}
@Test
public void should_support_dots_in_key_names() throws Exception {
Toml toml = new Toml().parse(file("should_support_dots_in_key_names"));
assertEquals(1, toml.getLong("a").intValue());
assertEquals(2, toml.getLong("b.c").intValue());
assertEquals(3, toml.getTable("b").getLong("c").intValue());
assertEquals(4, toml.getLong("b.a.b").intValue());
assertEquals(5, toml.getLong("d.e.a").intValue());
assertEquals(6, toml.getLong("d.e.a.b.c").intValue());
assertEquals(6, toml.getTable("d.e").getLong("a.b.c").intValue());
assertEquals(7, toml.getTables("f").get(0).getLong("a.b").intValue());
assertEquals(8, toml.getLong("f[1].a.b").intValue());
}
@Test
public void should_support_underscores_in_table_names() throws Exception {
Toml toml = new Toml().parse("[group_a]\na = 1");
assertEquals(1, toml.getLong("group_a.a").intValue());
}
@Test
public void should_support_blank_lines() throws Exception {
Toml toml = new Toml().parse(new File(getClass().getResource("should_support_blank_line.toml").getFile()));

View file

@ -0,0 +1,14 @@
{
"plain": {"type": "integer", "value": "1"},
"\"with.dot\"": {"type": "integer", "value": "2"},
"plain_table": {
"plain": {"type": "integer", "value": "3"},
"\"with.dot\"": {"type": "integer", "value": "4"}
},
"table": {
"withdot": {
"plain": {"type": "integer", "value": "5"},
"\"key.with.dots\"": {"type": "integer", "value": "6"}
}
}
}

View file

@ -0,0 +1,10 @@
plain = 1
"with.dot" = 2
[plain_table]
plain = 3
"with.dot" = 4
[table.withdot]
plain = 5
"key.with.dots" = 6

View file

@ -1,16 +0,0 @@
a = 1
b.c = 2
[b]
c = 3
a.b = 4
[d.e]
a = 5
a.b.c = 6
[[f]]
a.b = 7
[[f]]
a.b = 8