From 91e5c6557758a4abcdde1f3421236c49c2dcc884 Mon Sep 17 00:00:00 2001 From: "moandji.ezana" Date: Mon, 9 Feb 2015 16:29:20 +0200 Subject: [PATCH 01/10] Improved error handling code and line numbers included in error messages --- CHANGELOG.md | 4 + .../java/com/moandjiezana/toml/Results.java | 106 ++++++++++++++++-- src/main/java/com/moandjiezana/toml/Toml.java | 2 +- .../com/moandjiezana/toml/TomlParser.java | 15 +-- .../com/moandjiezana/toml/BareKeysTest.java | 22 +--- .../moandjiezana/toml/ErrorMessagesTest.java | 74 ++++++++++++ 6 files changed, 190 insertions(+), 33 deletions(-) create mode 100644 src/test/java/com/moandjiezana/toml/ErrorMessagesTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 6682454..d2ea678 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ * Dropped dependency on Parboiled and its significant transitive dependencies * Updated Gson to 2.3.1 +### Added + +* Line numbers included in error messages + ### Fixed * Fixed short-form Unicode escapes diff --git a/src/main/java/com/moandjiezana/toml/Results.java b/src/main/java/com/moandjiezana/toml/Results.java index 8eb5361..5d57565 100644 --- a/src/main/java/com/moandjiezana/toml/Results.java +++ b/src/main/java/com/moandjiezana/toml/Results.java @@ -7,8 +7,100 @@ import java.util.Map; import java.util.Set; class Results { + + static class Errors { + + private final StringBuilder sb = new StringBuilder(); + + void duplicateTable(String table, int line) { + sb.append("Duplicate table definition: [") + .append(table) + .append("]\n"); + } + + void emptyImplicitTable(String table, int line) { + sb.append("Invalid table definition due to empty implicit table name: "); + if (!table.startsWith("[")) { + sb.append('['); + } + sb.append(table); + if (!table.endsWith("]")) { + sb.append(']'); + } + sb.append("\n"); + } + + void invalidTable(String table, int line) { + sb.append("Invalid table definition on line ") + .append(line) + .append(": "); + if (!table.startsWith("[")) { + sb.append('['); + } + sb.append(table); + if (!table.endsWith("]")) { + sb.append(']'); + } + sb.append("]\n"); + } + + void duplicateKey(String key, int line) { + sb.append("Duplicate key: ") + .append(key) + .append('\n'); + } + + void invalidKey(String key, int line) { + sb.append("Invalid key"); + if (line > -1) { + sb.append(" on line ") + .append(line); + } + sb.append(": ") + .append(key) + .append('\n'); + } + + void invalidTableArray(String tableArray, int line) { + sb.append("Invalid table array definition on line ") + .append(line) + .append(": ") + .append(tableArray) + .append('\n'); + } + + void invalidValue(String key, String value, int line) { + sb.append("Invalid value on line ") + .append(line) + .append(": ") + .append(key) + .append(" = ") + .append(value) + .append('\n'); + } + + void unterminated(String key, String multiline, int line) { + sb.append("Unterminated multiline value on line ") + .append(line) + .append(": ") + .append(key) + .append(" = ") + .append(multiline.trim()) + .append('\n'); + } + + boolean hasErrors() { + return sb.length() > 0; + } + + @Override + public String toString() { + return sb.toString(); + } + } + Set tables = new HashSet(); - StringBuilder errors = new StringBuilder(); + final Errors errors = new Errors(); private Deque stack = new ArrayDeque(); Results() { @@ -20,7 +112,7 @@ class Results { if (currentTable.accepts(key)) { currentTable.put(key, value); } else { - errors.append("Key " + key + " is defined twice!\n"); + errors.duplicateKey(key, -1); } } @@ -56,7 +148,7 @@ class Results { stack.push(((Container.TableArray) newContainer).getCurrent()); } } else { - errors.append("Duplicate key and table definitions for " + tableName + "!\n"); + errors.duplicateTable(tableName, -1); break; } } @@ -64,11 +156,11 @@ class Results { void startTables(String tableName) { if (!tables.add(tableName)) { - errors.append("Table " + tableName + " defined twice!\n"); + errors.duplicateTable(tableName, -1); } if (tableName.endsWith(".")) { - errors.append("Implicit table name cannot be empty: " + tableName); + errors.emptyImplicitTable(tableName, -1); } while (stack.size() > 1) { @@ -80,7 +172,7 @@ class Results { String tablePart = tableParts[i].name; Container currentContainer = stack.peek(); if (tablePart.isEmpty()) { - errors.append("Empty implicit table: " + tableName + "!\n"); + errors.emptyImplicitTable(tableName, -1); } else if (currentContainer.get(tablePart) instanceof Container) { Container nextTable = (Container) currentContainer.get(tablePart); stack.push(nextTable); @@ -90,7 +182,7 @@ class Results { } else if (currentContainer.accepts(tablePart)) { startTable(tablePart); } else { - errors.append("Duplicate key and table definitions for " + tableName + "!\n"); + errors.duplicateTable(tableName, -1); break; } } diff --git a/src/main/java/com/moandjiezana/toml/Toml.java b/src/main/java/com/moandjiezana/toml/Toml.java index e7ec06f..2da371f 100644 --- a/src/main/java/com/moandjiezana/toml/Toml.java +++ b/src/main/java/com/moandjiezana/toml/Toml.java @@ -124,7 +124,7 @@ public class Toml { */ public Toml parse(String tomlString) throws IllegalStateException { Results results = new TomlParser().run(tomlString); - if (results.errors.length() > 0) { + if (results.errors.hasErrors()) { throw new IllegalStateException(results.errors.toString()); } diff --git a/src/main/java/com/moandjiezana/toml/TomlParser.java b/src/main/java/com/moandjiezana/toml/TomlParser.java index fd39cb1..2fa3ae9 100644 --- a/src/main/java/com/moandjiezana/toml/TomlParser.java +++ b/src/main/java/com/moandjiezana/toml/TomlParser.java @@ -19,6 +19,7 @@ class TomlParser { } String[] lines = tomlString.split("[\\n\\r]"); + int lastKeyLine = 1; StringBuilder multilineBuilder = new StringBuilder(); Multiline multiline = Multiline.NONE; @@ -41,7 +42,7 @@ class TomlParser { if (tableName != null) { results.startTableArray(tableName); } else { - results.errors.append("Invalid table array definition: " + line + "\n\n"); + results.errors.invalidTableArray(line, i + 1); } continue; @@ -52,14 +53,14 @@ class TomlParser { if (tableName != null) { results.startTables(tableName); } else { - results.errors.append("Invalid table definition: " + line + "\n\n"); + results.errors.invalidTable(line.trim(), i + 1); } continue; } if (multiline.isNotMultiline() && !line.contains("=")) { - results.errors.append("Invalid key definition: " + line); + results.errors.invalidKey(line, i + 1); continue; } @@ -139,24 +140,24 @@ class TomlParser { } else { key = Keys.getKey(pair[0]); if (key == null) { - results.errors.append("Invalid key name: " + pair[0] + "\n"); + results.errors.invalidKey(pair[0], i + 1); continue; } value = pair[1].trim(); } - + lastKeyLine = i + 1; Object convertedValue = VALUE_ANALYSIS.convert(value); if (convertedValue != INVALID) { results.addValue(key, convertedValue); } else { - results.errors.append("Invalid key/value: " + key + " = " + value + "\n"); + results.errors.invalidValue(key, value, i + 1); } } if (multiline != Multiline.NONE) { - results.errors.append("Unterminated multiline " + multiline.toString().toLowerCase().replace('_', ' ') + "\n"); + results.errors.unterminated(key, multilineBuilder.toString().trim(), lastKeyLine); } return results; diff --git a/src/test/java/com/moandjiezana/toml/BareKeysTest.java b/src/test/java/com/moandjiezana/toml/BareKeysTest.java index b2b066f..60df975 100644 --- a/src/test/java/com/moandjiezana/toml/BareKeysTest.java +++ b/src/test/java/com/moandjiezana/toml/BareKeysTest.java @@ -2,15 +2,10 @@ package com.moandjiezana.toml; import static org.junit.Assert.assertEquals; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; public class BareKeysTest { - @Rule - public final ExpectedException exception = ExpectedException.none(); - @Test public void should_ignore_spaces_around_key_segments() throws Exception { Toml toml = new Toml().parse("[ a . b . c ] \n key = \"a\""); @@ -18,32 +13,23 @@ public class BareKeysTest { assertEquals("a", toml.getString("a.b.c.key")); } - @Test + @Test(expected = IllegalStateException.class) public void should_fail_when_characters_outside_accept_range_are_used_in_table_name() throws Exception { - exception.expect(IllegalStateException.class); - exception.expectMessage("Invalid table definition: [~]"); - new Toml().parse("[~]"); } - @Test + @Test(expected = IllegalStateException.class) public void should_fail_when_characters_outside_accept_range_are_used_in_key_name() throws Exception { - exception.expect(IllegalStateException.class); - new Toml().parse("~ = 1"); } - @Test + @Test(expected = IllegalStateException.class) public void should_fail_on_sharp_sign_in_table_name() throws Exception { - exception.expect(IllegalStateException.class); - new Toml().parse("[group#]\nkey=1"); } - @Test + @Test(expected = IllegalStateException.class) public void should_fail_on_spaces_in_table_name() throws Exception { - exception.expect(IllegalStateException.class); - new Toml().parse("[valid key]"); } diff --git a/src/test/java/com/moandjiezana/toml/ErrorMessagesTest.java b/src/test/java/com/moandjiezana/toml/ErrorMessagesTest.java new file mode 100644 index 0000000..0c6ac37 --- /dev/null +++ b/src/test/java/com/moandjiezana/toml/ErrorMessagesTest.java @@ -0,0 +1,74 @@ +package com.moandjiezana.toml; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class ErrorMessagesTest { + + @Rule + public final ExpectedException e = ExpectedException.none(); + + @Test + public void should_message_invalid_table() throws Exception { + e.expectMessage("Invalid table definition on line 1: [in valid]"); + + new Toml().parse("[in valid]"); + } + + @Test + public void should_message_duplicate_table() throws Exception { + e.expectMessage("Duplicate table definition: [again]"); + + new Toml().parse("[again]\n[again]"); + } + + @Test + public void should_message_empty_implicit_table_name() throws Exception { + e.expectMessage("Invalid table definition due to empty implicit table name: [a..b]"); + + new Toml().parse("[a..b]"); + } + + @Test + public void should_message_duplicate_key() throws Exception { + e.expectMessage("Duplicate key: k"); + + new Toml().parse("k = 1\n k = 2"); + } + + @Test + public void should_message_invalid_key() throws Exception { + e.expectMessage("Invalid key on line 1: k\""); + + new Toml().parse("k\" = 1"); + } + + @Test + public void should_message_invalid_table_array() throws Exception { + e.expectMessage("Invalid table array definition on line 1: [[in valid]]"); + + new Toml().parse("[[in valid]]"); + } + + @Test + public void should_message_invalid_value() throws Exception { + e.expectMessage("Invalid value on line 1: k = 1 t"); + + new Toml().parse("k = 1 t"); + } + + @Test + public void should_message_unterminated_value() throws Exception { + e.expectMessage("Unterminated multiline value on line 1: k = '''abc"); + + new Toml().parse("k = '''abc"); + } + + @Test + public void should_message_key_without_equals() throws Exception { + e.expectMessage("Invalid key on line 2: k"); + + new Toml().parse("\nk\n=3"); + } +} From de1bcbdca2f83b57c22a043bbfb44dc3f1a53283 Mon Sep 17 00:00:00 2001 From: "moandji.ezana" Date: Mon, 9 Feb 2015 22:14:58 +0200 Subject: [PATCH 02/10] Reduced method visibility --- src/main/java/com/moandjiezana/toml/ArrayConverter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/moandjiezana/toml/ArrayConverter.java b/src/main/java/com/moandjiezana/toml/ArrayConverter.java index 014600d..a9c6501 100644 --- a/src/main/java/com/moandjiezana/toml/ArrayConverter.java +++ b/src/main/java/com/moandjiezana/toml/ArrayConverter.java @@ -22,7 +22,7 @@ class ArrayConverter implements ValueConverter { return convert(s, new AtomicInteger(1), true); } - public Object convert(String s, AtomicInteger sharedIndex, boolean topLevel) { + private Object convert(String s, AtomicInteger sharedIndex, boolean topLevel) { char[] chars = s.toCharArray(); List arrayItems = new ArrayList(); boolean terminated = false; From a5a6ab22dc22b96841d3ee3eb954f0627eb66892 Mon Sep 17 00:00:00 2001 From: "moandji.ezana" Date: Mon, 9 Feb 2015 22:15:37 +0200 Subject: [PATCH 03/10] Support for very basic inline tables --- .../toml/InlineTableConverter.java | 93 +++++++++++++++++++ .../java/com/moandjiezana/toml/Results.java | 10 +- .../com/moandjiezana/toml/TomlParser.java | 1 + .../moandjiezana/toml/ValueConverters.java | 3 +- .../moandjiezana/toml/InlineTableTest.java | 90 ++++++++++++++++++ ...should_support_array_of_inline_tables.toml | 3 + 6 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/moandjiezana/toml/InlineTableConverter.java create mode 100644 src/test/java/com/moandjiezana/toml/InlineTableTest.java create mode 100644 src/test/resources/com/moandjiezana/toml/should_support_array_of_inline_tables.toml diff --git a/src/main/java/com/moandjiezana/toml/InlineTableConverter.java b/src/main/java/com/moandjiezana/toml/InlineTableConverter.java new file mode 100644 index 0000000..a3a181c --- /dev/null +++ b/src/main/java/com/moandjiezana/toml/InlineTableConverter.java @@ -0,0 +1,93 @@ +package com.moandjiezana.toml; + +import static com.moandjiezana.toml.ValueConverterUtils.INVALID; + +import java.util.HashMap; + +class InlineTableConverter implements ValueConverter { + + static final InlineTableConverter INLINE_TABLE_PARSER = new InlineTableConverter(); + private static final ValueConverters CONVERTERS = new ValueConverters(); + + @Override + public boolean canConvert(String s) { + return s.startsWith("{"); + } + + @Override + public Object convert(String s) { + char[] chars = s.toCharArray(); + boolean inKey = true; + boolean pairHasKey = false; + boolean inValue = false; + boolean quoted = false; + boolean inString = false; + boolean terminated = false; + StringBuilder currentKey = new StringBuilder(); + StringBuilder current = new StringBuilder(); + HashMap results = new HashMap(); + + for (int i = 1; i < chars.length; i++) { + char c = chars[i]; + + if (terminated) { + if (Character.isWhitespace(c)) { + continue; + } + if (c == '#') { + break; + } + + return INVALID; + } + + if (c == '"') { + quoted = !quoted; + (inValue ? current : currentKey).append(c); + } else if (quoted) { + (inKey ? currentKey : current).append(c); + } else if (c == ',') { + Object converted = CONVERTERS.convert(current.toString().trim()); + + if (converted == INVALID) { + return INVALID; + } + + results.put(currentKey.toString().trim(), converted); + inKey = true; + pairHasKey = false; + inValue = false; + currentKey = new StringBuilder(); + current = new StringBuilder(); + } else if (c == '=') { + inKey = false; + pairHasKey = true; + inValue = true; + } else if (c == '}') { + terminated = true; + + if (current.toString().trim().length() == 0) { + continue; + } + + Object converted = CONVERTERS.convert(current.toString().trim()); + + if (converted == INVALID) { + return INVALID; + } + + results.put(currentKey.toString().trim(), converted); + } else { + (inKey ? currentKey : current).append(c); + } + } + + if (!terminated) { + return INVALID; + } + + return results; + } + + private InlineTableConverter() {} +} diff --git a/src/main/java/com/moandjiezana/toml/Results.java b/src/main/java/com/moandjiezana/toml/Results.java index 5d57565..97274d7 100644 --- a/src/main/java/com/moandjiezana/toml/Results.java +++ b/src/main/java/com/moandjiezana/toml/Results.java @@ -109,7 +109,15 @@ class Results { void addValue(String key, Object value) { Container currentTable = stack.peek(); - if (currentTable.accepts(key)) { + + if (value instanceof Map) { + startTable(key); + @SuppressWarnings("unchecked") + Map valueMap = (Map) value; + for (Map.Entry entry : valueMap.entrySet()) { + addValue(entry.getKey(), entry.getValue()); + } + } else if (currentTable.accepts(key)) { currentTable.put(key, value); } else { errors.duplicateKey(key, -1); diff --git a/src/main/java/com/moandjiezana/toml/TomlParser.java b/src/main/java/com/moandjiezana/toml/TomlParser.java index 2fa3ae9..2244a85 100644 --- a/src/main/java/com/moandjiezana/toml/TomlParser.java +++ b/src/main/java/com/moandjiezana/toml/TomlParser.java @@ -37,6 +37,7 @@ class TomlParser { continue; } + // TODO check that this works in multiline context if (isTableArray(line)) { String tableName = Keys.getTableArrayName(line); if (tableName != null) { diff --git a/src/main/java/com/moandjiezana/toml/ValueConverters.java b/src/main/java/com/moandjiezana/toml/ValueConverters.java index d4b8a96..aeb56e8 100644 --- a/src/main/java/com/moandjiezana/toml/ValueConverters.java +++ b/src/main/java/com/moandjiezana/toml/ValueConverters.java @@ -3,6 +3,7 @@ package com.moandjiezana.toml; import static com.moandjiezana.toml.ArrayConverter.ARRAY_PARSER; import static com.moandjiezana.toml.BooleanConverter.BOOLEAN_PARSER; import static com.moandjiezana.toml.DateConverter.DATE_PARSER; +import static com.moandjiezana.toml.InlineTableConverter.INLINE_TABLE_PARSER; import static com.moandjiezana.toml.LiteralStringConverter.LITERAL_STRING_PARSER; import static com.moandjiezana.toml.MultilineLiteralStringConverter.MULTILINE_LITERAL_STRING_CONVERTER; import static com.moandjiezana.toml.MultilineStringConverter.MULTILINE_STRING_PARSER; @@ -13,7 +14,7 @@ import static com.moandjiezana.toml.ValueConverterUtils.INVALID; class ValueConverters { private static final ValueConverter[] PARSERS = { - MULTILINE_STRING_PARSER, MULTILINE_LITERAL_STRING_CONVERTER, LITERAL_STRING_PARSER, STRING_PARSER, DATE_PARSER, NUMBER_PARSER, BOOLEAN_PARSER, ARRAY_PARSER + MULTILINE_STRING_PARSER, MULTILINE_LITERAL_STRING_CONVERTER, LITERAL_STRING_PARSER, STRING_PARSER, DATE_PARSER, NUMBER_PARSER, BOOLEAN_PARSER, ARRAY_PARSER, INLINE_TABLE_PARSER }; public Object convert(String value) { diff --git a/src/test/java/com/moandjiezana/toml/InlineTableTest.java b/src/test/java/com/moandjiezana/toml/InlineTableTest.java new file mode 100644 index 0000000..421b00f --- /dev/null +++ b/src/test/java/com/moandjiezana/toml/InlineTableTest.java @@ -0,0 +1,90 @@ +package com.moandjiezana.toml; + +import static org.hamcrest.Matchers.hasSize; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +import java.util.Calendar; +import java.util.TimeZone; + +import org.junit.Test; + +public class InlineTableTest { + + private static final TimeZone UTC = TimeZone.getTimeZone("UTC"); + + @Test + public void should_read_empty_inline_table() throws Exception { + Toml toml = new Toml().parse("key = {}"); + + assertNotNull(toml.getTable("key")); + } + + @Test + public void should_read_inline_table_with_strings() throws Exception { + Toml toml = new Toml().parse("name = { first = \"Tom\", last = \"Preston-Werner\"}"); + + assertEquals("Tom", toml.getTable("name").getString("first")); + assertEquals("Preston-Werner", toml.getString("name.last")); + } + + @Test + public void should_read_inline_table_with_integers() throws Exception { + Toml toml = new Toml().parse("point = { x = 1, y = 2 }"); + + assertEquals(1, toml.getTable("point").getLong("x").longValue()); + assertEquals(2, toml.getLong("point.y").longValue()); + } + + @Test + public void should_read_inline_table_with_floats() throws Exception { + Toml toml = new Toml().parse("point = { x = 1.5, y = 2.3 }"); + + assertEquals(1.5, toml.getTable("point").getDouble("x").doubleValue(), 0); + assertEquals(2.3, toml.getDouble("point.y").doubleValue(), 0); + } + + @Test + public void should_read_inline_table_with_booleans() throws Exception { + Toml toml = new Toml().parse("point = { x = false, y = true }"); + + assertTrue(toml.getTable("point").getBoolean("y")); + assertFalse(toml.getBoolean("point.x")); + } + + @Test + public void should_read_inline_table_with_dates() throws Exception { + Toml toml = new Toml().parse("point = { x = 2015-02-09T22:05:00Z, y = 2015-02-09T21:05:00Z }"); + + + Calendar x = Calendar.getInstance(UTC); + x.set(2015, Calendar.FEBRUARY, 9, 22, 5, 00); + x.set(Calendar.MILLISECOND, 0); + + Calendar y = Calendar.getInstance(UTC); + y.set(2015, Calendar.FEBRUARY, 9, 21, 5, 00); + y.set(Calendar.MILLISECOND, 0); + + assertEquals(x.getTime(), toml.getTable("point").getDate("x")); + assertEquals(y.getTime(), toml.getDate("point.y")); + } + + @Test + public void should_support_array_of_inline_tables() throws Exception { + Toml toml = new Toml().parse(getClass().getResourceAsStream("should_support_array_of_inline_tables.toml")); + + assertThat(toml.getList("points"), hasSize(4)); + assertEquals(1, toml.getLong("points[0].x").longValue()); + assertEquals(2, toml.getLong("points[0].y").longValue()); + assertEquals(3, toml.getLong("points[0].z").longValue()); + assertEquals(7, toml.getLong("points[1].x").longValue()); + assertEquals(8, toml.getLong("points[1].y").longValue()); + assertEquals(9, toml.getLong("points[1].z").longValue()); + assertEquals(2, toml.getLong("points[2].x").longValue()); + assertEquals(4, toml.getLong("points[2].y").longValue()); + assertEquals(8, toml.getLong("points[2].z").longValue()); + } +} diff --git a/src/test/resources/com/moandjiezana/toml/should_support_array_of_inline_tables.toml b/src/test/resources/com/moandjiezana/toml/should_support_array_of_inline_tables.toml new file mode 100644 index 0000000..58d6f86 --- /dev/null +++ b/src/test/resources/com/moandjiezana/toml/should_support_array_of_inline_tables.toml @@ -0,0 +1,3 @@ +points = [ { x = 1, y = 2, z = 3 }, + { x = 7, y = 8, z = 9 }, + { x = 2, y = 4, z = 8 } ] From 5bd87a6b56d44c4484aa4e990913170b48fcade6 Mon Sep 17 00:00:00 2001 From: "moandji.ezana" Date: Tue, 10 Feb 2015 14:32:44 +0200 Subject: [PATCH 04/10] Added support for arrays in inline tables --- .../com/moandjiezana/toml/ArrayConverter.java | 46 ++++++++++--------- .../toml/InlineTableConverter.java | 27 ++++++++++- .../java/com/moandjiezana/toml/ArrayTest.java | 28 +++++++++++ .../moandjiezana/toml/InlineTableTest.java | 46 +++++++++++++------ 4 files changed, 111 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/moandjiezana/toml/ArrayConverter.java b/src/main/java/com/moandjiezana/toml/ArrayConverter.java index a9c6501..f3100ba 100644 --- a/src/main/java/com/moandjiezana/toml/ArrayConverter.java +++ b/src/main/java/com/moandjiezana/toml/ArrayConverter.java @@ -19,32 +19,36 @@ class ArrayConverter implements ValueConverter { @Override public Object convert(String s) { - return convert(s, new AtomicInteger(1), true); + AtomicInteger sharedIndex = new AtomicInteger(1); + Object converted = convert(s, sharedIndex); + + char[] chars = s.toCharArray(); + + for (int i = sharedIndex.incrementAndGet(); i < chars.length; i++) { + char c = chars[i]; + + if (c == '#') { + break; + } + + if (!Character.isWhitespace(c)) { + return INVALID; + } + } + + return converted; } - private Object convert(String s, AtomicInteger sharedIndex, boolean topLevel) { + Object convert(String s, AtomicInteger sharedIndex) { char[] chars = s.toCharArray(); List arrayItems = new ArrayList(); boolean terminated = false; StringType stringType = StringType.NONE; StringBuilder current = new StringBuilder(); - for (int i = 1; i < chars.length; i++, sharedIndex.incrementAndGet()) { - char c = chars[i]; - - if (terminated && !topLevel) { - break; - } - - if (terminated) { - if (c == '#') { - break; - } - if (!Character.isWhitespace(c)) { - return INVALID; - } - continue; - } + for (; sharedIndex.get() < chars.length; sharedIndex.incrementAndGet()) { + int i = sharedIndex.get(); + char c = chars[sharedIndex.get()]; if (stringType == StringType.NONE) { if (c == ',') { @@ -56,8 +60,8 @@ class ArrayConverter implements ValueConverter { } if (c == '[') { - arrayItems.add(convert(s.substring(i), sharedIndex, false)); - i = sharedIndex.get(); + sharedIndex.incrementAndGet(); + arrayItems.add(convert(s, sharedIndex)); continue; } @@ -67,7 +71,7 @@ class ArrayConverter implements ValueConverter { arrayItems.add(current.toString()); } current = new StringBuilder(); - continue; + break; } } diff --git a/src/main/java/com/moandjiezana/toml/InlineTableConverter.java b/src/main/java/com/moandjiezana/toml/InlineTableConverter.java index a3a181c..10ddb74 100644 --- a/src/main/java/com/moandjiezana/toml/InlineTableConverter.java +++ b/src/main/java/com/moandjiezana/toml/InlineTableConverter.java @@ -3,6 +3,7 @@ package com.moandjiezana.toml; import static com.moandjiezana.toml.ValueConverterUtils.INVALID; import java.util.HashMap; +import java.util.concurrent.atomic.AtomicInteger; class InlineTableConverter implements ValueConverter { @@ -21,6 +22,7 @@ class InlineTableConverter implements ValueConverter { boolean pairHasKey = false; boolean inValue = false; boolean quoted = false; + boolean inArray = false; boolean inString = false; boolean terminated = false; StringBuilder currentKey = new StringBuilder(); @@ -46,14 +48,35 @@ class InlineTableConverter implements ValueConverter { (inValue ? current : currentKey).append(c); } else if (quoted) { (inKey ? currentKey : current).append(c); - } else if (c == ',') { - Object converted = CONVERTERS.convert(current.toString().trim()); + } else if (c == '[' && inValue) { + AtomicInteger sharedIndex = new AtomicInteger(i); + sharedIndex.incrementAndGet(); + Object converted = ArrayConverter.ARRAY_PARSER.convert(s, sharedIndex); if (converted == INVALID) { return INVALID; } results.put(currentKey.toString().trim(), converted); + i = sharedIndex.get(); + inArray = true; + continue; + } else if (c == ']' && inArray) { + current.append(']'); + inArray = false; + } else if (c == ',') { + if (inArray) { + inArray = false; + } else { + Object converted = CONVERTERS.convert(current.toString().trim()); + + if (converted == INVALID) { + return INVALID; + } + + results.put(currentKey.toString().trim(), converted); + } + inKey = true; pairHasKey = false; inValue = false; diff --git a/src/test/java/com/moandjiezana/toml/ArrayTest.java b/src/test/java/com/moandjiezana/toml/ArrayTest.java index ff48d15..b27f0be 100644 --- a/src/test/java/com/moandjiezana/toml/ArrayTest.java +++ b/src/test/java/com/moandjiezana/toml/ArrayTest.java @@ -2,11 +2,13 @@ package com.moandjiezana.toml; import static java.util.Arrays.asList; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.hasSize; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import java.io.File; +import java.util.Arrays; import java.util.List; import org.junit.Test; @@ -36,6 +38,16 @@ public class ArrayTest { assertEquals(asList(asList("gamma", "delta"), asList(1L, 2L)), clients.getList("data")); } + + @Test + public void should_get_deeply_nested_arrays() throws Exception { + List> data = new Toml().parse("data = [[[1], [2]], [3, 4]]").getList("data"); + + assertThat(data, hasSize(2)); + assertEquals(Arrays.asList(1L), data.get(0).get(0)); + assertEquals(asList(2L), data.get(0).get(1)); + assertEquals(asList(3L, 4L), data.get(1)); + } @Test @SuppressWarnings("unchecked") @@ -73,6 +85,22 @@ public class ArrayTest { assertThat(toml.getList("key"), contains("a]", "b]", "c]", "d]")); } + @Test + public void should_support_array_of_inline_tables() throws Exception { + Toml toml = new Toml().parse(getClass().getResourceAsStream("should_support_array_of_inline_tables.toml")); + + assertThat(toml.getList("points"), hasSize(4)); + assertEquals(1, toml.getLong("points[0].x").longValue()); + assertEquals(2, toml.getLong("points[0].y").longValue()); + assertEquals(3, toml.getLong("points[0].z").longValue()); + assertEquals(7, toml.getLong("points[1].x").longValue()); + assertEquals(8, toml.getLong("points[1].y").longValue()); + assertEquals(9, toml.getLong("points[1].z").longValue()); + assertEquals(2, toml.getLong("points[2].x").longValue()); + assertEquals(4, toml.getLong("points[2].y").longValue()); + assertEquals(8, toml.getLong("points[2].z").longValue()); + } + private File file(String file) { return Utils.file(getClass(), file); } diff --git a/src/test/java/com/moandjiezana/toml/InlineTableTest.java b/src/test/java/com/moandjiezana/toml/InlineTableTest.java index 421b00f..81623e9 100644 --- a/src/test/java/com/moandjiezana/toml/InlineTableTest.java +++ b/src/test/java/com/moandjiezana/toml/InlineTableTest.java @@ -1,5 +1,6 @@ package com.moandjiezana.toml; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.hasSize; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -8,6 +9,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import java.util.Calendar; +import java.util.List; import java.util.TimeZone; import org.junit.Test; @@ -59,7 +61,6 @@ public class InlineTableTest { public void should_read_inline_table_with_dates() throws Exception { Toml toml = new Toml().parse("point = { x = 2015-02-09T22:05:00Z, y = 2015-02-09T21:05:00Z }"); - Calendar x = Calendar.getInstance(UTC); x.set(2015, Calendar.FEBRUARY, 9, 22, 5, 00); x.set(Calendar.MILLISECOND, 0); @@ -73,18 +74,37 @@ public class InlineTableTest { } @Test - public void should_support_array_of_inline_tables() throws Exception { - Toml toml = new Toml().parse(getClass().getResourceAsStream("should_support_array_of_inline_tables.toml")); + public void should_read_arrays() throws Exception { + Toml toml = new Toml().parse("arrays = { integers = [1, 2, 3], strings = [\"a\", \"b\", \"c\"] }"); - assertThat(toml.getList("points"), hasSize(4)); - assertEquals(1, toml.getLong("points[0].x").longValue()); - assertEquals(2, toml.getLong("points[0].y").longValue()); - assertEquals(3, toml.getLong("points[0].z").longValue()); - assertEquals(7, toml.getLong("points[1].x").longValue()); - assertEquals(8, toml.getLong("points[1].y").longValue()); - assertEquals(9, toml.getLong("points[1].z").longValue()); - assertEquals(2, toml.getLong("points[2].x").longValue()); - assertEquals(4, toml.getLong("points[2].y").longValue()); - assertEquals(8, toml.getLong("points[2].z").longValue()); + assertThat(toml.getList("arrays.integers"), contains(1L, 2L, 3L)); + assertThat(toml.getList("arrays.strings"), contains("a", "b", "c")); + } + + @Test + public void should_read_nested_arrays() throws Exception { + Toml toml = new Toml().parse("arrays = { nested = [[1, 2, 3], [4, 5, 6]] }").getTable("arrays"); + + List> nested = toml.>getList("nested"); + assertThat(nested, hasSize(2)); + assertThat(nested.get(0), contains(1L, 2L, 3L)); + assertThat(nested.get(1), contains(4L, 5L, 6L)); + } + + @Test + public void should_read_mixed_inline_table() throws Exception { + Toml toml = new Toml().parse("point = { date = 2015-02-09T22:05:00Z, bool = true, integer = 123, float = 123.456, string = \"abc\", list = [5, 6, 7, 8] }").getTable("point"); + + + Calendar date = Calendar.getInstance(UTC); + date.set(2015, Calendar.FEBRUARY, 9, 22, 5, 00); + date.set(Calendar.MILLISECOND, 0); + + assertEquals(date.getTime(), toml.getDate("date")); + assertTrue(toml.getBoolean("bool")); + assertEquals(123, toml.getLong("integer").intValue()); + assertEquals(123.456, toml.getDouble("float"), 0); + assertEquals("abc", toml.getString("string")); + assertThat(toml.getList("list"), contains(5L, 6L, 7L, 8L)); } } From ddb061b9f991c09ad86521f6e526f2fc5e0c241a Mon Sep 17 00:00:00 2001 From: "moandji.ezana" Date: Tue, 10 Feb 2015 20:06:46 +0200 Subject: [PATCH 05/10] Added support for nested inline tables --- .../toml/InlineTableConverter.java | 82 +++++++++++-------- .../java/com/moandjiezana/toml/Results.java | 1 + .../moandjiezana/toml/InlineTableTest.java | 18 ++++ 3 files changed, 67 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/moandjiezana/toml/InlineTableConverter.java b/src/main/java/com/moandjiezana/toml/InlineTableConverter.java index 10ddb74..df7b455 100644 --- a/src/main/java/com/moandjiezana/toml/InlineTableConverter.java +++ b/src/main/java/com/moandjiezana/toml/InlineTableConverter.java @@ -17,39 +17,45 @@ class InlineTableConverter implements ValueConverter { @Override public Object convert(String s) { + AtomicInteger sharedIndex = new AtomicInteger(1); + Object converted = convert(s, sharedIndex); + char[] chars = s.toCharArray(); + + for (; sharedIndex.get() < s.length(); sharedIndex.incrementAndGet()) { + char c = chars[sharedIndex.get()]; + if (Character.isWhitespace(c)) { + continue; + } + if (c == '#') { + break; + } + + return INVALID; + } + + return converted; + } + + Object convert(String s, AtomicInteger sharedIndex) { char[] chars = s.toCharArray(); boolean inKey = true; - boolean pairHasKey = false; boolean inValue = false; boolean quoted = false; - boolean inArray = false; - boolean inString = false; boolean terminated = false; StringBuilder currentKey = new StringBuilder(); StringBuilder current = new StringBuilder(); HashMap results = new HashMap(); - for (int i = 1; i < chars.length; i++) { + for (; sharedIndex.get() < chars.length; sharedIndex.incrementAndGet()) { + int i = sharedIndex.get(); char c = chars[i]; - if (terminated) { - if (Character.isWhitespace(c)) { - continue; - } - if (c == '#') { - break; - } - - return INVALID; - } - if (c == '"') { quoted = !quoted; (inValue ? current : currentKey).append(c); } else if (quoted) { (inKey ? currentKey : current).append(c); } else if (c == '[' && inValue) { - AtomicInteger sharedIndex = new AtomicInteger(i); sharedIndex.incrementAndGet(); Object converted = ArrayConverter.ARRAY_PARSER.convert(s, sharedIndex); @@ -59,15 +65,23 @@ class InlineTableConverter implements ValueConverter { results.put(currentKey.toString().trim(), converted); i = sharedIndex.get(); - inArray = true; continue; - } else if (c == ']' && inArray) { - current.append(']'); - inArray = false; + } else if (c == '{') { + sharedIndex.incrementAndGet(); + Object converted = convert(s, sharedIndex); + + if (converted == INVALID) { + return INVALID; + } + + results.put(currentKey.toString().trim(), converted); + + inKey = true; + inValue = false; + currentKey = new StringBuilder(); + current = new StringBuilder(); } else if (c == ',') { - if (inArray) { - inArray = false; - } else { + if (!current.toString().trim().isEmpty()) { Object converted = CONVERTERS.convert(current.toString().trim()); if (converted == INVALID) { @@ -78,28 +92,28 @@ class InlineTableConverter implements ValueConverter { } inKey = true; - pairHasKey = false; inValue = false; currentKey = new StringBuilder(); current = new StringBuilder(); } else if (c == '=') { inKey = false; - pairHasKey = true; inValue = true; } else if (c == '}') { terminated = true; - if (current.toString().trim().length() == 0) { - continue; + String trimmed = current.toString().trim(); + if (!trimmed.isEmpty()) { + Object converted = CONVERTERS.convert(trimmed); + + if (converted == INVALID) { + return INVALID; + } + + results.put(currentKey.toString().trim(), converted); } - Object converted = CONVERTERS.convert(current.toString().trim()); - - if (converted == INVALID) { - return INVALID; - } - - results.put(currentKey.toString().trim(), converted); + sharedIndex.incrementAndGet(); + break; } else { (inKey ? currentKey : current).append(c); } diff --git a/src/main/java/com/moandjiezana/toml/Results.java b/src/main/java/com/moandjiezana/toml/Results.java index 97274d7..8b02882 100644 --- a/src/main/java/com/moandjiezana/toml/Results.java +++ b/src/main/java/com/moandjiezana/toml/Results.java @@ -117,6 +117,7 @@ class Results { for (Map.Entry entry : valueMap.entrySet()) { addValue(entry.getKey(), entry.getValue()); } + stack.pop(); } else if (currentTable.accepts(key)) { currentTable.put(key, value); } else { diff --git a/src/test/java/com/moandjiezana/toml/InlineTableTest.java b/src/test/java/com/moandjiezana/toml/InlineTableTest.java index 81623e9..573ad3a 100644 --- a/src/test/java/com/moandjiezana/toml/InlineTableTest.java +++ b/src/test/java/com/moandjiezana/toml/InlineTableTest.java @@ -107,4 +107,22 @@ public class InlineTableTest { assertEquals("abc", toml.getString("string")); assertThat(toml.getList("list"), contains(5L, 6L, 7L, 8L)); } + + @Test + public void should_read_nested_inline_tables() throws Exception { + Toml tables = new Toml().parse("tables = { t1 = { t1_1 = 1, t1_2 = 2}, t2 = { t2_1 = \"a\"} }").getTable("tables"); + + assertEquals(1L, tables.getLong("t1.t1_1").longValue()); + assertEquals(2L, tables.getLong("t1.t1_2").longValue()); + assertEquals("a", tables.getString("t2.t2_1")); + } + + @Test + public void should_read_all_string_types() throws Exception { + Toml strings = new Toml().parse("strings = { literal = 'ab]\"c', multiline = \"\"\"de]\"f\"\"\", multiline_literal = '''gh]\"i''' }").getTable("strings"); + + assertEquals("ab]\"c", strings.getString("literal")); + assertEquals("de]\"f", strings.getString("multiline")); + assertEquals("gh]\"i", strings.getString("multiline_literal")); + } } From d1d7145a030e5f000ae07c9be111ac51218ad378 Mon Sep 17 00:00:00 2001 From: "moandji.ezana" Date: Wed, 11 Feb 2015 00:59:48 +0200 Subject: [PATCH 06/10] Streamlined array and inline table converters to re-use other converters [ci skip] --- .../com/moandjiezana/toml/ArrayConverter.java | 119 +++++------------- .../moandjiezana/toml/BooleanConverter.java | 20 ++- .../com/moandjiezana/toml/DateConverter.java | 41 +++++- .../toml/InlineTableConverter.java | 67 +++------- .../toml/LiteralStringConverter.java | 52 ++++---- .../toml/MultilineLiteralStringConverter.java | 30 +++-- .../toml/MultilineStringConverter.java | 56 ++++++--- .../moandjiezana/toml/NumberConverter.java | 32 +++-- .../moandjiezana/toml/StringConverter.java | 24 +++- .../com/moandjiezana/toml/ValueConverter.java | 12 ++ .../moandjiezana/toml/ValueConverters.java | 16 ++- 11 files changed, 263 insertions(+), 206 deletions(-) diff --git a/src/main/java/com/moandjiezana/toml/ArrayConverter.java b/src/main/java/com/moandjiezana/toml/ArrayConverter.java index f3100ba..90849bb 100644 --- a/src/main/java/com/moandjiezana/toml/ArrayConverter.java +++ b/src/main/java/com/moandjiezana/toml/ArrayConverter.java @@ -19,7 +19,7 @@ class ArrayConverter implements ValueConverter { @Override public Object convert(String s) { - AtomicInteger sharedIndex = new AtomicInteger(1); + AtomicInteger sharedIndex = new AtomicInteger(); Object converted = convert(s, sharedIndex); char[] chars = s.toCharArray(); @@ -39,119 +39,56 @@ class ArrayConverter implements ValueConverter { return converted; } - Object convert(String s, AtomicInteger sharedIndex) { + @Override + public Object convert(String s, AtomicInteger index) { char[] chars = s.toCharArray(); List arrayItems = new ArrayList(); boolean terminated = false; - StringType stringType = StringType.NONE; - StringBuilder current = new StringBuilder(); - for (; sharedIndex.get() < chars.length; sharedIndex.incrementAndGet()) { - int i = sharedIndex.get(); - char c = chars[sharedIndex.get()]; + for (int i = index.incrementAndGet(); i < chars.length; i = index.incrementAndGet()) { - if (stringType == StringType.NONE) { - if (c == ',') { - if (current.toString().trim().length() > 0) { - arrayItems.add(current.toString()); - } - current = new StringBuilder(); - continue; - } + char c = chars[i]; - if (c == '[') { - sharedIndex.incrementAndGet(); - arrayItems.add(convert(s, sharedIndex)); - continue; - } - - if (c == ']') { - terminated = true; - if (current.toString().trim().length() > 0) { - arrayItems.add(current.toString()); - } - current = new StringBuilder(); - break; - } + if (Character.isWhitespace(c)) { + continue; + } + if (c == ',') { + continue; } - if (c == '"' && chars[i - 1] != '\\' && !stringType.accepts(c)) { - if (chars.length > i + 2 && chars[i + 1] == c && chars[i + 2] == c) { - stringType = stringType.flip(StringType.MULTILINE); - } else { - stringType = stringType.flip(StringType.BASIC); - } - } - - if (c == '\'' && !stringType.accepts(c)) { - if (chars.length > i + 2 && chars[i + 1] == c && chars[i + 2] == c) { - stringType = stringType.flip(StringType.MULTILINE_LITERAL); - } else { - stringType = stringType.flip(StringType.LITERAL); - } + if (c == '[') { + arrayItems.add(convert(s, index)); + continue; } - current.append(c); + if (c == ']') { + terminated = true; + break; + } + + arrayItems.add(VALUE_CONVERTERS.convert(s, index)); } if (!terminated) { return INVALID; } - return convertList(arrayItems); - } - - private Object convertList(List tokens) { - ArrayList nestedList = new ArrayList(); - - for (Object token : tokens) { - if (token instanceof String) { - Object converted = VALUE_CONVERTERS.convert(((String) token).trim()); - if (converted == INVALID) { - return INVALID; - } - if (isHomogenousArray(converted, nestedList)) { - nestedList.add(converted); - } else { - return INVALID; - } - } else if (token instanceof List) { - @SuppressWarnings("unchecked") - List convertedList = (List) token; - if (isHomogenousArray(convertedList, nestedList)) { - nestedList.add(convertedList); - } else { - return INVALID; - } + for (Object arrayItem : arrayItems) { + if (arrayItem == INVALID) { + return INVALID; + } + + if (!isHomogenousArray(arrayItem, arrayItems)) { + return INVALID; } } - - return nestedList; + + return arrayItems; } private boolean isHomogenousArray(Object o, List values) { return values.isEmpty() || values.get(0).getClass().isAssignableFrom(o.getClass()) || o.getClass().isAssignableFrom(values.get(0).getClass()); } - private static enum StringType { - NONE, BASIC, LITERAL, MULTILINE, MULTILINE_LITERAL; - - StringType flip(StringType to) { - return this == NONE ? to : NONE; - } - - boolean accepts(char c) { - if (this == BASIC || this == MULTILINE) { - return c != '"'; - } - - if (this == LITERAL || this == MULTILINE_LITERAL) { - return c != '\''; - } - - return false; - } - } - private ArrayConverter() {} } diff --git a/src/main/java/com/moandjiezana/toml/BooleanConverter.java b/src/main/java/com/moandjiezana/toml/BooleanConverter.java index 0dfc3e7..ef1a781 100644 --- a/src/main/java/com/moandjiezana/toml/BooleanConverter.java +++ b/src/main/java/com/moandjiezana/toml/BooleanConverter.java @@ -1,6 +1,9 @@ package com.moandjiezana.toml; import static com.moandjiezana.toml.ValueConverterUtils.INVALID; +import static com.moandjiezana.toml.ValueConverterUtils.isComment; + +import java.util.concurrent.atomic.AtomicInteger; class BooleanConverter implements ValueConverter { @@ -14,13 +17,24 @@ class BooleanConverter implements ValueConverter { @Override public Object convert(String s) { + AtomicInteger index = new AtomicInteger(); + Object converted = convert(s, index); + + if (!isComment(s.substring(index.incrementAndGet()))) { + return INVALID; + } + + return converted; + } + + @Override + public Object convert(String s, AtomicInteger index) { + s = s.substring(index.get()); Boolean b = s.startsWith("true") ? Boolean.TRUE : Boolean.FALSE; int endIndex = b == Boolean.TRUE ? 4 : 5; - if (!ValueConverterUtils.isComment(s.substring(endIndex))) { - return INVALID; - } + index.addAndGet(endIndex - 1); return b; } diff --git a/src/main/java/com/moandjiezana/toml/DateConverter.java b/src/main/java/com/moandjiezana/toml/DateConverter.java index c45d947..6c49ba7 100644 --- a/src/main/java/com/moandjiezana/toml/DateConverter.java +++ b/src/main/java/com/moandjiezana/toml/DateConverter.java @@ -1,8 +1,10 @@ package com.moandjiezana.toml; import static com.moandjiezana.toml.ValueConverterUtils.INVALID; +import static com.moandjiezana.toml.ValueConverterUtils.isComment; import java.text.SimpleDateFormat; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -15,13 +17,18 @@ class DateConverter implements ValueConverter { public boolean canConvert(String s) { Matcher matcher = DATE_REGEX.matcher(s); - return matcher.matches() && ValueConverterUtils.isComment(matcher.group(4)); + return matcher.matches(); } @Override public Object convert(String s) { Matcher matcher = DATE_REGEX.matcher(s); matcher.matches(); + + if (!isComment(matcher.group(4))) { + return INVALID; + } + s = matcher.group(1); String zone = matcher.group(3); String fractionalSeconds = matcher.group(2); @@ -36,6 +43,7 @@ class DateConverter implements ValueConverter { } else if (zone.contains(":")) { s += zone.replace(":", ""); } + try { SimpleDateFormat dateFormat = new SimpleDateFormat(format); dateFormat.setLenient(false); @@ -44,6 +52,37 @@ class DateConverter implements ValueConverter { return INVALID; } } + + @Override + public Object convert(String original, AtomicInteger index) { + String s = original.substring(index.get()); + Matcher matcher = DATE_REGEX.matcher(s); + matcher.matches(); + String dateString = matcher.group(1); + String zone = matcher.group(3); + String fractionalSeconds = matcher.group(2); + String format = "yyyy-MM-dd'T'HH:mm:ss"; + if (fractionalSeconds != null && !fractionalSeconds.isEmpty()) { + format += ".SSS"; + dateString += fractionalSeconds; + } + format += "Z"; + if ("Z".equals(zone)) { + dateString += "+0000"; + } else if (zone.contains(":")) { + dateString += zone.replace(":", ""); + } + + index.addAndGet(matcher.end(3) - 1); + + try { + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + dateFormat.setLenient(false); + return dateFormat.parse(dateString); + } catch (Exception e) { + return INVALID; + } + } private DateConverter() {} } diff --git a/src/main/java/com/moandjiezana/toml/InlineTableConverter.java b/src/main/java/com/moandjiezana/toml/InlineTableConverter.java index df7b455..54c3a26 100644 --- a/src/main/java/com/moandjiezana/toml/InlineTableConverter.java +++ b/src/main/java/com/moandjiezana/toml/InlineTableConverter.java @@ -17,55 +17,45 @@ class InlineTableConverter implements ValueConverter { @Override public Object convert(String s) { - AtomicInteger sharedIndex = new AtomicInteger(1); - Object converted = convert(s, sharedIndex); - char[] chars = s.toCharArray(); + AtomicInteger index = new AtomicInteger(); + Object converted = convert(s, index); - for (; sharedIndex.get() < s.length(); sharedIndex.incrementAndGet()) { - char c = chars[sharedIndex.get()]; - if (Character.isWhitespace(c)) { - continue; - } - if (c == '#') { - break; - } - + String substring = s.substring(index.incrementAndGet()); + if (converted == INVALID || !ValueConverterUtils.isComment(substring)) { return INVALID; } return converted; } - Object convert(String s, AtomicInteger sharedIndex) { + @Override + public Object convert(String s, AtomicInteger sharedIndex) { char[] chars = s.toCharArray(); boolean inKey = true; boolean inValue = false; boolean quoted = false; boolean terminated = false; StringBuilder currentKey = new StringBuilder(); - StringBuilder current = new StringBuilder(); HashMap results = new HashMap(); - for (; sharedIndex.get() < chars.length; sharedIndex.incrementAndGet()) { - int i = sharedIndex.get(); + for (int i = sharedIndex.incrementAndGet(); sharedIndex.get() < chars.length; i = sharedIndex.incrementAndGet()) { char c = chars[i]; - if (c == '"') { + if (c == '"' && inKey) { quoted = !quoted; - (inValue ? current : currentKey).append(c); + currentKey.append(c); } else if (quoted) { - (inKey ? currentKey : current).append(c); - } else if (c == '[' && inValue) { - sharedIndex.incrementAndGet(); - Object converted = ArrayConverter.ARRAY_PARSER.convert(s, sharedIndex); + currentKey.append(c); + } else if (inValue && !Character.isWhitespace(c)) { + Object converted = CONVERTERS.convert(s, sharedIndex); if (converted == INVALID) { return INVALID; } results.put(currentKey.toString().trim(), converted); - i = sharedIndex.get(); - continue; + currentKey = new StringBuilder(); + inValue = false; } else if (c == '{') { sharedIndex.incrementAndGet(); Object converted = convert(s, sharedIndex); @@ -79,43 +69,18 @@ class InlineTableConverter implements ValueConverter { inKey = true; inValue = false; currentKey = new StringBuilder(); - current = new StringBuilder(); } else if (c == ',') { - if (!current.toString().trim().isEmpty()) { - Object converted = CONVERTERS.convert(current.toString().trim()); - - if (converted == INVALID) { - return INVALID; - } - - results.put(currentKey.toString().trim(), converted); - } - inKey = true; inValue = false; currentKey = new StringBuilder(); - current = new StringBuilder(); } else if (c == '=') { inKey = false; inValue = true; } else if (c == '}') { terminated = true; - - String trimmed = current.toString().trim(); - if (!trimmed.isEmpty()) { - Object converted = CONVERTERS.convert(trimmed); - - if (converted == INVALID) { - return INVALID; - } - - results.put(currentKey.toString().trim(), converted); - } - - sharedIndex.incrementAndGet(); break; - } else { - (inKey ? currentKey : current).append(c); + } else if (inKey) { + currentKey.append(c); } } diff --git a/src/main/java/com/moandjiezana/toml/LiteralStringConverter.java b/src/main/java/com/moandjiezana/toml/LiteralStringConverter.java index 63cba93..c758814 100644 --- a/src/main/java/com/moandjiezana/toml/LiteralStringConverter.java +++ b/src/main/java/com/moandjiezana/toml/LiteralStringConverter.java @@ -1,6 +1,9 @@ package com.moandjiezana.toml; import static com.moandjiezana.toml.ValueConverterUtils.INVALID; +import static com.moandjiezana.toml.ValueConverterUtils.isComment; + +import java.util.concurrent.atomic.AtomicInteger; class LiteralStringConverter implements ValueConverter { @@ -14,31 +17,38 @@ class LiteralStringConverter implements ValueConverter { @Override public Object convert(String s) { - char[] chars = s.toCharArray(); - int endIndex = -1; + AtomicInteger index = new AtomicInteger(); + Object converted = convert(s, index); - for (int i = 1; i < chars.length; i++) { - char c = chars[i]; - - if (c == '\'') { - endIndex = i; - continue; - } - - if (endIndex > -1 && c == '#') { - break; - } - - if (endIndex > -1 && !Character.isWhitespace(c)) { - return INVALID; - } - } - - if (endIndex == -1) { + if (converted == INVALID || !isComment(s.substring(index.incrementAndGet()))) { return INVALID; } - return s.substring(1, endIndex); + return converted; + } + + @Override + public Object convert(String s, AtomicInteger index) { + char[] chars = s.toCharArray(); + boolean terminated = false; + int startIndex = index.incrementAndGet(); + + for (int i = index.get(); i < chars.length; i = index.incrementAndGet()) { + char c = chars[i]; + + if (c == '\'') { + terminated = true; + break; + } + } + + if (!terminated) { + return INVALID; + } + + String substring = s.substring(startIndex, index.get()); + + return substring; } private LiteralStringConverter() {} diff --git a/src/main/java/com/moandjiezana/toml/MultilineLiteralStringConverter.java b/src/main/java/com/moandjiezana/toml/MultilineLiteralStringConverter.java index d588817..37323d2 100644 --- a/src/main/java/com/moandjiezana/toml/MultilineLiteralStringConverter.java +++ b/src/main/java/com/moandjiezana/toml/MultilineLiteralStringConverter.java @@ -1,6 +1,9 @@ package com.moandjiezana.toml; import static com.moandjiezana.toml.ValueConverterUtils.INVALID; +import static com.moandjiezana.toml.ValueConverterUtils.isComment; + +import java.util.concurrent.atomic.AtomicInteger; class MultilineLiteralStringConverter implements ValueConverter { @@ -13,28 +16,33 @@ class MultilineLiteralStringConverter implements ValueConverter { @Override public Object convert(String s) { + AtomicInteger index = new AtomicInteger(); + Object converted = convert(s, index); + + if (converted == INVALID || !isComment(s.substring(index.incrementAndGet()))) { + return INVALID; + } + + return converted; + } + + @Override + public Object convert(String s, AtomicInteger index) { char[] chars = s.toCharArray(); + int startIndex = index.addAndGet(3); int endIndex = -1; - for (int i = 3; i < chars.length; i++) { + for (int i = startIndex; i < chars.length; i = index.incrementAndGet()) { char c = chars[i]; if (c == '\'' && chars.length > i + 2 && chars[i + 1] == '\'' && chars[i + 2] == '\'') { endIndex = i; - i += 2; - continue; - } - - if (endIndex > -1 && c == '#') { + index.addAndGet(2); break; } - - if (endIndex > -1 && !Character.isWhitespace(c)) { - return INVALID; - } } - return s.substring(3, endIndex); + return s.substring(startIndex, endIndex); } private MultilineLiteralStringConverter() {} diff --git a/src/main/java/com/moandjiezana/toml/MultilineStringConverter.java b/src/main/java/com/moandjiezana/toml/MultilineStringConverter.java index f5f7840..55417ac 100644 --- a/src/main/java/com/moandjiezana/toml/MultilineStringConverter.java +++ b/src/main/java/com/moandjiezana/toml/MultilineStringConverter.java @@ -1,9 +1,12 @@ package com.moandjiezana.toml; import static com.moandjiezana.toml.ValueConverterUtils.INVALID; +import static com.moandjiezana.toml.ValueConverterUtils.isComment; + +import java.util.concurrent.atomic.AtomicInteger; class MultilineStringConverter implements ValueConverter { - + static final MultilineStringConverter MULTILINE_STRING_PARSER = new MultilineStringConverter(); @Override @@ -13,22 +16,45 @@ class MultilineStringConverter implements ValueConverter { @Override public Object convert(String s) { - int terminator = s.indexOf("\"\"\"", 3); - - if (terminator == -1) { + AtomicInteger index = new AtomicInteger(); + Object converted = convert(s, index); + + if (converted == INVALID || !isComment(s.substring(index.incrementAndGet()))) { return INVALID; } - - if (!ValueConverterUtils.isComment(s.substring(terminator + 3))) { - return INVALID; - } - - s = s.substring(2, terminator + 1); - s = s.replaceAll("\\\\\\s+", ""); - - return StringConverter.STRING_PARSER.convert(s); + + return converted; + } + + @Override + public Object convert(String s, AtomicInteger index) { + char[] chars = s.toCharArray(); + int startIndex = index.addAndGet(3); + int endIndex = -1; + + for (int i = startIndex; i < chars.length; i = index.incrementAndGet()) { + char c = chars[i]; + + if (c == '"' && chars.length > i + 2 && chars[i + 1] == '"' && chars[i + 2] == '"') { + endIndex = i; + index.addAndGet(2); + break; + } + } + + if (endIndex == -1) { + return INVALID; + } + + s = s.substring(startIndex, endIndex); + s = s.replaceAll("\\\\\\s+", ""); + s = StringConverter.STRING_PARSER.replaceUnicodeCharacters(s); + s = StringConverter.STRING_PARSER.replaceSpecialCharacters(s); + + return s; + } + + private MultilineStringConverter() { } - - private MultilineStringConverter() {} } diff --git a/src/main/java/com/moandjiezana/toml/NumberConverter.java b/src/main/java/com/moandjiezana/toml/NumberConverter.java index b4c866a..56221b5 100644 --- a/src/main/java/com/moandjiezana/toml/NumberConverter.java +++ b/src/main/java/com/moandjiezana/toml/NumberConverter.java @@ -1,6 +1,9 @@ package com.moandjiezana.toml; import static com.moandjiezana.toml.ValueConverterUtils.INVALID; +import static com.moandjiezana.toml.ValueConverterUtils.isComment; + +import java.util.concurrent.atomic.AtomicInteger; class NumberConverter implements ValueConverter { static final NumberConverter NUMBER_PARSER = new NumberConverter(); @@ -14,20 +17,33 @@ class NumberConverter implements ValueConverter { @Override public Object convert(String s) { + AtomicInteger index = new AtomicInteger(); + Object converted = convert(s, index); + + if (converted == INVALID || (s.length() > index.get() + 1 && !isComment(s.substring(index.incrementAndGet())))) { + return INVALID; + } + + return converted; + } + + @Override + public Object convert(String s, AtomicInteger index) { char[] chars = s.toCharArray(); - boolean whitespace = false; boolean signable = true; boolean dottable = false; boolean exponentable = false; + boolean terminatable = false; String type = ""; StringBuilder sb = new StringBuilder(); - for (int i = 0; i < chars.length; i++) { + for (int i = index.get(); i < chars.length; i = index.incrementAndGet()) { char c = chars[i]; if (Character.isDigit(c)) { sb.append(c); signable = false; + terminatable = true; if (type.isEmpty()) { type = "integer"; dottable = true; @@ -35,26 +51,28 @@ class NumberConverter implements ValueConverter { exponentable = !type.equals("exponent"); } else if ((c == '+' || c == '-') && signable && chars.length > i + 1) { signable = false; + terminatable = false; if (c == '-') { sb.append('-'); } } else if (c == '.' && dottable && chars.length > i + 1) { sb.append('.'); type = "float"; + terminatable = false; dottable = false; exponentable = false; } else if ((c == 'E' || c == 'e') && exponentable && chars.length > i + 1) { sb.append('E'); type = "exponent"; + terminatable = false; signable = true; dottable = false; exponentable = false; - } else if (Character.isWhitespace(c)) { - whitespace = true; - } else if (whitespace && c == '#') { - break; } else { - type = ""; + if (!terminatable) { + type = ""; + } + index.decrementAndGet(); break; } } diff --git a/src/main/java/com/moandjiezana/toml/StringConverter.java b/src/main/java/com/moandjiezana/toml/StringConverter.java index be84cca..fdedd1b 100644 --- a/src/main/java/com/moandjiezana/toml/StringConverter.java +++ b/src/main/java/com/moandjiezana/toml/StringConverter.java @@ -3,6 +3,7 @@ package com.moandjiezana.toml; import static com.moandjiezana.toml.ValueConverterUtils.INVALID; import static com.moandjiezana.toml.ValueConverterUtils.isComment; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -18,22 +19,35 @@ class StringConverter implements ValueConverter { @Override public Object convert(String value) { - int stringTerminator = -1; + AtomicInteger index = new AtomicInteger(); + Object converted = convert(value, index); + + if (converted == INVALID || !isComment(value.substring(index.incrementAndGet()))) { + return INVALID; + } + + return converted; + } + + @Override + public Object convert(String value, AtomicInteger sharedIndex) { + int startIndex = sharedIndex.incrementAndGet(); + int endIndex = -1; char[] chars = value.toCharArray(); - for (int i = 1; i < chars.length; i++) { + for (int i = sharedIndex.get(); i < chars.length; i = sharedIndex.incrementAndGet()) { char ch = chars[i]; if (ch == '"' && chars[i - 1] != '\\') { - stringTerminator = i; + endIndex = i; break; } } - if (stringTerminator == -1 || !isComment(value.substring(stringTerminator + 1))) { + if (endIndex == -1) { return INVALID; } - value = value.substring(1, stringTerminator); + value = value.substring(startIndex, endIndex); value = replaceUnicodeCharacters(value); value = replaceSpecialCharacters(value); diff --git a/src/main/java/com/moandjiezana/toml/ValueConverter.java b/src/main/java/com/moandjiezana/toml/ValueConverter.java index cf318c5..5d9fe6a 100644 --- a/src/main/java/com/moandjiezana/toml/ValueConverter.java +++ b/src/main/java/com/moandjiezana/toml/ValueConverter.java @@ -1,5 +1,7 @@ package com.moandjiezana.toml; +import java.util.concurrent.atomic.AtomicInteger; + interface ValueConverter { /** @@ -9,6 +11,16 @@ interface ValueConverter { /** * @param s must already have been validated by {@link #canConvert(String)} + * @return a value or {@link ValueConverterUtils#INVALID} */ Object convert(String s); + + /** + * Partial validation. Stops after type terminator, rather than at EOI. + * + * @param s must already have been validated by {@link #canConvert(String)} + * @param index where to start in s + * @return a value or {@link ValueConverterUtils#INVALID} + */ + Object convert(String s, AtomicInteger index); } diff --git a/src/main/java/com/moandjiezana/toml/ValueConverters.java b/src/main/java/com/moandjiezana/toml/ValueConverters.java index aeb56e8..c68fe6a 100644 --- a/src/main/java/com/moandjiezana/toml/ValueConverters.java +++ b/src/main/java/com/moandjiezana/toml/ValueConverters.java @@ -11,13 +11,15 @@ import static com.moandjiezana.toml.NumberConverter.NUMBER_PARSER; import static com.moandjiezana.toml.StringConverter.STRING_PARSER; import static com.moandjiezana.toml.ValueConverterUtils.INVALID; +import java.util.concurrent.atomic.AtomicInteger; + class ValueConverters { private static final ValueConverter[] PARSERS = { MULTILINE_STRING_PARSER, MULTILINE_LITERAL_STRING_CONVERTER, LITERAL_STRING_PARSER, STRING_PARSER, DATE_PARSER, NUMBER_PARSER, BOOLEAN_PARSER, ARRAY_PARSER, INLINE_TABLE_PARSER }; - public Object convert(String value) { + Object convert(String value) { for (ValueConverter valueParser : PARSERS) { if (valueParser.canConvert(value)) { return valueParser.convert(value); @@ -26,4 +28,16 @@ class ValueConverters { return INVALID; } + + + Object convert(String value, AtomicInteger index) { + String substring = value.substring(index.get()); + for (ValueConverter valueParser : PARSERS) { + if (valueParser.canConvert(substring)) { + return valueParser.convert(value, index); + } + } + + return INVALID; + } } From 3e2d9fad370a18bbc44b86ee9f03161b99bfbe5c Mon Sep 17 00:00:00 2001 From: "moandji.ezana" Date: Wed, 11 Feb 2015 15:54:51 +0200 Subject: [PATCH 07/10] Simplified TomlParser --- .../com/moandjiezana/toml/ArrayConverter.java | 30 +-- .../com/moandjiezana/toml/DateConverter.java | 44 +++- .../com/moandjiezana/toml/Identifier.java | 60 +++++ .../toml/IdentifierConverter.java | 33 +++ .../toml/InlineTableConverter.java | 2 +- .../toml/MultilineLiteralStringConverter.java | 9 + .../toml/MultilineStringConverter.java | 10 +- .../java/com/moandjiezana/toml/Results.java | 33 ++- .../com/moandjiezana/toml/TomlParser.java | 240 ++++-------------- .../toml/ValueConverterUtils.java | 12 + .../moandjiezana/toml/ValueConverters.java | 15 +- 11 files changed, 263 insertions(+), 225 deletions(-) create mode 100644 src/main/java/com/moandjiezana/toml/Identifier.java create mode 100644 src/main/java/com/moandjiezana/toml/IdentifierConverter.java diff --git a/src/main/java/com/moandjiezana/toml/ArrayConverter.java b/src/main/java/com/moandjiezana/toml/ArrayConverter.java index 90849bb..770b7e5 100644 --- a/src/main/java/com/moandjiezana/toml/ArrayConverter.java +++ b/src/main/java/com/moandjiezana/toml/ArrayConverter.java @@ -1,6 +1,7 @@ package com.moandjiezana.toml; import static com.moandjiezana.toml.ValueConverterUtils.INVALID; +import static com.moandjiezana.toml.ValueConverters.CONVERTERS; import java.util.ArrayList; import java.util.List; @@ -10,8 +11,6 @@ class ArrayConverter implements ValueConverter { static final ArrayConverter ARRAY_PARSER = new ArrayConverter(); - private static final ValueConverters VALUE_CONVERTERS = new ValueConverters(); - @Override public boolean canConvert(String s) { return s.startsWith("["); @@ -41,36 +40,35 @@ class ArrayConverter implements ValueConverter { @Override public Object convert(String s, AtomicInteger index) { + int startIndex = index.get(); char[] chars = s.toCharArray(); List arrayItems = new ArrayList(); boolean terminated = false; + boolean inComment = false; for (int i = index.incrementAndGet(); i < chars.length; i = index.incrementAndGet()) { char c = chars[i]; - - if (Character.isWhitespace(c)) { + + if (c == '#' && !inComment) { + inComment = true; + } else if (c == '\n') { + inComment = false; + } else if (inComment || Character.isWhitespace(c) || c == ',') { continue; - } - if (c == ',') { - continue; - } - - if (c == '[') { + } else if (c == '[') { arrayItems.add(convert(s, index)); continue; - } - - if (c == ']') { + } else if (c == ']') { terminated = true; break; + } else { + arrayItems.add(CONVERTERS.convert(s, index)); } - - arrayItems.add(VALUE_CONVERTERS.convert(s, index)); } if (!terminated) { - return INVALID; + return ValueConverterUtils.unterminated(s.substring(startIndex, s.length())); } for (Object arrayItem : arrayItems) { diff --git a/src/main/java/com/moandjiezana/toml/DateConverter.java b/src/main/java/com/moandjiezana/toml/DateConverter.java index 6c49ba7..0f79071 100644 --- a/src/main/java/com/moandjiezana/toml/DateConverter.java +++ b/src/main/java/com/moandjiezana/toml/DateConverter.java @@ -15,9 +15,25 @@ class DateConverter implements ValueConverter { @Override public boolean canConvert(String s) { - Matcher matcher = DATE_REGEX.matcher(s); + if (s.length() < 5) { + return false; + } + + char[] chars = s.toCharArray(); - return matcher.matches(); + for (int i = 0; i < 5; i++) { + char c = chars[i]; + + if (i < 4) { + if (!Character.isDigit(c)) { + return false; + } + } else if (c != '-') { + return false; + } + } + + return true; } @Override @@ -55,9 +71,25 @@ class DateConverter implements ValueConverter { @Override public Object convert(String original, AtomicInteger index) { - String s = original.substring(index.get()); + StringBuilder sb = new StringBuilder(); + + for (int i = index.get(); i < original.length(); i = index.incrementAndGet()) { + char c = original.charAt(i); + if (Character.isDigit(c) || c == '-' || c == ':' || c == '.' || c == 'T' || c == 'Z') { + sb.append(c); + } else { + index.decrementAndGet(); + break; + } + } + + String s = sb.toString(); Matcher matcher = DATE_REGEX.matcher(s); - matcher.matches(); + + if (!matcher.matches()) { + return INVALID; + } + String dateString = matcher.group(1); String zone = matcher.group(3); String fractionalSeconds = matcher.group(2); @@ -72,9 +104,7 @@ class DateConverter implements ValueConverter { } else if (zone.contains(":")) { dateString += zone.replace(":", ""); } - - index.addAndGet(matcher.end(3) - 1); - + try { SimpleDateFormat dateFormat = new SimpleDateFormat(format); dateFormat.setLenient(false); diff --git a/src/main/java/com/moandjiezana/toml/Identifier.java b/src/main/java/com/moandjiezana/toml/Identifier.java new file mode 100644 index 0000000..acf44fc --- /dev/null +++ b/src/main/java/com/moandjiezana/toml/Identifier.java @@ -0,0 +1,60 @@ +package com.moandjiezana.toml; + +class Identifier { + + static final Identifier INVALID = new Identifier(""); + + private final String name; + private final Type type; + + Identifier(String name) { + this.name = name; + if (name.startsWith("[[")) { + this.type = Type.TABLE_ARRAY; + } else if (name.startsWith("[")) { + this.type = Type.TABLE; + } else { + this.type = Type.KEY; + } + } + + boolean acceptsNext(char c) { + if (isKey()) { + return c == '='; + } + + return c == '\n' || c == '#'; + } + + String getName() { + return name; + } + + boolean isKey() { + return type == Type.KEY; + } + + boolean isTable() { + return type == Type.TABLE; + } + + boolean isTableArray() { + return type == Type.TABLE_ARRAY; + } + + boolean isValid() { + if (isKey()) { + return Keys.getKey(name) != null; + } + + if (isTable()) { + return Keys.getTableName(name) != null; + } + + return Keys.getTableArrayName(name) != null; + } + + private static enum Type { + KEY, TABLE, TABLE_ARRAY; + } +} diff --git a/src/main/java/com/moandjiezana/toml/IdentifierConverter.java b/src/main/java/com/moandjiezana/toml/IdentifierConverter.java new file mode 100644 index 0000000..44d6aa5 --- /dev/null +++ b/src/main/java/com/moandjiezana/toml/IdentifierConverter.java @@ -0,0 +1,33 @@ +package com.moandjiezana.toml; + +import java.util.concurrent.atomic.AtomicInteger; + +class IdentifierConverter { + + static final IdentifierConverter IDENTIFIER_CONVERTER = new IdentifierConverter(); + + Identifier convert(char[] chars, AtomicInteger index) { + boolean quoted = false; + StringBuilder name = new StringBuilder(); + Identifier identifier = null; + + for (int i = index.get(); i < chars.length; i = index.incrementAndGet()) { + char c = chars[i]; + if (c == '"' && (i == 0 || chars[i - 1] != '\\')) { + quoted = !quoted; + name.append('"'); + } else if (c == '\n' || (!quoted && (c == '#' || c == '='))) { + return new Identifier(name.toString().trim()); + } else if (i == chars.length - 1 && identifier == null) { + name.append(c); + return new Identifier(name.toString().trim()); + } else { + name.append(c); + } + } + + return identifier != null ? identifier : Identifier.INVALID; + } + + private IdentifierConverter() {} +} diff --git a/src/main/java/com/moandjiezana/toml/InlineTableConverter.java b/src/main/java/com/moandjiezana/toml/InlineTableConverter.java index 54c3a26..bcfda23 100644 --- a/src/main/java/com/moandjiezana/toml/InlineTableConverter.java +++ b/src/main/java/com/moandjiezana/toml/InlineTableConverter.java @@ -1,6 +1,7 @@ package com.moandjiezana.toml; import static com.moandjiezana.toml.ValueConverterUtils.INVALID; +import static com.moandjiezana.toml.ValueConverters.CONVERTERS; import java.util.HashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -8,7 +9,6 @@ import java.util.concurrent.atomic.AtomicInteger; class InlineTableConverter implements ValueConverter { static final InlineTableConverter INLINE_TABLE_PARSER = new InlineTableConverter(); - private static final ValueConverters CONVERTERS = new ValueConverters(); @Override public boolean canConvert(String s) { diff --git a/src/main/java/com/moandjiezana/toml/MultilineLiteralStringConverter.java b/src/main/java/com/moandjiezana/toml/MultilineLiteralStringConverter.java index 37323d2..5210727 100644 --- a/src/main/java/com/moandjiezana/toml/MultilineLiteralStringConverter.java +++ b/src/main/java/com/moandjiezana/toml/MultilineLiteralStringConverter.java @@ -29,9 +29,14 @@ class MultilineLiteralStringConverter implements ValueConverter { @Override public Object convert(String s, AtomicInteger index) { char[] chars = s.toCharArray(); + int originalStartIndex = index.get(); int startIndex = index.addAndGet(3); int endIndex = -1; + if (chars[startIndex] == '\n') { + startIndex = index.incrementAndGet(); + } + for (int i = startIndex; i < chars.length; i = index.incrementAndGet()) { char c = chars[i]; @@ -41,6 +46,10 @@ class MultilineLiteralStringConverter implements ValueConverter { break; } } + + if (endIndex == -1) { + return ValueConverterUtils.unterminated(s.substring(originalStartIndex, s.length())); + } return s.substring(startIndex, endIndex); } diff --git a/src/main/java/com/moandjiezana/toml/MultilineStringConverter.java b/src/main/java/com/moandjiezana/toml/MultilineStringConverter.java index 55417ac..631e964 100644 --- a/src/main/java/com/moandjiezana/toml/MultilineStringConverter.java +++ b/src/main/java/com/moandjiezana/toml/MultilineStringConverter.java @@ -2,6 +2,7 @@ package com.moandjiezana.toml; import static com.moandjiezana.toml.ValueConverterUtils.INVALID; import static com.moandjiezana.toml.ValueConverterUtils.isComment; +import static com.moandjiezana.toml.ValueConverterUtils.unterminated; import java.util.concurrent.atomic.AtomicInteger; @@ -29,9 +30,14 @@ class MultilineStringConverter implements ValueConverter { @Override public Object convert(String s, AtomicInteger index) { char[] chars = s.toCharArray(); + int originalStartIndex = index.get(); int startIndex = index.addAndGet(3); int endIndex = -1; - + + if (chars[startIndex] == '\n') { + startIndex = index.incrementAndGet(); + } + for (int i = startIndex; i < chars.length; i = index.incrementAndGet()) { char c = chars[i]; @@ -43,7 +49,7 @@ class MultilineStringConverter implements ValueConverter { } if (endIndex == -1) { - return INVALID; + return unterminated(s.substring(originalStartIndex, s.length())); } s = s.substring(startIndex, endIndex); diff --git a/src/main/java/com/moandjiezana/toml/Results.java b/src/main/java/com/moandjiezana/toml/Results.java index 8b02882..5a64497 100644 --- a/src/main/java/com/moandjiezana/toml/Results.java +++ b/src/main/java/com/moandjiezana/toml/Results.java @@ -50,6 +50,33 @@ class Results { .append('\n'); } + void invalidIdentifier(Identifier identifier, int line) { + if (identifier.isKey()) { + invalidKey(identifier.getName(), line); + } else if (identifier.isTable()) { + invalidTable(identifier.getName(), line); + } else if (identifier.isTableArray()) { + invalidTableArray(identifier.getName(), line); + } + } + + void invalidTextAfterIdentifier(Identifier identifier, char text, int line) { + if (identifier.isKey() && text == '\n') { + sb.append("Key ") + .append(identifier.getName()) + .append(" is not followed by an equals sign on line ") + .append(line) + .append('\n'); + } else { + sb.append("Invalid text after key ") + .append(identifier.getName()) + .append(" on line ") + .append(line) + .append(". Make sure to terminate the value or add a comment (#).") + .append('\n'); + } + } + void invalidKey(String key, int line) { sb.append("Invalid key"); if (line > -1) { @@ -79,13 +106,13 @@ class Results { .append('\n'); } - void unterminated(String key, String multiline, int line) { - sb.append("Unterminated multiline value on line ") + void unterminated(String key, String value, int line) { + sb.append("Unterminated value on line ") .append(line) .append(": ") .append(key) .append(" = ") - .append(multiline.trim()) + .append(value.trim()) .append('\n'); } diff --git a/src/main/java/com/moandjiezana/toml/TomlParser.java b/src/main/java/com/moandjiezana/toml/TomlParser.java index 2244a85..eeb3bc9 100644 --- a/src/main/java/com/moandjiezana/toml/TomlParser.java +++ b/src/main/java/com/moandjiezana/toml/TomlParser.java @@ -1,203 +1,73 @@ package com.moandjiezana.toml; +import static com.moandjiezana.toml.IdentifierConverter.IDENTIFIER_CONVERTER; import static com.moandjiezana.toml.ValueConverterUtils.INVALID; -import static com.moandjiezana.toml.ValueConverterUtils.isComment; -import java.util.regex.Pattern; +import java.util.concurrent.atomic.AtomicInteger; + +import com.moandjiezana.toml.ValueConverterUtils.Unterminated; class TomlParser { - private static final String STRING_LITERAL_DELIMITER = "'''"; - private static final Pattern MULTILINE_ARRAY_REGEX = Pattern.compile("\\s*\\[([^\\]]*)"); - private static final Pattern MULTILINE_ARRAY_REGEX_END = Pattern.compile("\\s*\\]"); - private static final ValueConverters VALUE_ANALYSIS = new ValueConverters(); - - private final Results results = new Results(); Results run(String tomlString) { + final Results results = new Results(); + if (tomlString.isEmpty()) { return results; } - - String[] lines = tomlString.split("[\\n\\r]"); - int lastKeyLine = 1; - StringBuilder multilineBuilder = new StringBuilder(); - Multiline multiline = Multiline.NONE; - String key = null; - String value = null; - - for (int i = 0; i < lines.length; i++) { - String line = lines[i]; - - if (line != null && multiline.isTrimmable()) { - line = line.trim(); - } - - if (isComment(line) || line.isEmpty()) { - continue; - } - - // TODO check that this works in multiline context - if (isTableArray(line)) { - String tableName = Keys.getTableArrayName(line); - if (tableName != null) { - results.startTableArray(tableName); - } else { - results.errors.invalidTableArray(line, i + 1); - } - - continue; - } - - if (multiline.isNotMultiline() && isTable(line)) { - String tableName = Keys.getTableName(line); - if (tableName != null) { - results.startTables(tableName); - } else { - results.errors.invalidTable(line.trim(), i + 1); - } - - continue; - } - - if (multiline.isNotMultiline() && !line.contains("=")) { - results.errors.invalidKey(line, i + 1); - continue; - } - - String[] pair = line.split("=", 2); - - if (multiline.isNotMultiline() && MULTILINE_ARRAY_REGEX.matcher(pair[1].trim()).matches()) { - multiline = Multiline.ARRAY; - key = pair[0].trim(); - multilineBuilder.append(removeComment(pair[1])); - continue; - } - - if (multiline.isNotMultiline() && pair[1].trim().startsWith("\"\"\"")) { - multiline = Multiline.STRING; - multilineBuilder.append(pair[1]); - key = pair[0].trim(); - - if (pair[1].trim().indexOf("\"\"\"", 3) > -1) { - multiline = Multiline.NONE; - pair[1] = multilineBuilder.toString().trim(); - multilineBuilder.delete(0, multilineBuilder.length()); - } else { - if (multilineBuilder.toString().trim().length() > 3) { - multilineBuilder.append('\n'); - } - continue; - } - } - - if (multiline.isNotMultiline() && pair[1].trim().startsWith(STRING_LITERAL_DELIMITER)) { - multiline = Multiline.STRING_LITERAL; - multilineBuilder.append(pair[1]); - key = pair[0].trim(); - - if (pair[1].trim().indexOf(STRING_LITERAL_DELIMITER, 3) > -1) { - multiline = Multiline.NONE; - pair[1] = multilineBuilder.toString().trim(); - multilineBuilder.delete(0, multilineBuilder.length()); - } else { - if (multilineBuilder.toString().trim().length() > 3) { - multilineBuilder.append('\n'); - } - continue; - } - } - - if (multiline == Multiline.ARRAY) { - String lineWithoutComment = removeComment(line); - multilineBuilder.append(lineWithoutComment); - if (MULTILINE_ARRAY_REGEX_END.matcher(lineWithoutComment).matches()) { - multiline = Multiline.NONE; - value = multilineBuilder.toString(); - multilineBuilder.delete(0, multilineBuilder.length()); - } else { - continue; - } - } else if (multiline == Multiline.STRING) { - multilineBuilder.append(line); - if (line.contains("\"\"\"")) { - multiline = Multiline.NONE; - value = multilineBuilder.toString().trim(); - multilineBuilder.delete(0, multilineBuilder.length()); - } else { - multilineBuilder.append('\n'); - continue; - } - } else if (multiline == Multiline.STRING_LITERAL) { - multilineBuilder.append(line); - if (line.contains(STRING_LITERAL_DELIMITER)) { - multiline = Multiline.NONE; - value = multilineBuilder.toString().trim(); - multilineBuilder.delete(0, multilineBuilder.length()); - } else { - multilineBuilder.append('\n'); - continue; - } - } else { - key = Keys.getKey(pair[0]); - if (key == null) { - results.errors.invalidKey(pair[0], i + 1); - continue; - } - value = pair[1].trim(); - } - - lastKeyLine = i + 1; - Object convertedValue = VALUE_ANALYSIS.convert(value); - - if (convertedValue != INVALID) { - results.addValue(key, convertedValue); - } else { - results.errors.invalidValue(key, value, i + 1); - } - } + char[] chars = tomlString.toCharArray(); + AtomicInteger index = new AtomicInteger(); + boolean inComment = false; + AtomicInteger line = new AtomicInteger(1); + Identifier identifier = null; + Object value = null; - if (multiline != Multiline.NONE) { - results.errors.unterminated(key, multilineBuilder.toString().trim(), lastKeyLine); + for (int i = index.get(); i < chars.length; i = index.incrementAndGet()) { + char c = chars[i]; + + if (c == '#' && !inComment) { + inComment = true; + } else if (!Character.isWhitespace(c) && !inComment && identifier == null) { + Identifier id = IDENTIFIER_CONVERTER.convert(chars, index); + + if (id.isValid()) { + char next = chars[index.get()]; + if (index.get() < chars.length -1 && !id.acceptsNext(next)) { + results.errors.invalidTextAfterIdentifier(id, next, line.get()); + } else if (id.isKey()) { + identifier = id; + } else if (id.isTable()) { + results.startTables(Keys.getTableName(id.getName())); + } else if (id.isTableArray()) { + results.startTableArray(Keys.getTableArrayName(id.getName())); + } + inComment = next == '#'; + } else { + results.errors.invalidIdentifier(id, line.get()); + } + } else if (c == '\n') { + inComment = false; + identifier = null; + value = null; + line.incrementAndGet(); + } else if (!inComment && identifier != null && identifier.isKey() && value == null && !Character.isWhitespace(c)) { + int startIndex = index.get(); + Object converted = ValueConverters.CONVERTERS.convert(tomlString, index); + value = converted; + + if (converted == INVALID) { + results.errors.invalidValue(identifier.getName(), tomlString.substring(startIndex, Math.min(index.get(), tomlString.length() - 1)), line.get()); + } else if (converted instanceof Unterminated) { + results.errors.unterminated(identifier.getName(), ((Unterminated) converted).payload, line.get()); + } else { + results.addValue(identifier.getName(), converted); + } + } else if (value != null && !inComment && !Character.isWhitespace(c)) { + results.errors.invalidTextAfterIdentifier(identifier, c, line.get()); + } } return results; } - - private boolean isTableArray(String line) { - return line.startsWith("[["); - } - - private boolean isTable(String line) { - return line.startsWith("["); - } - - private String removeComment(String line) { - line = line.trim(); - if (line.startsWith("\"")) { - int startOfComment = line.indexOf('#', line.lastIndexOf('"')); - if (startOfComment > -1) { - return line.substring(0, startOfComment - 1).trim(); - } - } else { - int startOfComment = line.indexOf('#'); - if (startOfComment > -1) { - return line.substring(0, startOfComment - 1).trim(); - } - } - - return line; - } - - private static enum Multiline { - NONE, ARRAY, STRING, STRING_LITERAL; - - public boolean isNotMultiline() { - return this == NONE; - } - - public boolean isTrimmable() { - return this == NONE || this == ARRAY; - } - } } diff --git a/src/main/java/com/moandjiezana/toml/ValueConverterUtils.java b/src/main/java/com/moandjiezana/toml/ValueConverterUtils.java index a50efb8..5603d69 100644 --- a/src/main/java/com/moandjiezana/toml/ValueConverterUtils.java +++ b/src/main/java/com/moandjiezana/toml/ValueConverterUtils.java @@ -4,6 +4,18 @@ package com.moandjiezana.toml; class ValueConverterUtils { static final Object INVALID = new Object(); + static Unterminated unterminated(String payload) { + return new Unterminated(payload); + } + + static class Unterminated { + final String payload; + + private Unterminated(String payload) { + this.payload = payload; + } + } + static boolean isComment(String line) { if (line == null || line.isEmpty()) { return true; diff --git a/src/main/java/com/moandjiezana/toml/ValueConverters.java b/src/main/java/com/moandjiezana/toml/ValueConverters.java index c68fe6a..2800188 100644 --- a/src/main/java/com/moandjiezana/toml/ValueConverters.java +++ b/src/main/java/com/moandjiezana/toml/ValueConverters.java @@ -15,21 +15,12 @@ import java.util.concurrent.atomic.AtomicInteger; class ValueConverters { + static final ValueConverters CONVERTERS = new ValueConverters(); + private static final ValueConverter[] PARSERS = { MULTILINE_STRING_PARSER, MULTILINE_LITERAL_STRING_CONVERTER, LITERAL_STRING_PARSER, STRING_PARSER, DATE_PARSER, NUMBER_PARSER, BOOLEAN_PARSER, ARRAY_PARSER, INLINE_TABLE_PARSER }; - Object convert(String value) { - for (ValueConverter valueParser : PARSERS) { - if (valueParser.canConvert(value)) { - return valueParser.convert(value); - } - } - - return INVALID; - } - - Object convert(String value, AtomicInteger index) { String substring = value.substring(index.get()); for (ValueConverter valueParser : PARSERS) { @@ -40,4 +31,6 @@ class ValueConverters { return INVALID; } + + private ValueConverters() {} } From 21c5dfd2351539e908b37c3b61a99a3c9c4d6c38 Mon Sep 17 00:00:00 2001 From: "moandji.ezana" Date: Wed, 11 Feb 2015 15:55:02 +0200 Subject: [PATCH 08/10] Improved error messages --- .../moandjiezana/toml/ErrorMessagesTest.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/moandjiezana/toml/ErrorMessagesTest.java b/src/test/java/com/moandjiezana/toml/ErrorMessagesTest.java index 0c6ac37..2ea7fe7 100644 --- a/src/test/java/com/moandjiezana/toml/ErrorMessagesTest.java +++ b/src/test/java/com/moandjiezana/toml/ErrorMessagesTest.java @@ -53,21 +53,35 @@ public class ErrorMessagesTest { @Test public void should_message_invalid_value() throws Exception { - e.expectMessage("Invalid value on line 1: k = 1 t"); + e.expectMessage("Invalid text after key k on line 1"); new Toml().parse("k = 1 t"); } @Test - public void should_message_unterminated_value() throws Exception { - e.expectMessage("Unterminated multiline value on line 1: k = '''abc"); + public void should_message_unterminated_multiline_literal_string() throws Exception { + e.expectMessage("Unterminated value on line 1: k = '''abc"); new Toml().parse("k = '''abc"); } + @Test + public void should_message_unterminated_multiline_string() throws Exception { + e.expectMessage("Unterminated value on line 1: k = \"\"\"abc\"\""); + + new Toml().parse("k = \"\"\"abc\"\""); + } + + @Test + public void should_message_unterminated_array() throws Exception { + e.expectMessage("Unterminated value on line 1: k = [\"abc\""); + + new Toml().parse("k = [\"abc\""); + } + @Test public void should_message_key_without_equals() throws Exception { - e.expectMessage("Invalid key on line 2: k"); + e.expectMessage("Key k is not followed by an equals sign on line 2"); new Toml().parse("\nk\n=3"); } From 66f7946542729752dd2ad0ef3ebb9c75a2667c33 Mon Sep 17 00:00:00 2001 From: "moandji.ezana" Date: Wed, 11 Feb 2015 16:02:30 +0200 Subject: [PATCH 09/10] Added test for mixed inline tables in array --- src/test/java/com/moandjiezana/toml/ArrayTest.java | 3 +++ .../toml/should_support_array_of_inline_tables.toml | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/moandjiezana/toml/ArrayTest.java b/src/test/java/com/moandjiezana/toml/ArrayTest.java index b27f0be..9e1f06a 100644 --- a/src/test/java/com/moandjiezana/toml/ArrayTest.java +++ b/src/test/java/com/moandjiezana/toml/ArrayTest.java @@ -99,6 +99,9 @@ public class ArrayTest { assertEquals(2, toml.getLong("points[2].x").longValue()); assertEquals(4, toml.getLong("points[2].y").longValue()); assertEquals(8, toml.getLong("points[2].z").longValue()); + assertEquals("3", toml.getString("points[3].x")); + assertEquals("6", toml.getString("points[3].y")); + assertEquals("12", toml.getString("points[3].z")); } private File file(String file) { diff --git a/src/test/resources/com/moandjiezana/toml/should_support_array_of_inline_tables.toml b/src/test/resources/com/moandjiezana/toml/should_support_array_of_inline_tables.toml index 58d6f86..9f4e928 100644 --- a/src/test/resources/com/moandjiezana/toml/should_support_array_of_inline_tables.toml +++ b/src/test/resources/com/moandjiezana/toml/should_support_array_of_inline_tables.toml @@ -1,3 +1,5 @@ points = [ { x = 1, y = 2, z = 3 }, { x = 7, y = 8, z = 9 }, - { x = 2, y = 4, z = 8 } ] + { x = 2, y = 4, z = 8 }, + { x = "3", y = "6", z = "12" } +] From 5567f6c8bd11ebb6d387a7f7143781062084aea3 Mon Sep 17 00:00:00 2001 From: "moandji.ezana" Date: Wed, 11 Feb 2015 16:02:43 +0200 Subject: [PATCH 10/10] Updated change log --- CHANGELOG.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2ea678..cb833a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,8 @@ ### Changed -* Toml#getList(String) replaced Toml#getList(String, Class) +* __REAKING:__ Toml#getList(String) replaced Toml#getList(String, Class) * Dropped dependency on Parboiled and its significant transitive dependencies -* Updated Gson to 2.3.1 ### Added @@ -14,8 +13,8 @@ ### Fixed -* Fixed short-form Unicode escapes -* Fixed exponent handling +* Short-form Unicode escape handling +* Exponent handling ## 0.3.1 / 2014-12-16 * Support for [TOML 0.3.1](https://github.com/toml-lang/toml/tree/v0.3.1) spec