Plexus-Mirror/toml4j
1
0
Fork 0
mirror of https://github.com/plexusorg/toml4j.git synced 2024-05-16 17:41:35 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'inline_tables' into wip

Browse source 
Conflicts:
	src/main/java/com/moandjiezana/toml/Results.java
	src/main/java/com/moandjiezana/toml/TomlParser.java
	src/test/java/com/moandjiezana/toml/ErrorMessagesTest.java
This commit is contained in:
moandji.ezana 2015-02-11 16:43:48 +02:00
parent d32ead52d4 5567f6c8bd
commit c4027ed2d5
22 changed files with 796 additions and 378 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
CHANGELOG.md
View file

@ -4,9 +4,12 @@
### Changed
* Toml#getList(String) replaced Toml#getList(String, Class)
* __REAKING:__ Toml#getList(String) replaced Toml#getList(String, Class)
* Dropped dependency on Parboiled and its significant transitive dependencies
* Updated Gson to 2.3.1
### Added
* Line numbers included in error messages
### Added
@ -14,8 +17,8 @@
### Fixed
* Fixed short-form Unicode escapes
* Fixed exponent handling
* Short-form Unicode escape handling
* Exponent handling
## 0.3.1 / 2014-12-16
* Support for [TOML 0.3.1](https://github.com/toml-lang/toml/tree/v0.3.1) spec

159
src/main/java/com/moandjiezana/toml/ArrayConverter.java
View file

@ -1,6 +1,7 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverters.CONVERTERS;
import java.util.ArrayList;
import java.util.List;
@ -10,8 +11,6 @@ class ArrayConverter implements ValueConverter {
static final ArrayConverter ARRAY_PARSER = new ArrayConverter();
private static final ValueConverters VALUE_CONVERTERS = new ValueConverters();
@Override
public boolean canConvert(String s) {
return s.startsWith("[");
@ -19,135 +18,75 @@ class ArrayConverter implements ValueConverter {
@Override
public Object convert(String s) {
return convert(s, new AtomicInteger(1), true);
AtomicInteger sharedIndex = new AtomicInteger();
Object converted = convert(s, sharedIndex);
char[] chars = s.toCharArray();
for (int i = sharedIndex.incrementAndGet(); i < chars.length; i++) {
char c = chars[i];
if (c == '#') {
break;
}
if (!Character.isWhitespace(c)) {
return INVALID;
}
}
return converted;
}
public Object convert(String s, AtomicInteger sharedIndex, boolean topLevel) {
@Override
public Object convert(String s, AtomicInteger index) {
int startIndex = index.get();
char[] chars = s.toCharArray();
List<Object> arrayItems = new ArrayList<Object>();
boolean terminated = false;
StringType stringType = StringType.NONE;
StringBuilder current = new StringBuilder();
boolean inComment = false;
for (int i = 1; i < chars.length; i++, sharedIndex.incrementAndGet()) {
for (int i = index.incrementAndGet(); i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (terminated && !topLevel) {
break;
}
if (terminated) {
if (c == '#') {
break;
}
if (!Character.isWhitespace(c)) {
return INVALID;
}
continue;
}
if (stringType == StringType.NONE) {
if (c == ',') {
if (current.toString().trim().length() > 0) {
arrayItems.add(current.toString());
}
current = new StringBuilder();
continue;
}
if (c == '[') {
arrayItems.add(convert(s.substring(i), sharedIndex, false));
i = sharedIndex.get();
continue;
}
if (c == ']') {
terminated = true;
if (current.toString().trim().length() > 0) {
arrayItems.add(current.toString());
}
current = new StringBuilder();
continue;
}
}
if (c == '"' && chars[i - 1] != '\\' && !stringType.accepts(c)) {
if (chars.length > i + 2 && chars[i + 1] == c && chars[i + 2] == c) {
stringType = stringType.flip(StringType.MULTILINE);
} else {
stringType = stringType.flip(StringType.BASIC);
}
}
if (c == '\'' && !stringType.accepts(c)) {
if (chars.length > i + 2 && chars[i + 1] == c && chars[i + 2] == c) {
stringType = stringType.flip(StringType.MULTILINE_LITERAL);
} else {
stringType = stringType.flip(StringType.LITERAL);
}
if (c == '#' && !inComment) {
inComment = true;
} else if (c == '\n') {
inComment = false;
} else if (inComment || Character.isWhitespace(c) || c == ',') {
continue;
} else if (c == '[') {
arrayItems.add(convert(s, index));
continue;
} else if (c == ']') {
terminated = true;
break;
} else {
arrayItems.add(CONVERTERS.convert(s, index));
}
current.append(c);
}
if (!terminated) {
return INVALID;
return ValueConverterUtils.unterminated(s.substring(startIndex, s.length()));
}
return convertList(arrayItems);
}
private Object convertList(List<Object> tokens) {
ArrayList<Object> nestedList = new ArrayList<Object>();
for (Object token : tokens) {
if (token instanceof String) {
Object converted = VALUE_CONVERTERS.convert(((String) token).trim());
if (converted == INVALID) {
return INVALID;
}
if (isHomogenousArray(converted, nestedList)) {
nestedList.add(converted);
} else {
return INVALID;
}
} else if (token instanceof List) {
@SuppressWarnings("unchecked")
List<Object> convertedList = (List<Object>) token;
if (isHomogenousArray(convertedList, nestedList)) {
nestedList.add(convertedList);
} else {
return INVALID;
}
for (Object arrayItem : arrayItems) {
if (arrayItem == INVALID) {
return INVALID;
}
if (!isHomogenousArray(arrayItem, arrayItems)) {
return INVALID;
}
}
return nestedList;
return arrayItems;
}
private boolean isHomogenousArray(Object o, List<?> values) {
return values.isEmpty() || values.get(0).getClass().isAssignableFrom(o.getClass()) || o.getClass().isAssignableFrom(values.get(0).getClass());
}
private static enum StringType {
NONE, BASIC, LITERAL, MULTILINE, MULTILINE_LITERAL;
StringType flip(StringType to) {
return this == NONE ? to : NONE;
}
boolean accepts(char c) {
if (this == BASIC || this == MULTILINE) {
return c != '"';
}
if (this == LITERAL || this == MULTILINE_LITERAL) {
return c != '\'';
}
return false;
}
}
private ArrayConverter() {}
}

20
src/main/java/com/moandjiezana/toml/BooleanConverter.java
View file

@ -1,6 +1,9 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.util.concurrent.atomic.AtomicInteger;
class BooleanConverter implements ValueConverter {
@ -14,13 +17,24 @@ class BooleanConverter implements ValueConverter {
@Override
public Object convert(String s) {
AtomicInteger index = new AtomicInteger();
Object converted = convert(s, index);
if (!isComment(s.substring(index.incrementAndGet()))) {
return INVALID;
}
return converted;
}
@Override
public Object convert(String s, AtomicInteger index) {
s = s.substring(index.get());
Boolean b = s.startsWith("true") ? Boolean.TRUE : Boolean.FALSE;
int endIndex = b == Boolean.TRUE ? 4 : 5;
if (!ValueConverterUtils.isComment(s.substring(endIndex))) {
return INVALID;
}
index.addAndGet(endIndex - 1);
return b;
}

73
src/main/java/com/moandjiezana/toml/DateConverter.java
View file

@ -1,8 +1,10 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.text.SimpleDateFormat;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -13,15 +15,36 @@ class DateConverter implements ValueConverter {
@Override
public boolean canConvert(String s) {
Matcher matcher = DATE_REGEX.matcher(s);
if (s.length() < 5) {
return false;
}
char[] chars = s.toCharArray();
return matcher.matches() && ValueConverterUtils.isComment(matcher.group(4));
for (int i = 0; i < 5; i++) {
char c = chars[i];
if (i < 4) {
if (!Character.isDigit(c)) {
return false;
}
} else if (c != '-') {
return false;
}
}
return true;
}
@Override
public Object convert(String s) {
Matcher matcher = DATE_REGEX.matcher(s);
matcher.matches();
if (!isComment(matcher.group(4))) {
return INVALID;
}
s = matcher.group(1);
String zone = matcher.group(3);
String fractionalSeconds = matcher.group(2);
@ -36,6 +59,7 @@ class DateConverter implements ValueConverter {
} else if (zone.contains(":")) {
s += zone.replace(":", "");
}
try {
SimpleDateFormat dateFormat = new SimpleDateFormat(format);
dateFormat.setLenient(false);
@ -44,6 +68,51 @@ class DateConverter implements ValueConverter {
return INVALID;
}
}
@Override
public Object convert(String original, AtomicInteger index) {
StringBuilder sb = new StringBuilder();
for (int i = index.get(); i < original.length(); i = index.incrementAndGet()) {
char c = original.charAt(i);
if (Character.isDigit(c) || c == '-' || c == ':' || c == '.' || c == 'T' || c == 'Z') {
sb.append(c);
} else {
index.decrementAndGet();
break;
}
}
String s = sb.toString();
Matcher matcher = DATE_REGEX.matcher(s);
if (!matcher.matches()) {
return INVALID;
}
String dateString = matcher.group(1);
String zone = matcher.group(3);
String fractionalSeconds = matcher.group(2);
String format = "yyyy-MM-dd'T'HH:mm:ss";
if (fractionalSeconds != null && !fractionalSeconds.isEmpty()) {
format += ".SSS";
dateString += fractionalSeconds;
}
format += "Z";
if ("Z".equals(zone)) {
dateString += "+0000";
} else if (zone.contains(":")) {
dateString += zone.replace(":", "");
}
try {
SimpleDateFormat dateFormat = new SimpleDateFormat(format);
dateFormat.setLenient(false);
return dateFormat.parse(dateString);
} catch (Exception e) {
return INVALID;
}
}
private DateConverter() {}
}

60
src/main/java/com/moandjiezana/toml/Identifier.java Normal file
View file

@ -0,0 +1,60 @@
package com.moandjiezana.toml;
class Identifier {
static final Identifier INVALID = new Identifier("");
private final String name;
private final Type type;
Identifier(String name) {
this.name = name;
if (name.startsWith("[[")) {
this.type = Type.TABLE_ARRAY;
} else if (name.startsWith("[")) {
this.type = Type.TABLE;
} else {
this.type = Type.KEY;
}
}
boolean acceptsNext(char c) {
if (isKey()) {
return c == '=';
}
return c == '\n' || c == '#';
}
String getName() {
return name;
}
boolean isKey() {
return type == Type.KEY;
}
boolean isTable() {
return type == Type.TABLE;
}
boolean isTableArray() {
return type == Type.TABLE_ARRAY;
}
boolean isValid() {
if (isKey()) {
return Keys.getKey(name) != null;
}
if (isTable()) {
return Keys.getTableName(name) != null;
}
return Keys.getTableArrayName(name) != null;
}
private static enum Type {
KEY, TABLE, TABLE_ARRAY;
}
}

33
src/main/java/com/moandjiezana/toml/IdentifierConverter.java Normal file
View file

@ -0,0 +1,33 @@
package com.moandjiezana.toml;
import java.util.concurrent.atomic.AtomicInteger;
class IdentifierConverter {
static final IdentifierConverter IDENTIFIER_CONVERTER = new IdentifierConverter();
Identifier convert(char[] chars, AtomicInteger index) {
boolean quoted = false;
StringBuilder name = new StringBuilder();
Identifier identifier = null;
for (int i = index.get(); i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (c == '"' && (i == 0 || chars[i - 1] != '\\')) {
quoted = !quoted;
name.append('"');
} else if (c == '\n' || (!quoted && (c == '#' || c == '='))) {
return new Identifier(name.toString().trim());
} else if (i == chars.length - 1 && identifier == null) {
name.append(c);
return new Identifier(name.toString().trim());
} else {
name.append(c);
}
}
return identifier != null ? identifier : Identifier.INVALID;
}
private IdentifierConverter() {}
}

95
src/main/java/com/moandjiezana/toml/InlineTableConverter.java Normal file
View file

@ -0,0 +1,95 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverters.CONVERTERS;
import java.util.HashMap;
import java.util.concurrent.atomic.AtomicInteger;
class InlineTableConverter implements ValueConverter {
static final InlineTableConverter INLINE_TABLE_PARSER = new InlineTableConverter();
@Override
public boolean canConvert(String s) {
return s.startsWith("{");
}
@Override
public Object convert(String s) {
AtomicInteger index = new AtomicInteger();
Object converted = convert(s, index);
String substring = s.substring(index.incrementAndGet());
if (converted == INVALID || !ValueConverterUtils.isComment(substring)) {
return INVALID;
}
return converted;
}
@Override
public Object convert(String s, AtomicInteger sharedIndex) {
char[] chars = s.toCharArray();
boolean inKey = true;
boolean inValue = false;
boolean quoted = false;
boolean terminated = false;
StringBuilder currentKey = new StringBuilder();
HashMap<String, Object> results = new HashMap<String, Object>();
for (int i = sharedIndex.incrementAndGet(); sharedIndex.get() < chars.length; i = sharedIndex.incrementAndGet()) {
char c = chars[i];
if (c == '"' && inKey) {
quoted = !quoted;
currentKey.append(c);
} else if (quoted) {
currentKey.append(c);
} else if (inValue && !Character.isWhitespace(c)) {
Object converted = CONVERTERS.convert(s, sharedIndex);
if (converted == INVALID) {
return INVALID;
}
results.put(currentKey.toString().trim(), converted);
currentKey = new StringBuilder();
inValue = false;
} else if (c == '{') {
sharedIndex.incrementAndGet();
Object converted = convert(s, sharedIndex);
if (converted == INVALID) {
return INVALID;
}
results.put(currentKey.toString().trim(), converted);
inKey = true;
inValue = false;
currentKey = new StringBuilder();
} else if (c == ',') {
inKey = true;
inValue = false;
currentKey = new StringBuilder();
} else if (c == '=') {
inKey = false;
inValue = true;
} else if (c == '}') {
terminated = true;
break;
} else if (inKey) {
currentKey.append(c);
}
}
if (!terminated) {
return INVALID;
}
return results;
}
private InlineTableConverter() {}
}

52
src/main/java/com/moandjiezana/toml/LiteralStringConverter.java
View file

@ -1,6 +1,9 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.util.concurrent.atomic.AtomicInteger;
class LiteralStringConverter implements ValueConverter {
@ -14,31 +17,38 @@ class LiteralStringConverter implements ValueConverter {
@Override
public Object convert(String s) {
char[] chars = s.toCharArray();
int endIndex = -1;
AtomicInteger index = new AtomicInteger();
Object converted = convert(s, index);
for (int i = 1; i < chars.length; i++) {
char c = chars[i];
if (c == '\'') {
endIndex = i;
continue;
}
if (endIndex > -1 && c == '#') {
break;
}
if (endIndex > -1 && !Character.isWhitespace(c)) {
return INVALID;
}
}
if (endIndex == -1) {
if (converted == INVALID || !isComment(s.substring(index.incrementAndGet()))) {
return INVALID;
}
return s.substring(1, endIndex);
return converted;
}
@Override
public Object convert(String s, AtomicInteger index) {
char[] chars = s.toCharArray();
boolean terminated = false;
int startIndex = index.incrementAndGet();
for (int i = index.get(); i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (c == '\'') {
terminated = true;
break;
}
}
if (!terminated) {
return INVALID;
}
String substring = s.substring(startIndex, index.get());
return substring;
}
private LiteralStringConverter() {}

39
src/main/java/com/moandjiezana/toml/MultilineLiteralStringConverter.java
View file

@ -1,6 +1,9 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.util.concurrent.atomic.AtomicInteger;
class MultilineLiteralStringConverter implements ValueConverter {
@ -13,28 +16,42 @@ class MultilineLiteralStringConverter implements ValueConverter {
@Override
public Object convert(String s) {
AtomicInteger index = new AtomicInteger();
Object converted = convert(s, index);
if (converted == INVALID || !isComment(s.substring(index.incrementAndGet()))) {
return INVALID;
}
return converted;
}
@Override
public Object convert(String s, AtomicInteger index) {
char[] chars = s.toCharArray();
int originalStartIndex = index.get();
int startIndex = index.addAndGet(3);
int endIndex = -1;
for (int i = 3; i < chars.length; i++) {
if (chars[startIndex] == '\n') {
startIndex = index.incrementAndGet();
}
for (int i = startIndex; i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (c == '\'' && chars.length > i + 2 && chars[i + 1] == '\'' && chars[i + 2] == '\'') {
endIndex = i;
i += 2;
continue;
}
if (endIndex > -1 && c == '#') {
index.addAndGet(2);
break;
}
if (endIndex > -1 && !Character.isWhitespace(c)) {
return INVALID;
}
}
if (endIndex == -1) {
return ValueConverterUtils.unterminated(s.substring(originalStartIndex, s.length()));
}
return s.substring(3, endIndex);
return s.substring(startIndex, endIndex);
}
private MultilineLiteralStringConverter() {}

62
src/main/java/com/moandjiezana/toml/MultilineStringConverter.java
View file

@ -1,9 +1,13 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import static com.moandjiezana.toml.ValueConverterUtils.unterminated;
import java.util.concurrent.atomic.AtomicInteger;
class MultilineStringConverter implements ValueConverter {
static final MultilineStringConverter MULTILINE_STRING_PARSER = new MultilineStringConverter();
@Override
@ -13,22 +17,50 @@ class MultilineStringConverter implements ValueConverter {
@Override
public Object convert(String s) {
int terminator = s.indexOf("\"\"\"", 3);
if (terminator == -1) {
AtomicInteger index = new AtomicInteger();
Object converted = convert(s, index);
if (converted == INVALID || !isComment(s.substring(index.incrementAndGet()))) {
return INVALID;
}
if (!ValueConverterUtils.isComment(s.substring(terminator + 3))) {
return INVALID;
}
s = s.substring(2, terminator + 1);
s = s.replaceAll("\\\\\\s+", "");
return StringConverter.STRING_PARSER.convert(s);
return converted;
}
@Override
public Object convert(String s, AtomicInteger index) {
char[] chars = s.toCharArray();
int originalStartIndex = index.get();
int startIndex = index.addAndGet(3);
int endIndex = -1;
if (chars[startIndex] == '\n') {
startIndex = index.incrementAndGet();
}
for (int i = startIndex; i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (c == '"' && chars.length > i + 2 && chars[i + 1] == '"' && chars[i + 2] == '"') {
endIndex = i;
index.addAndGet(2);
break;
}
}
if (endIndex == -1) {
return unterminated(s.substring(originalStartIndex, s.length()));
}
s = s.substring(startIndex, endIndex);
s = s.replaceAll("\\\\\\s+", "");
s = StringConverter.STRING_PARSER.replaceUnicodeCharacters(s);
s = StringConverter.STRING_PARSER.replaceSpecialCharacters(s);
return s;
}
private MultilineStringConverter() {
}
private MultilineStringConverter() {}
}

32
src/main/java/com/moandjiezana/toml/NumberConverter.java
View file

@ -1,6 +1,9 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.util.concurrent.atomic.AtomicInteger;
class NumberConverter implements ValueConverter {
static final NumberConverter NUMBER_PARSER = new NumberConverter();
@ -14,20 +17,33 @@ class NumberConverter implements ValueConverter {
@Override
public Object convert(String s) {
AtomicInteger index = new AtomicInteger();
Object converted = convert(s, index);
if (converted == INVALID || (s.length() > index.get() + 1 && !isComment(s.substring(index.incrementAndGet())))) {
return INVALID;
}
return converted;
}
@Override
public Object convert(String s, AtomicInteger index) {
char[] chars = s.toCharArray();
boolean whitespace = false;
boolean signable = true;
boolean dottable = false;
boolean exponentable = false;
boolean terminatable = false;
String type = "";
StringBuilder sb = new StringBuilder();
for (int i = 0; i < chars.length; i++) {
for (int i = index.get(); i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (Character.isDigit(c)) {
sb.append(c);
signable = false;
terminatable = true;
if (type.isEmpty()) {
type = "integer";
dottable = true;
@ -35,26 +51,28 @@ class NumberConverter implements ValueConverter {
exponentable = !type.equals("exponent");
} else if ((c == '+' || c == '-') && signable && chars.length > i + 1) {
signable = false;
terminatable = false;
if (c == '-') {
sb.append('-');
}
} else if (c == '.' && dottable && chars.length > i + 1) {
sb.append('.');
type = "float";
terminatable = false;
dottable = false;
exponentable = false;
} else if ((c == 'E' || c == 'e') && exponentable && chars.length > i + 1) {
sb.append('E');
type = "exponent";
terminatable = false;
signable = true;
dottable = false;
exponentable = false;
} else if (Character.isWhitespace(c)) {
whitespace = true;
} else if (whitespace && c == '#') {
break;
} else {
type = "";
if (!terminatable) {
type = "";
}
index.decrementAndGet();
break;
}
}

47
src/main/java/com/moandjiezana/toml/Results.java
View file

@ -50,6 +50,33 @@ class Results {
.append('\n');
}
void invalidIdentifier(Identifier identifier, int line) {
if (identifier.isKey()) {
invalidKey(identifier.getName(), line);
} else if (identifier.isTable()) {
invalidTable(identifier.getName(), line);
} else if (identifier.isTableArray()) {
invalidTableArray(identifier.getName(), line);
}
}
void invalidTextAfterIdentifier(Identifier identifier, char text, int line) {
if (identifier.isKey() && text == '\n') {
sb.append("Key ")
.append(identifier.getName())
.append(" is not followed by an equals sign on line ")
.append(line)
.append('\n');
} else {
sb.append("Invalid text after key ")
.append(identifier.getName())
.append(" on line ")
.append(line)
.append(". Make sure to terminate the value or add a comment (#).")
.append('\n');
}
}
void invalidKey(String key, int line) {
sb.append("Invalid key");
if (line > -1) {
@ -79,13 +106,13 @@ class Results {
.append('\n');
}
void unterminated(String key, String multiline, int line) {
sb.append("Unterminated multiline value on line ")
void unterminated(String key, String value, int line) {
sb.append("Unterminated value on line ")
.append(line)
.append(": ")
.append(key)
.append(" = ")
.append(multiline.trim())
.append(value.trim())
.append('\n');
}
@ -98,8 +125,7 @@ class Results {
return sb.toString();
}
}
Set<String> tables = new HashSet<String>();
Set<String> tables = new HashSet<String>();
final Errors errors = new Errors();
private Deque<Container> stack = new ArrayDeque<Container>();
@ -109,7 +135,16 @@ class Results {
void addValue(String key, Object value) {
Container currentTable = stack.peek();
if (currentTable.accepts(key)) {
if (value instanceof Map) {
startTable(key);
@SuppressWarnings("unchecked")
Map<String, Object> valueMap = (Map<String, Object>) value;
for (Map.Entry<String, Object> entry : valueMap.entrySet()) {
addValue(entry.getKey(), entry.getValue());
}
stack.pop();
} else if (currentTable.accepts(key)) {
currentTable.put(key, value);
} else {
errors.duplicateKey(key, -1);

24
src/main/java/com/moandjiezana/toml/StringConverter.java
View file

@ -3,6 +3,7 @@ package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -18,22 +19,35 @@ class StringConverter implements ValueConverter {
@Override
public Object convert(String value) {
int stringTerminator = -1;
AtomicInteger index = new AtomicInteger();
Object converted = convert(value, index);
if (converted == INVALID || !isComment(value.substring(index.incrementAndGet()))) {
return INVALID;
}
return converted;
}
@Override
public Object convert(String value, AtomicInteger sharedIndex) {
int startIndex = sharedIndex.incrementAndGet();
int endIndex = -1;
char[] chars = value.toCharArray();
for (int i = 1; i < chars.length; i++) {
for (int i = sharedIndex.get(); i < chars.length; i = sharedIndex.incrementAndGet()) {
char ch = chars[i];
if (ch == '"' && chars[i - 1] != '\\') {
stringTerminator = i;
endIndex = i;
break;
}
}
if (stringTerminator == -1 || !isComment(value.substring(stringTerminator + 1))) {
if (endIndex == -1) {
return INVALID;
}
value = value.substring(1, stringTerminator);
value = value.substring(startIndex, endIndex);
value = replaceUnicodeCharacters(value);
value = replaceSpecialCharacters(value);

239
src/main/java/com/moandjiezana/toml/TomlParser.java
View file

@ -1,202 +1,73 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.IdentifierConverter.IDENTIFIER_CONVERTER;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.util.regex.Pattern;
import java.util.concurrent.atomic.AtomicInteger;
import com.moandjiezana.toml.ValueConverterUtils.Unterminated;
class TomlParser {
private static final String STRING_LITERAL_DELIMITER = "'''";
private static final Pattern MULTILINE_ARRAY_REGEX = Pattern.compile("\\s*\\[([^\\]]*)");
private static final Pattern MULTILINE_ARRAY_REGEX_END = Pattern.compile("\\s*\\]");
private static final ValueConverters VALUE_ANALYSIS = new ValueConverters();
private final Results results = new Results();
Results run(String tomlString) {
final Results results = new Results();
if (tomlString.isEmpty()) {
return results;
}
String[] lines = tomlString.split("[\\n\\r]");
int lastKeyLine = 1;
StringBuilder multilineBuilder = new StringBuilder();
Multiline multiline = Multiline.NONE;
String key = null;
String value = null;
for (int i = 0; i < lines.length; i++) {
String line = lines[i];
if (line != null && multiline.isTrimmable()) {
line = line.trim();
}
if (isComment(line) || line.isEmpty()) {
continue;
}
if (isTableArray(line)) {
String tableName = Keys.getTableArrayName(line);
if (tableName != null) {
results.startTableArray(tableName);
} else {
results.errors.invalidTableArray(line, i + 1);
}
continue;
}
if (multiline.isNotMultiline() && isTable(line)) {
String tableName = Keys.getTableName(line);
if (tableName != null) {
results.startTables(tableName);
} else {
results.errors.invalidTable(line.trim(), i + 1);
}
continue;
}
if (multiline.isNotMultiline() && !line.contains("=")) {
results.errors.invalidKey(line, i + 1);
continue;
}
String[] pair = line.split("=", 2);
if (multiline.isNotMultiline() && MULTILINE_ARRAY_REGEX.matcher(pair[1].trim()).matches()) {
multiline = Multiline.ARRAY;
key = pair[0].trim();
multilineBuilder.append(removeComment(pair[1]));
continue;
}
if (multiline.isNotMultiline() && pair[1].trim().startsWith("\"\"\"")) {
multiline = Multiline.STRING;
multilineBuilder.append(pair[1]);
key = pair[0].trim();
if (pair[1].trim().indexOf("\"\"\"", 3) > -1) {
multiline = Multiline.NONE;
pair[1] = multilineBuilder.toString().trim();
multilineBuilder.delete(0, multilineBuilder.length());
} else {
if (multilineBuilder.toString().trim().length() > 3) {
multilineBuilder.append('\n');
}
continue;
}
}
if (multiline.isNotMultiline() && pair[1].trim().startsWith(STRING_LITERAL_DELIMITER)) {
multiline = Multiline.STRING_LITERAL;
multilineBuilder.append(pair[1]);
key = pair[0].trim();
if (pair[1].trim().indexOf(STRING_LITERAL_DELIMITER, 3) > -1) {
multiline = Multiline.NONE;
pair[1] = multilineBuilder.toString().trim();
multilineBuilder.delete(0, multilineBuilder.length());
} else {
if (multilineBuilder.toString().trim().length() > 3) {
multilineBuilder.append('\n');
}
continue;
}
}
if (multiline == Multiline.ARRAY) {
String lineWithoutComment = removeComment(line);
multilineBuilder.append(lineWithoutComment);
if (MULTILINE_ARRAY_REGEX_END.matcher(lineWithoutComment).matches()) {
multiline = Multiline.NONE;
value = multilineBuilder.toString();
multilineBuilder.delete(0, multilineBuilder.length());
} else {
continue;
}
} else if (multiline == Multiline.STRING) {
multilineBuilder.append(line);
if (line.contains("\"\"\"")) {
multiline = Multiline.NONE;
value = multilineBuilder.toString().trim();
multilineBuilder.delete(0, multilineBuilder.length());
} else {
multilineBuilder.append('\n');
continue;
}
} else if (multiline == Multiline.STRING_LITERAL) {
multilineBuilder.append(line);
if (line.contains(STRING_LITERAL_DELIMITER)) {
multiline = Multiline.NONE;
value = multilineBuilder.toString().trim();
multilineBuilder.delete(0, multilineBuilder.length());
} else {
multilineBuilder.append('\n');
continue;
}
} else {
key = Keys.getKey(pair[0]);
if (key == null) {
results.errors.invalidKey(pair[0], i + 1);
continue;
}
value = pair[1].trim();
}
lastKeyLine = i + 1;
Object convertedValue = VALUE_ANALYSIS.convert(value);
if (convertedValue != INVALID) {
results.addValue(key, convertedValue);
} else {
results.errors.invalidValue(key, value, i + 1);
}
}
char[] chars = tomlString.toCharArray();
AtomicInteger index = new AtomicInteger();
boolean inComment = false;
AtomicInteger line = new AtomicInteger(1);
Identifier identifier = null;
Object value = null;
if (multiline != Multiline.NONE) {
results.errors.unterminated(key, multilineBuilder.toString().trim(), lastKeyLine);
for (int i = index.get(); i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (c == '#' && !inComment) {
inComment = true;
} else if (!Character.isWhitespace(c) && !inComment && identifier == null) {
Identifier id = IDENTIFIER_CONVERTER.convert(chars, index);
if (id.isValid()) {
char next = chars[index.get()];
if (index.get() < chars.length -1 && !id.acceptsNext(next)) {
results.errors.invalidTextAfterIdentifier(id, next, line.get());
} else if (id.isKey()) {
identifier = id;
} else if (id.isTable()) {
results.startTables(Keys.getTableName(id.getName()));
} else if (id.isTableArray()) {
results.startTableArray(Keys.getTableArrayName(id.getName()));
}
inComment = next == '#';
} else {
results.errors.invalidIdentifier(id, line.get());
}
} else if (c == '\n') {
inComment = false;
identifier = null;
value = null;
line.incrementAndGet();
} else if (!inComment && identifier != null && identifier.isKey() && value == null && !Character.isWhitespace(c)) {
int startIndex = index.get();
Object converted = ValueConverters.CONVERTERS.convert(tomlString, index);
value = converted;
if (converted == INVALID) {
results.errors.invalidValue(identifier.getName(), tomlString.substring(startIndex, Math.min(index.get(), tomlString.length() - 1)), line.get());
} else if (converted instanceof Unterminated) {
results.errors.unterminated(identifier.getName(), ((Unterminated) converted).payload, line.get());
} else {
results.addValue(identifier.getName(), converted);
}
} else if (value != null && !inComment && !Character.isWhitespace(c)) {
results.errors.invalidTextAfterIdentifier(identifier, c, line.get());
}
}
return results;
}
private boolean isTableArray(String line) {
return line.startsWith("[[");
}
private boolean isTable(String line) {
return line.startsWith("[");
}
private String removeComment(String line) {
line = line.trim();
if (line.startsWith("\"")) {
int startOfComment = line.indexOf('#', line.lastIndexOf('"'));
if (startOfComment > -1) {
return line.substring(0, startOfComment - 1).trim();
}
} else {
int startOfComment = line.indexOf('#');
if (startOfComment > -1) {
return line.substring(0, startOfComment - 1).trim();
}
}
return line;
}
private static enum Multiline {
NONE, ARRAY, STRING, STRING_LITERAL;
public boolean isNotMultiline() {
return this == NONE;
}
public boolean isTrimmable() {
return this == NONE || this == ARRAY;
}
}
}

12
src/main/java/com/moandjiezana/toml/ValueConverter.java
View file

@ -1,5 +1,7 @@
package com.moandjiezana.toml;
import java.util.concurrent.atomic.AtomicInteger;
interface ValueConverter {
/**
@ -9,6 +11,16 @@ interface ValueConverter {
/**
* @param s must already have been validated by {@link #canConvert(String)}
* @return a value or {@link ValueConverterUtils#INVALID}
*/
Object convert(String s);
/**
* Partial validation. Stops after type terminator, rather than at EOI.
*
* @param s must already have been validated by {@link #canConvert(String)}
* @param index where to start in s
* @return a value or {@link ValueConverterUtils#INVALID}
*/
Object convert(String s, AtomicInteger index);
}

12
src/main/java/com/moandjiezana/toml/ValueConverterUtils.java
View file

@ -4,6 +4,18 @@ package com.moandjiezana.toml;
class ValueConverterUtils {
static final Object INVALID = new Object();
static Unterminated unterminated(String payload) {
return new Unterminated(payload);
}
static class Unterminated {
final String payload;
private Unterminated(String payload) {
this.payload = payload;
}
}
static boolean isComment(String line) {
if (line == null || line.isEmpty()) {
return true;

16
src/main/java/com/moandjiezana/toml/ValueConverters.java
View file

@ -3,6 +3,7 @@ package com.moandjiezana.toml;
import static com.moandjiezana.toml.ArrayConverter.ARRAY_PARSER;
import static com.moandjiezana.toml.BooleanConverter.BOOLEAN_PARSER;
import static com.moandjiezana.toml.DateConverter.DATE_PARSER;
import static com.moandjiezana.toml.InlineTableConverter.INLINE_TABLE_PARSER;
import static com.moandjiezana.toml.LiteralStringConverter.LITERAL_STRING_PARSER;
import static com.moandjiezana.toml.MultilineLiteralStringConverter.MULTILINE_LITERAL_STRING_CONVERTER;
import static com.moandjiezana.toml.MultilineStringConverter.MULTILINE_STRING_PARSER;
@ -10,19 +11,26 @@ import static com.moandjiezana.toml.NumberConverter.NUMBER_PARSER;
import static com.moandjiezana.toml.StringConverter.STRING_PARSER;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import java.util.concurrent.atomic.AtomicInteger;
class ValueConverters {
static final ValueConverters CONVERTERS = new ValueConverters();
private static final ValueConverter[] PARSERS = {
MULTILINE_STRING_PARSER, MULTILINE_LITERAL_STRING_CONVERTER, LITERAL_STRING_PARSER, STRING_PARSER, DATE_PARSER, NUMBER_PARSER, BOOLEAN_PARSER, ARRAY_PARSER
MULTILINE_STRING_PARSER, MULTILINE_LITERAL_STRING_CONVERTER, LITERAL_STRING_PARSER, STRING_PARSER, DATE_PARSER, NUMBER_PARSER, BOOLEAN_PARSER, ARRAY_PARSER, INLINE_TABLE_PARSER
};
public Object convert(String value) {
Object convert(String value, AtomicInteger index) {
String substring = value.substring(index.get());
for (ValueConverter valueParser : PARSERS) {
if (valueParser.canConvert(value)) {
return valueParser.convert(value);
if (valueParser.canConvert(substring)) {
return valueParser.convert(value, index);
}
}
return INVALID;
}
private ValueConverters() {}
}

31
src/test/java/com/moandjiezana/toml/ArrayTest.java
View file

@ -2,11 +2,13 @@ package com.moandjiezana.toml;
import static java.util.Arrays.asList;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.util.Arrays;
import java.util.List;
import org.junit.Test;
@ -36,6 +38,16 @@ public class ArrayTest {
assertEquals(asList(asList("gamma", "delta"), asList(1L, 2L)), clients.<String>getList("data"));
}
@Test
public void should_get_deeply_nested_arrays() throws Exception {
List<List<?>> data = new Toml().parse("data = [[[1], [2]], [3, 4]]").getList("data");
assertThat(data, hasSize(2));
assertEquals(Arrays.asList(1L), data.get(0).get(0));
assertEquals(asList(2L), data.get(0).get(1));
assertEquals(asList(3L, 4L), data.get(1));
}
@Test
@SuppressWarnings("unchecked")
@ -73,6 +85,25 @@ public class ArrayTest {
assertThat(toml.<String>getList("key"), contains("a]", "b]", "c]", "d]"));
}
@Test
public void should_support_array_of_inline_tables() throws Exception {
Toml toml = new Toml().parse(getClass().getResourceAsStream("should_support_array_of_inline_tables.toml"));
assertThat(toml.getList("points"), hasSize(4));
assertEquals(1, toml.getLong("points[0].x").longValue());
assertEquals(2, toml.getLong("points[0].y").longValue());
assertEquals(3, toml.getLong("points[0].z").longValue());
assertEquals(7, toml.getLong("points[1].x").longValue());
assertEquals(8, toml.getLong("points[1].y").longValue());
assertEquals(9, toml.getLong("points[1].z").longValue());
assertEquals(2, toml.getLong("points[2].x").longValue());
assertEquals(4, toml.getLong("points[2].y").longValue());
assertEquals(8, toml.getLong("points[2].z").longValue());
assertEquals("3", toml.getString("points[3].x"));
assertEquals("6", toml.getString("points[3].y"));
assertEquals("12", toml.getString("points[3].z"));
}
private File file(String file) {
return Utils.file(getClass(), file);
}

2
src/test/java/com/moandjiezana/toml/BareKeysTest.java
View file

@ -2,9 +2,7 @@ package com.moandjiezana.toml;
import static org.junit.Assert.assertEquals;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
public class BareKeysTest {

22
src/test/java/com/moandjiezana/toml/ErrorMessagesTest.java
View file

@ -53,21 +53,35 @@ public class ErrorMessagesTest {
@Test
public void should_message_invalid_value() throws Exception {
e.expectMessage("Invalid value on line 1: k = 1 t");
e.expectMessage("Invalid text after key k on line 1");
new Toml().parse("k = 1 t");
}
@Test
public void should_message_unterminated_value() throws Exception {
e.expectMessage("Unterminated multiline value on line 1: k = '''abc");
public void should_message_unterminated_multiline_literal_string() throws Exception {
e.expectMessage("Unterminated value on line 1: k = '''abc");
new Toml().parse("k = '''abc");
}
@Test
public void should_message_unterminated_multiline_string() throws Exception {
e.expectMessage("Unterminated value on line 1: k = \"\"\"abc\"\"");
new Toml().parse("k = \"\"\"abc\"\"");
}
@Test
public void should_message_unterminated_array() throws Exception {
e.expectMessage("Unterminated value on line 1: k = [\"abc\"");
new Toml().parse("k = [\"abc\"");
}
@Test
public void should_message_key_without_equals() throws Exception {
e.expectMessage("Invalid key on line 2: k");
e.expectMessage("Key k is not followed by an equals sign on line 2");
new Toml().parse("\nk\n=3");
}

128
src/test/java/com/moandjiezana/toml/InlineTableTest.java Normal file
View file

@ -0,0 +1,128 @@
package com.moandjiezana.toml;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import java.util.Calendar;
import java.util.List;
import java.util.TimeZone;
import org.junit.Test;
public class InlineTableTest {
private static final TimeZone UTC = TimeZone.getTimeZone("UTC");
@Test
public void should_read_empty_inline_table() throws Exception {
Toml toml = new Toml().parse("key = {}");
assertNotNull(toml.getTable("key"));
}
@Test
public void should_read_inline_table_with_strings() throws Exception {
Toml toml = new Toml().parse("name = { first = \"Tom\", last = \"Preston-Werner\"}");
assertEquals("Tom", toml.getTable("name").getString("first"));
assertEquals("Preston-Werner", toml.getString("name.last"));
}
@Test
public void should_read_inline_table_with_integers() throws Exception {
Toml toml = new Toml().parse("point = { x = 1, y = 2 }");
assertEquals(1, toml.getTable("point").getLong("x").longValue());
assertEquals(2, toml.getLong("point.y").longValue());
}
@Test
public void should_read_inline_table_with_floats() throws Exception {
Toml toml = new Toml().parse("point = { x = 1.5, y = 2.3 }");
assertEquals(1.5, toml.getTable("point").getDouble("x").doubleValue(), 0);
assertEquals(2.3, toml.getDouble("point.y").doubleValue(), 0);
}
@Test
public void should_read_inline_table_with_booleans() throws Exception {
Toml toml = new Toml().parse("point = { x = false, y = true }");
assertTrue(toml.getTable("point").getBoolean("y"));
assertFalse(toml.getBoolean("point.x"));
}
@Test
public void should_read_inline_table_with_dates() throws Exception {
Toml toml = new Toml().parse("point = { x = 2015-02-09T22:05:00Z, y = 2015-02-09T21:05:00Z }");
Calendar x = Calendar.getInstance(UTC);
x.set(2015, Calendar.FEBRUARY, 9, 22, 5, 00);
x.set(Calendar.MILLISECOND, 0);
Calendar y = Calendar.getInstance(UTC);
y.set(2015, Calendar.FEBRUARY, 9, 21, 5, 00);
y.set(Calendar.MILLISECOND, 0);
assertEquals(x.getTime(), toml.getTable("point").getDate("x"));
assertEquals(y.getTime(), toml.getDate("point.y"));
}
@Test
public void should_read_arrays() throws Exception {
Toml toml = new Toml().parse("arrays = { integers = [1, 2, 3], strings = [\"a\", \"b\", \"c\"] }");
assertThat(toml.<Long>getList("arrays.integers"), contains(1L, 2L, 3L));
assertThat(toml.<String>getList("arrays.strings"), contains("a", "b", "c"));
}
@Test
public void should_read_nested_arrays() throws Exception {
Toml toml = new Toml().parse("arrays = { nested = [[1, 2, 3], [4, 5, 6]] }").getTable("arrays");
List<List<Long>> nested = toml.<List<Long>>getList("nested");
assertThat(nested, hasSize(2));
assertThat(nested.get(0), contains(1L, 2L, 3L));
assertThat(nested.get(1), contains(4L, 5L, 6L));
}
@Test
public void should_read_mixed_inline_table() throws Exception {
Toml toml = new Toml().parse("point = { date = 2015-02-09T22:05:00Z, bool = true, integer = 123, float = 123.456, string = \"abc\", list = [5, 6, 7, 8] }").getTable("point");
Calendar date = Calendar.getInstance(UTC);
date.set(2015, Calendar.FEBRUARY, 9, 22, 5, 00);
date.set(Calendar.MILLISECOND, 0);
assertEquals(date.getTime(), toml.getDate("date"));
assertTrue(toml.getBoolean("bool"));
assertEquals(123, toml.getLong("integer").intValue());
assertEquals(123.456, toml.getDouble("float"), 0);
assertEquals("abc", toml.getString("string"));
assertThat(toml.<Long>getList("list"), contains(5L, 6L, 7L, 8L));
}
@Test
public void should_read_nested_inline_tables() throws Exception {
Toml tables = new Toml().parse("tables = { t1 = { t1_1 = 1, t1_2 = 2}, t2 = { t2_1 = \"a\"} }").getTable("tables");
assertEquals(1L, tables.getLong("t1.t1_1").longValue());
assertEquals(2L, tables.getLong("t1.t1_2").longValue());
assertEquals("a", tables.getString("t2.t2_1"));
}
@Test
public void should_read_all_string_types() throws Exception {
Toml strings = new Toml().parse("strings = { literal = 'ab]\"c', multiline = \"\"\"de]\"f\"\"\", multiline_literal = '''gh]\"i''' }").getTable("strings");
assertEquals("ab]\"c", strings.getString("literal"));
assertEquals("de]\"f", strings.getString("multiline"));
assertEquals("gh]\"i", strings.getString("multiline_literal"));
}
}

5
src/test/resources/com/moandjiezana/toml/should_support_array_of_inline_tables.toml Normal file
View file

@ -0,0 +1,5 @@
points = [ { x = 1, y = 2, z = 3 },
{ x = 7, y = 8, z = 9 },
{ x = 2, y = 4, z = 8 },
{ x = "3", y = "6", z = "12" }
]