Plexus-Mirror/toml4j
1
0
Fork 0
mirror of https://github.com/plexusorg/toml4j.git synced 2024-12-28 19:24:15 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'inline_tables' into wip

Browse source 
Conflicts:
	src/main/java/com/moandjiezana/toml/Results.java
	src/main/java/com/moandjiezana/toml/TomlParser.java
	src/test/java/com/moandjiezana/toml/ErrorMessagesTest.java
This commit is contained in:
moandji.ezana 2015-02-11 16:43:48 +02:00
commit c4027ed2d5
22 changed files with 796 additions and 378 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
CHANGELOG.md
View file

@ -4,9 +4,12 @@
### Changed
* Toml#getList(String) replaced Toml#getList(String, Class)
* __REAKING:__ Toml#getList(String) replaced Toml#getList(String, Class)
* Dropped dependency on Parboiled and its significant transitive dependencies
* Updated Gson to 2.3.1
### Added
* Line numbers included in error messages
### Added
@ -14,8 +17,8 @@
### Fixed
* Fixed short-form Unicode escapes
* Fixed exponent handling
* Short-form Unicode escape handling
* Exponent handling
## 0.3.1 / 2014-12-16
* Support for [TOML 0.3.1](https://github.com/toml-lang/toml/tree/v0.3.1) spec

159
src/main/java/com/moandjiezana/toml/ArrayConverter.java
View file

@ -1,6 +1,7 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverters.CONVERTERS;
import java.util.ArrayList;
import java.util.List;
@ -10,8 +11,6 @@ class ArrayConverter implements ValueConverter {
static final ArrayConverter ARRAY_PARSER = new ArrayConverter();
private static final ValueConverters VALUE_CONVERTERS = new ValueConverters();
@Override
public boolean canConvert(String s) {
return s.startsWith("[");
@ -19,135 +18,75 @@ class ArrayConverter implements ValueConverter {
@Override
public Object convert(String s) {
return convert(s, new AtomicInteger(1), true);
AtomicInteger sharedIndex = new AtomicInteger();
Object converted = convert(s, sharedIndex);
char[] chars = s.toCharArray();
for (int i = sharedIndex.incrementAndGet(); i < chars.length; i++) {
char c = chars[i];
if (c == '#') {
break;
}
if (!Character.isWhitespace(c)) {
return INVALID;
}
}
return converted;
}
public Object convert(String s, AtomicInteger sharedIndex, boolean topLevel) {
@Override
public Object convert(String s, AtomicInteger index) {
int startIndex = index.get();
char[] chars = s.toCharArray();
List<Object> arrayItems = new ArrayList<Object>();
boolean terminated = false;
StringType stringType = StringType.NONE;
StringBuilder current = new StringBuilder();
boolean inComment = false;
for (int i = 1; i < chars.length; i++, sharedIndex.incrementAndGet()) {
for (int i = index.incrementAndGet(); i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (terminated && !topLevel) {
break;
}
if (terminated) {
if (c == '#') {
break;
}
if (!Character.isWhitespace(c)) {
return INVALID;
}
continue;
}
if (stringType == StringType.NONE) {
if (c == ',') {
if (current.toString().trim().length() > 0) {
arrayItems.add(current.toString());
}
current = new StringBuilder();
continue;
}
if (c == '[') {
arrayItems.add(convert(s.substring(i), sharedIndex, false));
i = sharedIndex.get();
continue;
}
if (c == ']') {
terminated = true;
if (current.toString().trim().length() > 0) {
arrayItems.add(current.toString());
}
current = new StringBuilder();
continue;
}
}
if (c == '"' && chars[i - 1] != '\\' && !stringType.accepts(c)) {
if (chars.length > i + 2 && chars[i + 1] == c && chars[i + 2] == c) {
stringType = stringType.flip(StringType.MULTILINE);
} else {
stringType = stringType.flip(StringType.BASIC);
}
}
if (c == '\'' && !stringType.accepts(c)) {
if (chars.length > i + 2 && chars[i + 1] == c && chars[i + 2] == c) {
stringType = stringType.flip(StringType.MULTILINE_LITERAL);
} else {
stringType = stringType.flip(StringType.LITERAL);
}
if (c == '#' && !inComment) {
inComment = true;
} else if (c == '\n') {
inComment = false;
} else if (inComment || Character.isWhitespace(c) || c == ',') {
continue;
} else if (c == '[') {
arrayItems.add(convert(s, index));
continue;
} else if (c == ']') {
terminated = true;
break;
} else {
arrayItems.add(CONVERTERS.convert(s, index));
}
current.append(c);
}
if (!terminated) {
return INVALID;
return ValueConverterUtils.unterminated(s.substring(startIndex, s.length()));
}
return convertList(arrayItems);
}
private Object convertList(List<Object> tokens) {
ArrayList<Object> nestedList = new ArrayList<Object>();
for (Object token : tokens) {
if (token instanceof String) {
Object converted = VALUE_CONVERTERS.convert(((String) token).trim());
if (converted == INVALID) {
return INVALID;
}
if (isHomogenousArray(converted, nestedList)) {
nestedList.add(converted);
} else {
return INVALID;
}
} else if (token instanceof List) {
@SuppressWarnings("unchecked")
List<Object> convertedList = (List<Object>) token;
if (isHomogenousArray(convertedList, nestedList)) {
nestedList.add(convertedList);
} else {
return INVALID;
}
for (Object arrayItem : arrayItems) {
if (arrayItem == INVALID) {
return INVALID;
}
if (!isHomogenousArray(arrayItem, arrayItems)) {
return INVALID;
}
}
return nestedList;
return arrayItems;
}
private boolean isHomogenousArray(Object o, List<?> values) {
return values.isEmpty() || values.get(0).getClass().isAssignableFrom(o.getClass()) || o.getClass().isAssignableFrom(values.get(0).getClass());
}
private static enum StringType {
NONE, BASIC, LITERAL, MULTILINE, MULTILINE_LITERAL;
StringType flip(StringType to) {
return this == NONE ? to : NONE;
}
boolean accepts(char c) {
if (this == BASIC || this == MULTILINE) {
return c != '"';
}
if (this == LITERAL || this == MULTILINE_LITERAL) {
return c != '\'';
}
return false;
}
}
private ArrayConverter() {}
}

20
src/main/java/com/moandjiezana/toml/BooleanConverter.java
View file

@ -1,6 +1,9 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.util.concurrent.atomic.AtomicInteger;
class BooleanConverter implements ValueConverter {
@ -14,13 +17,24 @@ class BooleanConverter implements ValueConverter {
@Override
public Object convert(String s) {
AtomicInteger index = new AtomicInteger();
Object converted = convert(s, index);
if (!isComment(s.substring(index.incrementAndGet()))) {
return INVALID;
}
return converted;
}
@Override
public Object convert(String s, AtomicInteger index) {
s = s.substring(index.get());
Boolean b = s.startsWith("true") ? Boolean.TRUE : Boolean.FALSE;
int endIndex = b == Boolean.TRUE ? 4 : 5;
if (!ValueConverterUtils.isComment(s.substring(endIndex))) {
return INVALID;
}
index.addAndGet(endIndex - 1);
return b;
}

73
src/main/java/com/moandjiezana/toml/DateConverter.java
View file

@ -1,8 +1,10 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.text.SimpleDateFormat;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -13,15 +15,36 @@ class DateConverter implements ValueConverter {
@Override
public boolean canConvert(String s) {
Matcher matcher = DATE_REGEX.matcher(s);
if (s.length() < 5) {
return false;
}
char[] chars = s.toCharArray();
return matcher.matches() && ValueConverterUtils.isComment(matcher.group(4));
for (int i = 0; i < 5; i++) {
char c = chars[i];
if (i < 4) {
if (!Character.isDigit(c)) {
return false;
}
} else if (c != '-') {
return false;
}
}
return true;
}
@Override
public Object convert(String s) {
Matcher matcher = DATE_REGEX.matcher(s);
matcher.matches();
if (!isComment(matcher.group(4))) {
return INVALID;
}
s = matcher.group(1);
String zone = matcher.group(3);
String fractionalSeconds = matcher.group(2);
@ -36,6 +59,7 @@ class DateConverter implements ValueConverter {
} else if (zone.contains(":")) {
s += zone.replace(":", "");
}
try {
SimpleDateFormat dateFormat = new SimpleDateFormat(format);
dateFormat.setLenient(false);
@ -44,6 +68,51 @@ class DateConverter implements ValueConverter {
return INVALID;
}
}
@Override
public Object convert(String original, AtomicInteger index) {
StringBuilder sb = new StringBuilder();
for (int i = index.get(); i < original.length(); i = index.incrementAndGet()) {
char c = original.charAt(i);
if (Character.isDigit(c) || c == '-' || c == ':' || c == '.' || c == 'T' || c == 'Z') {
sb.append(c);
} else {
index.decrementAndGet();
break;
}
}
String s = sb.toString();
Matcher matcher = DATE_REGEX.matcher(s);
if (!matcher.matches()) {
return INVALID;
}
String dateString = matcher.group(1);
String zone = matcher.group(3);
String fractionalSeconds = matcher.group(2);
String format = "yyyy-MM-dd'T'HH:mm:ss";
if (fractionalSeconds != null && !fractionalSeconds.isEmpty()) {
format += ".SSS";
dateString += fractionalSeconds;
}
format += "Z";
if ("Z".equals(zone)) {
dateString += "+0000";
} else if (zone.contains(":")) {
dateString += zone.replace(":", "");
}
try {
SimpleDateFormat dateFormat = new SimpleDateFormat(format);
dateFormat.setLenient(false);
return dateFormat.parse(dateString);
} catch (Exception e) {
return INVALID;
}
}
private DateConverter() {}
}

60
src/main/java/com/moandjiezana/toml/Identifier.java Normal file
View file

@ -0,0 +1,60 @@
package com.moandjiezana.toml;
class Identifier {
static final Identifier INVALID = new Identifier("");
private final String name;
private final Type type;
Identifier(String name) {
this.name = name;
if (name.startsWith("[[")) {
this.type = Type.TABLE_ARRAY;
} else if (name.startsWith("[")) {
this.type = Type.TABLE;
} else {
this.type = Type.KEY;
}
}
boolean acceptsNext(char c) {
if (isKey()) {
return c == '=';
}
return c == '\n' || c == '#';
}
String getName() {
return name;
}
boolean isKey() {
return type == Type.KEY;
}
boolean isTable() {
return type == Type.TABLE;
}
boolean isTableArray() {
return type == Type.TABLE_ARRAY;
}
boolean isValid() {
if (isKey()) {
return Keys.getKey(name) != null;
}
if (isTable()) {
return Keys.getTableName(name) != null;
}
return Keys.getTableArrayName(name) != null;
}
private static enum Type {
KEY, TABLE, TABLE_ARRAY;
}
}

33
src/main/java/com/moandjiezana/toml/IdentifierConverter.java Normal file
View file

@ -0,0 +1,33 @@
package com.moandjiezana.toml;
import java.util.concurrent.atomic.AtomicInteger;
class IdentifierConverter {
static final IdentifierConverter IDENTIFIER_CONVERTER = new IdentifierConverter();
Identifier convert(char[] chars, AtomicInteger index) {
boolean quoted = false;
StringBuilder name = new StringBuilder();
Identifier identifier = null;
for (int i = index.get(); i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (c == '"' && (i == 0 || chars[i - 1] != '\\')) {
quoted = !quoted;
name.append('"');
} else if (c == '\n' || (!quoted && (c == '#' || c == '='))) {
return new Identifier(name.toString().trim());
} else if (i == chars.length - 1 && identifier == null) {
name.append(c);
return new Identifier(name.toString().trim());
} else {
name.append(c);
}
}
return identifier != null ? identifier : Identifier.INVALID;
}
private IdentifierConverter() {}
}

95
src/main/java/com/moandjiezana/toml/InlineTableConverter.java Normal file
View file

@ -0,0 +1,95 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverters.CONVERTERS;
import java.util.HashMap;
import java.util.concurrent.atomic.AtomicInteger;
class InlineTableConverter implements ValueConverter {
static final InlineTableConverter INLINE_TABLE_PARSER = new InlineTableConverter();
@Override
public boolean canConvert(String s) {
return s.startsWith("{");
}
@Override
public Object convert(String s) {
AtomicInteger index = new AtomicInteger();
Object converted = convert(s, index);
String substring = s.substring(index.incrementAndGet());
if (converted == INVALID || !ValueConverterUtils.isComment(substring)) {
return INVALID;
}
return converted;
}
@Override
public Object convert(String s, AtomicInteger sharedIndex) {
char[] chars = s.toCharArray();
boolean inKey = true;
boolean inValue = false;
boolean quoted = false;
boolean terminated = false;
StringBuilder currentKey = new StringBuilder();
HashMap<String, Object> results = new HashMap<String, Object>();
for (int i = sharedIndex.incrementAndGet(); sharedIndex.get() < chars.length; i = sharedIndex.incrementAndGet()) {
char c = chars[i];
if (c == '"' && inKey) {
quoted = !quoted;
currentKey.append(c);
} else if (quoted) {
currentKey.append(c);
} else if (inValue && !Character.isWhitespace(c)) {
Object converted = CONVERTERS.convert(s, sharedIndex);
if (converted == INVALID) {
return INVALID;
}
results.put(currentKey.toString().trim(), converted);
currentKey = new StringBuilder();
inValue = false;
} else if (c == '{') {
sharedIndex.incrementAndGet();
Object converted = convert(s, sharedIndex);
if (converted == INVALID) {
return INVALID;
}
results.put(currentKey.toString().trim(), converted);
inKey = true;
inValue = false;
currentKey = new StringBuilder();
} else if (c == ',') {
inKey = true;
inValue = false;
currentKey = new StringBuilder();
} else if (c == '=') {
inKey = false;
inValue = true;
} else if (c == '}') {
terminated = true;
break;
} else if (inKey) {
currentKey.append(c);
}
}
if (!terminated) {
return INVALID;
}
return results;
}
private InlineTableConverter() {}
}

52
src/main/java/com/moandjiezana/toml/LiteralStringConverter.java
View file

@ -1,6 +1,9 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.util.concurrent.atomic.AtomicInteger;
class LiteralStringConverter implements ValueConverter {
@ -14,31 +17,38 @@ class LiteralStringConverter implements ValueConverter {
@Override
public Object convert(String s) {
char[] chars = s.toCharArray();
int endIndex = -1;
AtomicInteger index = new AtomicInteger();
Object converted = convert(s, index);
for (int i = 1; i < chars.length; i++) {
char c = chars[i];
if (c == '\'') {
endIndex = i;
continue;
}
if (endIndex > -1 && c == '#') {
break;
}
if (endIndex > -1 && !Character.isWhitespace(c)) {
return INVALID;
}
}
if (endIndex == -1) {
if (converted == INVALID || !isComment(s.substring(index.incrementAndGet()))) {
return INVALID;
}
return s.substring(1, endIndex);
return converted;
}
@Override
public Object convert(String s, AtomicInteger index) {
char[] chars = s.toCharArray();
boolean terminated = false;
int startIndex = index.incrementAndGet();
for (int i = index.get(); i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (c == '\'') {
terminated = true;
break;
}
}
if (!terminated) {
return INVALID;
}
String substring = s.substring(startIndex, index.get());
return substring;
}
private LiteralStringConverter() {}

39
src/main/java/com/moandjiezana/toml/MultilineLiteralStringConverter.java
View file

@ -1,6 +1,9 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.util.concurrent.atomic.AtomicInteger;
class MultilineLiteralStringConverter implements ValueConverter {
@ -13,28 +16,42 @@ class MultilineLiteralStringConverter implements ValueConverter {
@Override
public Object convert(String s) {
AtomicInteger index = new AtomicInteger();
Object converted = convert(s, index);
if (converted == INVALID || !isComment(s.substring(index.incrementAndGet()))) {
return INVALID;
}
return converted;
}
@Override
public Object convert(String s, AtomicInteger index) {
char[] chars = s.toCharArray();
int originalStartIndex = index.get();
int startIndex = index.addAndGet(3);
int endIndex = -1;
for (int i = 3; i < chars.length; i++) {
if (chars[startIndex] == '\n') {
startIndex = index.incrementAndGet();
}
for (int i = startIndex; i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (c == '\'' && chars.length > i + 2 && chars[i + 1] == '\'' && chars[i + 2] == '\'') {
endIndex = i;
i += 2;
continue;
}
if (endIndex > -1 && c == '#') {
index.addAndGet(2);
break;
}
if (endIndex > -1 && !Character.isWhitespace(c)) {
return INVALID;
}
}
if (endIndex == -1) {
return ValueConverterUtils.unterminated(s.substring(originalStartIndex, s.length()));
}
return s.substring(3, endIndex);
return s.substring(startIndex, endIndex);
}
private MultilineLiteralStringConverter() {}

62
src/main/java/com/moandjiezana/toml/MultilineStringConverter.java
View file

@ -1,9 +1,13 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import static com.moandjiezana.toml.ValueConverterUtils.unterminated;
import java.util.concurrent.atomic.AtomicInteger;
class MultilineStringConverter implements ValueConverter {
static final MultilineStringConverter MULTILINE_STRING_PARSER = new MultilineStringConverter();
@Override
@ -13,22 +17,50 @@ class MultilineStringConverter implements ValueConverter {
@Override
public Object convert(String s) {
int terminator = s.indexOf("\"\"\"", 3);
if (terminator == -1) {
AtomicInteger index = new AtomicInteger();
Object converted = convert(s, index);
if (converted == INVALID || !isComment(s.substring(index.incrementAndGet()))) {
return INVALID;
}
if (!ValueConverterUtils.isComment(s.substring(terminator + 3))) {
return INVALID;
}
s = s.substring(2, terminator + 1);
s = s.replaceAll("\\\\\\s+", "");
return StringConverter.STRING_PARSER.convert(s);
return converted;
}
@Override
public Object convert(String s, AtomicInteger index) {
char[] chars = s.toCharArray();
int originalStartIndex = index.get();
int startIndex = index.addAndGet(3);
int endIndex = -1;
if (chars[startIndex] == '\n') {
startIndex = index.incrementAndGet();
}
for (int i = startIndex; i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (c == '"' && chars.length > i + 2 && chars[i + 1] == '"' && chars[i + 2] == '"') {
endIndex = i;
index.addAndGet(2);
break;
}
}
if (endIndex == -1) {
return unterminated(s.substring(originalStartIndex, s.length()));
}
s = s.substring(startIndex, endIndex);
s = s.replaceAll("\\\\\\s+", "");
s = StringConverter.STRING_PARSER.replaceUnicodeCharacters(s);
s = StringConverter.STRING_PARSER.replaceSpecialCharacters(s);
return s;
}
private MultilineStringConverter() {
}
private MultilineStringConverter() {}
}

32
src/main/java/com/moandjiezana/toml/NumberConverter.java
View file

@ -1,6 +1,9 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.util.concurrent.atomic.AtomicInteger;
class NumberConverter implements ValueConverter {
static final NumberConverter NUMBER_PARSER = new NumberConverter();
@ -14,20 +17,33 @@ class NumberConverter implements ValueConverter {
@Override
public Object convert(String s) {
AtomicInteger index = new AtomicInteger();
Object converted = convert(s, index);
if (converted == INVALID || (s.length() > index.get() + 1 && !isComment(s.substring(index.incrementAndGet())))) {
return INVALID;
}
return converted;
}
@Override
public Object convert(String s, AtomicInteger index) {
char[] chars = s.toCharArray();
boolean whitespace = false;
boolean signable = true;
boolean dottable = false;
boolean exponentable = false;
boolean terminatable = false;
String type = "";
StringBuilder sb = new StringBuilder();
for (int i = 0; i < chars.length; i++) {
for (int i = index.get(); i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (Character.isDigit(c)) {
sb.append(c);
signable = false;
terminatable = true;
if (type.isEmpty()) {
type = "integer";
dottable = true;
@ -35,26 +51,28 @@ class NumberConverter implements ValueConverter {
exponentable = !type.equals("exponent");
} else if ((c == '+' || c == '-') && signable && chars.length > i + 1) {
signable = false;
terminatable = false;
if (c == '-') {
sb.append('-');
}
} else if (c == '.' && dottable && chars.length > i + 1) {
sb.append('.');
type = "float";
terminatable = false;
dottable = false;
exponentable = false;
} else if ((c == 'E' || c == 'e') && exponentable && chars.length > i + 1) {
sb.append('E');
type = "exponent";
terminatable = false;
signable = true;
dottable = false;
exponentable = false;
} else if (Character.isWhitespace(c)) {
whitespace = true;
} else if (whitespace && c == '#') {
break;
} else {
type = "";
if (!terminatable) {
type = "";
}
index.decrementAndGet();
break;
}
}

47
src/main/java/com/moandjiezana/toml/Results.java
View file

@ -50,6 +50,33 @@ class Results {
.append('\n');
}
void invalidIdentifier(Identifier identifier, int line) {
if (identifier.isKey()) {
invalidKey(identifier.getName(), line);
} else if (identifier.isTable()) {
invalidTable(identifier.getName(), line);
} else if (identifier.isTableArray()) {
invalidTableArray(identifier.getName(), line);
}
}
void invalidTextAfterIdentifier(Identifier identifier, char text, int line) {
if (identifier.isKey() && text == '\n') {
sb.append("Key ")
.append(identifier.getName())
.append(" is not followed by an equals sign on line ")
.append(line)
.append('\n');
} else {
sb.append("Invalid text after key ")
.append(identifier.getName())
.append(" on line ")
.append(line)
.append(". Make sure to terminate the value or add a comment (#).")
.append('\n');
}
}
void invalidKey(String key, int line) {
sb.append("Invalid key");
if (line > -1) {
@ -79,13 +106,13 @@ class Results {
.append('\n');
}
void unterminated(String key, String multiline, int line) {
sb.append("Unterminated multiline value on line ")
void unterminated(String key, String value, int line) {
sb.append("Unterminated value on line ")
.append(line)
.append(": ")
.append(key)
.append(" = ")
.append(multiline.trim())
.append(value.trim())
.append('\n');
}
@ -98,8 +125,7 @@ class Results {
return sb.toString();
}
}
Set<String> tables = new HashSet<String>();
Set<String> tables = new HashSet<String>();
final Errors errors = new Errors();
private Deque<Container> stack = new ArrayDeque<Container>();
@ -109,7 +135,16 @@ class Results {
void addValue(String key, Object value) {
Container currentTable = stack.peek();
if (currentTable.accepts(key)) {
if (value instanceof Map) {
startTable(key);
@SuppressWarnings("unchecked")
Map<String, Object> valueMap = (Map<String, Object>) value;
for (Map.Entry<String, Object> entry : valueMap.entrySet()) {
addValue(entry.getKey(), entry.getValue());
}
stack.pop();
} else if (currentTable.accepts(key)) {
currentTable.put(key, value);
} else {
errors.duplicateKey(key, -1);

24
src/main/java/com/moandjiezana/toml/StringConverter.java
View file

@ -3,6 +3,7 @@ package com.moandjiezana.toml;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -18,22 +19,35 @@ class StringConverter implements ValueConverter {
@Override
public Object convert(String value) {
int stringTerminator = -1;
AtomicInteger index = new AtomicInteger();
Object converted = convert(value, index);
if (converted == INVALID || !isComment(value.substring(index.incrementAndGet()))) {
return INVALID;
}
return converted;
}
@Override
public Object convert(String value, AtomicInteger sharedIndex) {
int startIndex = sharedIndex.incrementAndGet();
int endIndex = -1;
char[] chars = value.toCharArray();
for (int i = 1; i < chars.length; i++) {
for (int i = sharedIndex.get(); i < chars.length; i = sharedIndex.incrementAndGet()) {
char ch = chars[i];
if (ch == '"' && chars[i - 1] != '\\') {
stringTerminator = i;
endIndex = i;
break;
}
}
if (stringTerminator == -1 || !isComment(value.substring(stringTerminator + 1))) {
if (endIndex == -1) {
return INVALID;
}
value = value.substring(1, stringTerminator);
value = value.substring(startIndex, endIndex);
value = replaceUnicodeCharacters(value);
value = replaceSpecialCharacters(value);

239
src/main/java/com/moandjiezana/toml/TomlParser.java
View file

@ -1,202 +1,73 @@
package com.moandjiezana.toml;
import static com.moandjiezana.toml.IdentifierConverter.IDENTIFIER_CONVERTER;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import static com.moandjiezana.toml.ValueConverterUtils.isComment;
import java.util.regex.Pattern;
import java.util.concurrent.atomic.AtomicInteger;
import com.moandjiezana.toml.ValueConverterUtils.Unterminated;
class TomlParser {
private static final String STRING_LITERAL_DELIMITER = "'''";
private static final Pattern MULTILINE_ARRAY_REGEX = Pattern.compile("\\s*\\[([^\\]]*)");
private static final Pattern MULTILINE_ARRAY_REGEX_END = Pattern.compile("\\s*\\]");
private static final ValueConverters VALUE_ANALYSIS = new ValueConverters();
private final Results results = new Results();
Results run(String tomlString) {
final Results results = new Results();
if (tomlString.isEmpty()) {
return results;
}
String[] lines = tomlString.split("[\\n\\r]");
int lastKeyLine = 1;
StringBuilder multilineBuilder = new StringBuilder();
Multiline multiline = Multiline.NONE;
String key = null;
String value = null;
for (int i = 0; i < lines.length; i++) {
String line = lines[i];
if (line != null && multiline.isTrimmable()) {
line = line.trim();
}
if (isComment(line) || line.isEmpty()) {
continue;
}
if (isTableArray(line)) {
String tableName = Keys.getTableArrayName(line);
if (tableName != null) {
results.startTableArray(tableName);
} else {
results.errors.invalidTableArray(line, i + 1);
}
continue;
}
if (multiline.isNotMultiline() && isTable(line)) {
String tableName = Keys.getTableName(line);
if (tableName != null) {
results.startTables(tableName);
} else {
results.errors.invalidTable(line.trim(), i + 1);
}
continue;
}
if (multiline.isNotMultiline() && !line.contains("=")) {
results.errors.invalidKey(line, i + 1);
continue;
}
String[] pair = line.split("=", 2);
if (multiline.isNotMultiline() && MULTILINE_ARRAY_REGEX.matcher(pair[1].trim()).matches()) {
multiline = Multiline.ARRAY;
key = pair[0].trim();
multilineBuilder.append(removeComment(pair[1]));
continue;
}
if (multiline.isNotMultiline() && pair[1].trim().startsWith("\"\"\"")) {
multiline = Multiline.STRING;
multilineBuilder.append(pair[1]);
key = pair[0].trim();
if (pair[1].trim().indexOf("\"\"\"", 3) > -1) {
multiline = Multiline.NONE;
pair[1] = multilineBuilder.toString().trim();
multilineBuilder.delete(0, multilineBuilder.length());
} else {
if (multilineBuilder.toString().trim().length() > 3) {
multilineBuilder.append('\n');
}
continue;
}
}
if (multiline.isNotMultiline() && pair[1].trim().startsWith(STRING_LITERAL_DELIMITER)) {
multiline = Multiline.STRING_LITERAL;
multilineBuilder.append(pair[1]);
key = pair[0].trim();
if (pair[1].trim().indexOf(STRING_LITERAL_DELIMITER, 3) > -1) {
multiline = Multiline.NONE;
pair[1] = multilineBuilder.toString().trim();
multilineBuilder.delete(0, multilineBuilder.length());
} else {
if (multilineBuilder.toString().trim().length() > 3) {
multilineBuilder.append('\n');
}
continue;
}
}
if (multiline == Multiline.ARRAY) {
String lineWithoutComment = removeComment(line);
multilineBuilder.append(lineWithoutComment);
if (MULTILINE_ARRAY_REGEX_END.matcher(lineWithoutComment).matches()) {
multiline = Multiline.NONE;
value = multilineBuilder.toString();
multilineBuilder.delete(0, multilineBuilder.length());
} else {
continue;
}
} else if (multiline == Multiline.STRING) {
multilineBuilder.append(line);
if (line.contains("\"\"\"")) {
multiline = Multiline.NONE;
value = multilineBuilder.toString().trim();
multilineBuilder.delete(0, multilineBuilder.length());
} else {
multilineBuilder.append('\n');
continue;
}
} else if (multiline == Multiline.STRING_LITERAL) {
multilineBuilder.append(line);
if (line.contains(STRING_LITERAL_DELIMITER)) {
multiline = Multiline.NONE;
value = multilineBuilder.toString().trim();
multilineBuilder.delete(0, multilineBuilder.length());
} else {
multilineBuilder.append('\n');
continue;
}
} else {
key = Keys.getKey(pair[0]);
if (key == null) {
results.errors.invalidKey(pair[0], i + 1);
continue;
}
value = pair[1].trim();
}
lastKeyLine = i + 1;
Object convertedValue = VALUE_ANALYSIS.convert(value);
if (convertedValue != INVALID) {
results.addValue(key, convertedValue);
} else {
results.errors.invalidValue(key, value, i + 1);
}
}
char[] chars = tomlString.toCharArray();
AtomicInteger index = new AtomicInteger();
boolean inComment = false;
AtomicInteger line = new AtomicInteger(1);
Identifier identifier = null;
Object value = null;
if (multiline != Multiline.NONE) {
results.errors.unterminated(key, multilineBuilder.toString().trim(), lastKeyLine);
for (int i = index.get(); i < chars.length; i = index.incrementAndGet()) {
char c = chars[i];
if (c == '#' && !inComment) {
inComment = true;
} else if (!Character.isWhitespace(c) && !inComment && identifier == null) {
Identifier id = IDENTIFIER_CONVERTER.convert(chars, index);
if (id.isValid()) {
char next = chars[index.get()];
if (index.get() < chars.length -1 && !id.acceptsNext(next)) {
results.errors.invalidTextAfterIdentifier(id, next, line.get());
} else if (id.isKey()) {
identifier = id;
} else if (id.isTable()) {
results.startTables(Keys.getTableName(id.getName()));
} else if (id.isTableArray()) {
results.startTableArray(Keys.getTableArrayName(id.getName()));
}
inComment = next == '#';
} else {
results.errors.invalidIdentifier(id, line.get());
}
} else if (c == '\n') {
inComment = false;
identifier = null;
value = null;
line.incrementAndGet();
} else if (!inComment && identifier != null && identifier.isKey() && value == null && !Character.isWhitespace(c)) {
int startIndex = index.get();
Object converted = ValueConverters.CONVERTERS.convert(tomlString, index);
value = converted;
if (converted == INVALID) {
results.errors.invalidValue(identifier.getName(), tomlString.substring(startIndex, Math.min(index.get(), tomlString.length() - 1)), line.get());
} else if (converted instanceof Unterminated) {
results.errors.unterminated(identifier.getName(), ((Unterminated) converted).payload, line.get());
} else {
results.addValue(identifier.getName(), converted);
}
} else if (value != null && !inComment && !Character.isWhitespace(c)) {
results.errors.invalidTextAfterIdentifier(identifier, c, line.get());
}
}
return results;
}
private boolean isTableArray(String line) {
return line.startsWith("[[");
}
private boolean isTable(String line) {
return line.startsWith("[");
}
private String removeComment(String line) {
line = line.trim();
if (line.startsWith("\"")) {
int startOfComment = line.indexOf('#', line.lastIndexOf('"'));
if (startOfComment > -1) {
return line.substring(0, startOfComment - 1).trim();
}
} else {
int startOfComment = line.indexOf('#');
if (startOfComment > -1) {
return line.substring(0, startOfComment - 1).trim();
}
}
return line;
}
private static enum Multiline {
NONE, ARRAY, STRING, STRING_LITERAL;
public boolean isNotMultiline() {
return this == NONE;
}
public boolean isTrimmable() {
return this == NONE || this == ARRAY;
}
}
}

12
src/main/java/com/moandjiezana/toml/ValueConverter.java
View file

@ -1,5 +1,7 @@
package com.moandjiezana.toml;
import java.util.concurrent.atomic.AtomicInteger;
interface ValueConverter {
/**
@ -9,6 +11,16 @@ interface ValueConverter {
/**
* @param s must already have been validated by {@link #canConvert(String)}
* @return a value or {@link ValueConverterUtils#INVALID}
*/
Object convert(String s);
/**
* Partial validation. Stops after type terminator, rather than at EOI.
*
* @param s must already have been validated by {@link #canConvert(String)}
* @param index where to start in s
* @return a value or {@link ValueConverterUtils#INVALID}
*/
Object convert(String s, AtomicInteger index);
}

12
src/main/java/com/moandjiezana/toml/ValueConverterUtils.java
View file

@ -4,6 +4,18 @@ package com.moandjiezana.toml;
class ValueConverterUtils {
static final Object INVALID = new Object();
static Unterminated unterminated(String payload) {
return new Unterminated(payload);
}
static class Unterminated {
final String payload;
private Unterminated(String payload) {
this.payload = payload;
}
}
static boolean isComment(String line) {
if (line == null || line.isEmpty()) {
return true;

16
src/main/java/com/moandjiezana/toml/ValueConverters.java
View file

@ -3,6 +3,7 @@ package com.moandjiezana.toml;
import static com.moandjiezana.toml.ArrayConverter.ARRAY_PARSER;
import static com.moandjiezana.toml.BooleanConverter.BOOLEAN_PARSER;
import static com.moandjiezana.toml.DateConverter.DATE_PARSER;
import static com.moandjiezana.toml.InlineTableConverter.INLINE_TABLE_PARSER;
import static com.moandjiezana.toml.LiteralStringConverter.LITERAL_STRING_PARSER;
import static com.moandjiezana.toml.MultilineLiteralStringConverter.MULTILINE_LITERAL_STRING_CONVERTER;
import static com.moandjiezana.toml.MultilineStringConverter.MULTILINE_STRING_PARSER;
@ -10,19 +11,26 @@ import static com.moandjiezana.toml.NumberConverter.NUMBER_PARSER;
import static com.moandjiezana.toml.StringConverter.STRING_PARSER;
import static com.moandjiezana.toml.ValueConverterUtils.INVALID;
import java.util.concurrent.atomic.AtomicInteger;
class ValueConverters {
static final ValueConverters CONVERTERS = new ValueConverters();
private static final ValueConverter[] PARSERS = {
MULTILINE_STRING_PARSER, MULTILINE_LITERAL_STRING_CONVERTER, LITERAL_STRING_PARSER, STRING_PARSER, DATE_PARSER, NUMBER_PARSER, BOOLEAN_PARSER, ARRAY_PARSER
MULTILINE_STRING_PARSER, MULTILINE_LITERAL_STRING_CONVERTER, LITERAL_STRING_PARSER, STRING_PARSER, DATE_PARSER, NUMBER_PARSER, BOOLEAN_PARSER, ARRAY_PARSER, INLINE_TABLE_PARSER
};
public Object convert(String value) {
Object convert(String value, AtomicInteger index) {
String substring = value.substring(index.get());
for (ValueConverter valueParser : PARSERS) {
if (valueParser.canConvert(value)) {
return valueParser.convert(value);
if (valueParser.canConvert(substring)) {
return valueParser.convert(value, index);
}
}
return INVALID;
}
private ValueConverters() {}
}

31
src/test/java/com/moandjiezana/toml/ArrayTest.java
View file

@ -2,11 +2,13 @@ package com.moandjiezana.toml;
import static java.util.Arrays.asList;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.util.Arrays;
import java.util.List;
import org.junit.Test;
@ -36,6 +38,16 @@ public class ArrayTest {
assertEquals(asList(asList("gamma", "delta"), asList(1L, 2L)), clients.<String>getList("data"));
}
@Test
public void should_get_deeply_nested_arrays() throws Exception {
List<List<?>> data = new Toml().parse("data = [[[1], [2]], [3, 4]]").getList("data");
assertThat(data, hasSize(2));
assertEquals(Arrays.asList(1L), data.get(0).get(0));
assertEquals(asList(2L), data.get(0).get(1));
assertEquals(asList(3L, 4L), data.get(1));
}
@Test
@SuppressWarnings("unchecked")
@ -73,6 +85,25 @@ public class ArrayTest {
assertThat(toml.<String>getList("key"), contains("a]", "b]", "c]", "d]"));
}
@Test
public void should_support_array_of_inline_tables() throws Exception {
Toml toml = new Toml().parse(getClass().getResourceAsStream("should_support_array_of_inline_tables.toml"));
assertThat(toml.getList("points"), hasSize(4));
assertEquals(1, toml.getLong("points[0].x").longValue());
assertEquals(2, toml.getLong("points[0].y").longValue());
assertEquals(3, toml.getLong("points[0].z").longValue());
assertEquals(7, toml.getLong("points[1].x").longValue());
assertEquals(8, toml.getLong("points[1].y").longValue());
assertEquals(9, toml.getLong("points[1].z").longValue());
assertEquals(2, toml.getLong("points[2].x").longValue());
assertEquals(4, toml.getLong("points[2].y").longValue());
assertEquals(8, toml.getLong("points[2].z").longValue());
assertEquals("3", toml.getString("points[3].x"));
assertEquals("6", toml.getString("points[3].y"));
assertEquals("12", toml.getString("points[3].z"));
}
private File file(String file) {
return Utils.file(getClass(), file);
}

2
src/test/java/com/moandjiezana/toml/BareKeysTest.java
View file

@ -2,9 +2,7 @@ package com.moandjiezana.toml;
import static org.junit.Assert.assertEquals;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
public class BareKeysTest {

22
src/test/java/com/moandjiezana/toml/ErrorMessagesTest.java
View file

@ -53,21 +53,35 @@ public class ErrorMessagesTest {
@Test
public void should_message_invalid_value() throws Exception {
e.expectMessage("Invalid value on line 1: k = 1 t");
e.expectMessage("Invalid text after key k on line 1");
new Toml().parse("k = 1 t");
}
@Test
public void should_message_unterminated_value() throws Exception {
e.expectMessage("Unterminated multiline value on line 1: k = '''abc");
public void should_message_unterminated_multiline_literal_string() throws Exception {
e.expectMessage("Unterminated value on line 1: k = '''abc");
new Toml().parse("k = '''abc");
}
@Test
public void should_message_unterminated_multiline_string() throws Exception {
e.expectMessage("Unterminated value on line 1: k = \"\"\"abc\"\"");
new Toml().parse("k = \"\"\"abc\"\"");
}
@Test
public void should_message_unterminated_array() throws Exception {
e.expectMessage("Unterminated value on line 1: k = [\"abc\"");
new Toml().parse("k = [\"abc\"");
}
@Test
public void should_message_key_without_equals() throws Exception {
e.expectMessage("Invalid key on line 2: k");
e.expectMessage("Key k is not followed by an equals sign on line 2");
new Toml().parse("\nk\n=3");
}

128
src/test/java/com/moandjiezana/toml/InlineTableTest.java Normal file
View file

@ -0,0 +1,128 @@
package com.moandjiezana.toml;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import java.util.Calendar;
import java.util.List;
import java.util.TimeZone;
import org.junit.Test;
public class InlineTableTest {
private static final TimeZone UTC = TimeZone.getTimeZone("UTC");
@Test
public void should_read_empty_inline_table() throws Exception {
Toml toml = new Toml().parse("key = {}");
assertNotNull(toml.getTable("key"));
}
@Test
public void should_read_inline_table_with_strings() throws Exception {
Toml toml = new Toml().parse("name = { first = \"Tom\", last = \"Preston-Werner\"}");
assertEquals("Tom", toml.getTable("name").getString("first"));
assertEquals("Preston-Werner", toml.getString("name.last"));
}
@Test
public void should_read_inline_table_with_integers() throws Exception {
Toml toml = new Toml().parse("point = { x = 1, y = 2 }");
assertEquals(1, toml.getTable("point").getLong("x").longValue());
assertEquals(2, toml.getLong("point.y").longValue());
}
@Test
public void should_read_inline_table_with_floats() throws Exception {
Toml toml = new Toml().parse("point = { x = 1.5, y = 2.3 }");
assertEquals(1.5, toml.getTable("point").getDouble("x").doubleValue(), 0);
assertEquals(2.3, toml.getDouble("point.y").doubleValue(), 0);
}
@Test
public void should_read_inline_table_with_booleans() throws Exception {
Toml toml = new Toml().parse("point = { x = false, y = true }");
assertTrue(toml.getTable("point").getBoolean("y"));
assertFalse(toml.getBoolean("point.x"));
}
@Test
public void should_read_inline_table_with_dates() throws Exception {
Toml toml = new Toml().parse("point = { x = 2015-02-09T22:05:00Z, y = 2015-02-09T21:05:00Z }");
Calendar x = Calendar.getInstance(UTC);
x.set(2015, Calendar.FEBRUARY, 9, 22, 5, 00);
x.set(Calendar.MILLISECOND, 0);
Calendar y = Calendar.getInstance(UTC);
y.set(2015, Calendar.FEBRUARY, 9, 21, 5, 00);
y.set(Calendar.MILLISECOND, 0);
assertEquals(x.getTime(), toml.getTable("point").getDate("x"));
assertEquals(y.getTime(), toml.getDate("point.y"));
}
@Test
public void should_read_arrays() throws Exception {
Toml toml = new Toml().parse("arrays = { integers = [1, 2, 3], strings = [\"a\", \"b\", \"c\"] }");
assertThat(toml.<Long>getList("arrays.integers"), contains(1L, 2L, 3L));
assertThat(toml.<String>getList("arrays.strings"), contains("a", "b", "c"));
}
@Test
public void should_read_nested_arrays() throws Exception {
Toml toml = new Toml().parse("arrays = { nested = [[1, 2, 3], [4, 5, 6]] }").getTable("arrays");
List<List<Long>> nested = toml.<List<Long>>getList("nested");
assertThat(nested, hasSize(2));
assertThat(nested.get(0), contains(1L, 2L, 3L));
assertThat(nested.get(1), contains(4L, 5L, 6L));
}
@Test
public void should_read_mixed_inline_table() throws Exception {
Toml toml = new Toml().parse("point = { date = 2015-02-09T22:05:00Z, bool = true, integer = 123, float = 123.456, string = \"abc\", list = [5, 6, 7, 8] }").getTable("point");
Calendar date = Calendar.getInstance(UTC);
date.set(2015, Calendar.FEBRUARY, 9, 22, 5, 00);
date.set(Calendar.MILLISECOND, 0);
assertEquals(date.getTime(), toml.getDate("date"));
assertTrue(toml.getBoolean("bool"));
assertEquals(123, toml.getLong("integer").intValue());
assertEquals(123.456, toml.getDouble("float"), 0);
assertEquals("abc", toml.getString("string"));
assertThat(toml.<Long>getList("list"), contains(5L, 6L, 7L, 8L));
}
@Test
public void should_read_nested_inline_tables() throws Exception {
Toml tables = new Toml().parse("tables = { t1 = { t1_1 = 1, t1_2 = 2}, t2 = { t2_1 = \"a\"} }").getTable("tables");
assertEquals(1L, tables.getLong("t1.t1_1").longValue());
assertEquals(2L, tables.getLong("t1.t1_2").longValue());
assertEquals("a", tables.getString("t2.t2_1"));
}
@Test
public void should_read_all_string_types() throws Exception {
Toml strings = new Toml().parse("strings = { literal = 'ab]\"c', multiline = \"\"\"de]\"f\"\"\", multiline_literal = '''gh]\"i''' }").getTable("strings");
assertEquals("ab]\"c", strings.getString("literal"));
assertEquals("de]\"f", strings.getString("multiline"));
assertEquals("gh]\"i", strings.getString("multiline_literal"));
}
}

5
src/test/resources/com/moandjiezana/toml/should_support_array_of_inline_tables.toml Normal file
View file

@ -0,0 +1,5 @@
points = [ { x = 1, y = 2, z = 3 },
{ x = 7, y = 8, z = 9 },
{ x = 2, y = 4, z = 8 },
{ x = "3", y = "6", z = "12" }
]