From 62ca97ae5bc3379a7048cbde149077da2b3bfdd4 Mon Sep 17 00:00:00 2001 From: zt515 Date: Wed, 2 Aug 2017 23:28:52 +0800 Subject: [PATCH] Feature: NeoLang supports quoted string --- NeoLang/example/color-scheme.nl | 9 + NeoLang/example/main.neo | 4 - .../java/io/neolang/ast/NeoLangTokenType.kt | 2 +- .../java/io/neolang/ast/NeoLangTokenValue.kt | 19 +- .../java/io/neolang/ast/base/NeoLangAst.kt | 6 +- .../java/io/neolang/ast/visitor/AstVisitor.kt | 12 ++ .../neolang/ast/visitor/IVisitorCallback.kt | 7 + .../neolang/ast/visitor/NeoLangAstVisitor.kt | 8 - .../io/neolang/ast/visitor/VisitorFactory.kt | 18 ++ .../java/io/neolang/main/DisplayAstVisitor.kt | 9 + .../java/io/neolang/{command => main}/Main.kt | 21 +- .../java/io/neolang/parser/NeoLangLexer.kt | 185 +++++++++++++++--- .../java/io/neolang/parser/NeoLangParser.kt | 15 +- app/build.gradle | 2 +- .../customize/service/NeoServiceManager.kt | 10 + app/src/test/java/io/neoterm/NeoLangTest.kt | 6 +- 16 files changed, 262 insertions(+), 71 deletions(-) create mode 100644 NeoLang/example/color-scheme.nl delete mode 100644 NeoLang/example/main.neo create mode 100644 NeoLang/src/main/java/io/neolang/ast/visitor/AstVisitor.kt create mode 100644 NeoLang/src/main/java/io/neolang/ast/visitor/IVisitorCallback.kt delete mode 100644 NeoLang/src/main/java/io/neolang/ast/visitor/NeoLangAstVisitor.kt create mode 100644 NeoLang/src/main/java/io/neolang/ast/visitor/VisitorFactory.kt create mode 100644 NeoLang/src/main/java/io/neolang/main/DisplayAstVisitor.kt rename NeoLang/src/main/java/io/neolang/{command => main}/Main.kt (62%) create mode 100644 app/src/main/java/io/neoterm/customize/service/NeoServiceManager.kt diff --git a/NeoLang/example/color-scheme.nl b/NeoLang/example/color-scheme.nl new file mode 100644 index 0000000..7eea051 --- /dev/null +++ b/NeoLang/example/color-scheme.nl @@ -0,0 +1,9 @@ +color-scheme: { + name: "NeoTerm Default Color Scheme" + version: 1.1 + + colors: { + background: #000000 + foreground: #212121 + } +} diff --git a/NeoLang/example/main.neo b/NeoLang/example/main.neo deleted file mode 100644 index 00f3c5c..0000000 --- a/NeoLang/example/main.neo +++ /dev/null @@ -1,4 +0,0 @@ -NeoLang { - version : 1 - x-letter : ${NeoLang@version} >= 2 -} diff --git a/NeoLang/src/main/java/io/neolang/ast/NeoLangTokenType.kt b/NeoLang/src/main/java/io/neolang/ast/NeoLangTokenType.kt index 75fb23f..7e9d414 100644 --- a/NeoLang/src/main/java/io/neolang/ast/NeoLangTokenType.kt +++ b/NeoLang/src/main/java/io/neolang/ast/NeoLangTokenType.kt @@ -6,11 +6,11 @@ package io.neolang.ast enum class NeoLangTokenType { NUMBER, + ID, STRING, BRACKET_START, BRACKET_END, COLON, - QUOTE, EOL, EOF, } diff --git a/NeoLang/src/main/java/io/neolang/ast/NeoLangTokenValue.kt b/NeoLang/src/main/java/io/neolang/ast/NeoLangTokenValue.kt index e060bf8..1d1ccd7 100644 --- a/NeoLang/src/main/java/io/neolang/ast/NeoLangTokenValue.kt +++ b/NeoLang/src/main/java/io/neolang/ast/NeoLangTokenValue.kt @@ -3,21 +3,26 @@ package io.neolang.ast /** * @author kiva */ -enum class NeoLangTokenValue(val value: String) { - COLON(":"), - BRACKET_START("{"), - BRACKET_END("}"), - QUOTE("\""), - EOF(""); +class NeoLangTokenValue(val value: String) { + + override fun toString(): String { + return value + } companion object { + val COLON = NeoLangTokenValue(":") + val BRACKET_START = NeoLangTokenValue("{") + val BRACKET_END = NeoLangTokenValue("}") + val QUOTE = NeoLangTokenValue("\"") + val EOF = NeoLangTokenValue("") + fun wrap(tokenText: String): NeoLangTokenValue { return when (tokenText) { COLON.value -> COLON BRACKET_START.value -> BRACKET_START BRACKET_END.value -> BRACKET_END QUOTE.value -> QUOTE - else -> EOF + else -> NeoLangTokenValue(tokenText) } } } diff --git a/NeoLang/src/main/java/io/neolang/ast/base/NeoLangAst.kt b/NeoLang/src/main/java/io/neolang/ast/base/NeoLangAst.kt index d7e33a8..33eea61 100644 --- a/NeoLang/src/main/java/io/neolang/ast/base/NeoLangAst.kt +++ b/NeoLang/src/main/java/io/neolang/ast/base/NeoLangAst.kt @@ -1,12 +1,12 @@ package io.neolang.ast.base -import io.neolang.ast.visitor.NeoLangAstVisitor +import io.neolang.ast.visitor.VisitorFactory /** * @author kiva */ open class NeoLangAst { - fun visit(): NeoLangAstVisitor { - return NeoLangAstVisitor(this) + fun visit(): VisitorFactory { + return VisitorFactory(this) } } \ No newline at end of file diff --git a/NeoLang/src/main/java/io/neolang/ast/visitor/AstVisitor.kt b/NeoLang/src/main/java/io/neolang/ast/visitor/AstVisitor.kt new file mode 100644 index 0000000..755861f --- /dev/null +++ b/NeoLang/src/main/java/io/neolang/ast/visitor/AstVisitor.kt @@ -0,0 +1,12 @@ +package io.neolang.ast.visitor + +import io.neolang.ast.base.NeoLangAst + +/** + * @author kiva + */ +class AstVisitor(private val ast: NeoLangAst, private val visitorCallback: IVisitorCallback) { + fun start() { + // TODO visitor + } +} \ No newline at end of file diff --git a/NeoLang/src/main/java/io/neolang/ast/visitor/IVisitorCallback.kt b/NeoLang/src/main/java/io/neolang/ast/visitor/IVisitorCallback.kt new file mode 100644 index 0000000..fc5fa2c --- /dev/null +++ b/NeoLang/src/main/java/io/neolang/ast/visitor/IVisitorCallback.kt @@ -0,0 +1,7 @@ +package io.neolang.ast.visitor + +/** + * @author kiva + */ +interface IVisitorCallback { +} \ No newline at end of file diff --git a/NeoLang/src/main/java/io/neolang/ast/visitor/NeoLangAstVisitor.kt b/NeoLang/src/main/java/io/neolang/ast/visitor/NeoLangAstVisitor.kt deleted file mode 100644 index 5253583..0000000 --- a/NeoLang/src/main/java/io/neolang/ast/visitor/NeoLangAstVisitor.kt +++ /dev/null @@ -1,8 +0,0 @@ -package io.neolang.ast.visitor - -import io.neolang.ast.base.NeoLangAst - -/** - * @author kiva - */ -open class NeoLangAstVisitor(ast: NeoLangAst) \ No newline at end of file diff --git a/NeoLang/src/main/java/io/neolang/ast/visitor/VisitorFactory.kt b/NeoLang/src/main/java/io/neolang/ast/visitor/VisitorFactory.kt new file mode 100644 index 0000000..1b1b11c --- /dev/null +++ b/NeoLang/src/main/java/io/neolang/ast/visitor/VisitorFactory.kt @@ -0,0 +1,18 @@ +package io.neolang.ast.visitor + +import io.neolang.ast.base.NeoLangAst + +/** + * @author kiva + */ + +class VisitorFactory(private val ast: NeoLangAst) { + + fun getVisitor(callbackInterface: Class): AstVisitor? { + try { + return AstVisitor(ast, callbackInterface.newInstance()) + } catch (e: Exception) { + return null + } + } +} diff --git a/NeoLang/src/main/java/io/neolang/main/DisplayAstVisitor.kt b/NeoLang/src/main/java/io/neolang/main/DisplayAstVisitor.kt new file mode 100644 index 0000000..01a308e --- /dev/null +++ b/NeoLang/src/main/java/io/neolang/main/DisplayAstVisitor.kt @@ -0,0 +1,9 @@ +package io.neolang.main + +import io.neolang.ast.visitor.IVisitorCallback + +/** + * @author kiva + */ +class DisplayAstVisitor : IVisitorCallback { +} \ No newline at end of file diff --git a/NeoLang/src/main/java/io/neolang/command/Main.kt b/NeoLang/src/main/java/io/neolang/main/Main.kt similarity index 62% rename from NeoLang/src/main/java/io/neolang/command/Main.kt rename to NeoLang/src/main/java/io/neolang/main/Main.kt index ac83a2b..0fa4089 100644 --- a/NeoLang/src/main/java/io/neolang/command/Main.kt +++ b/NeoLang/src/main/java/io/neolang/main/Main.kt @@ -1,4 +1,4 @@ -package io.neolang.command +package io.neolang.main import io.neolang.parser.NeoLangParser import java.io.FileInputStream @@ -20,20 +20,21 @@ class Main { val programCode = readFully(it) parser.setInputSource(programCode) val ast = parser.parse() - println("Compile `$it' -> $ast") + + val visitor = ast.visit().getVisitor(DisplayAstVisitor::class.java) + if (visitor != null) { + println("Compile `$it' -> $ast") + visitor.start() + } } return } private fun readFully(file: String): String { - try { - FileInputStream(file).use { - val bytes = ByteArray(it.available()) - it.read(bytes) - return String(bytes) - } - } catch (e: Exception) { - return "" + FileInputStream(file).use { + val bytes = ByteArray(it.available()) + it.read(bytes) + return String(bytes) } } } diff --git a/NeoLang/src/main/java/io/neolang/parser/NeoLangLexer.kt b/NeoLang/src/main/java/io/neolang/parser/NeoLangLexer.kt index fa38af0..d643694 100755 --- a/NeoLang/src/main/java/io/neolang/parser/NeoLangLexer.kt +++ b/NeoLang/src/main/java/io/neolang/parser/NeoLangLexer.kt @@ -11,7 +11,7 @@ import java.util.* * prog: group (group)* * group: attribute (attribute)* * attribute: TEXT COLON block - * block: NUMBER | TEXT | (BRACKET_START group BRACKET_STOP) + * block: NUMBER | TEXT | (BRACKET_START group BRACKET_END) * ] */ @@ -30,25 +30,31 @@ class NeoLangLexer { internal fun lex(): List { val programCode = this.programCode ?: return listOf() + val tokens = ArrayList() currentPosition = 0 lineNumber = 1 - currentChar = programCode[currentPosition] - val tokens = ArrayList() - while (currentPosition < programCode.length) { - val token = nextToken - if (token is NeoLangEOFToken) { - break + if (programCode.isNotEmpty()) { + currentChar = programCode[currentPosition] + + while (currentPosition < programCode.length) { + val token = nextToken + if (token is NeoLangEOFToken) { + break + } + tokens.add(token) } - tokens.add(token) } return tokens } - private fun moveToNextChar(): Boolean { + private fun moveToNextChar(eofThrow: Boolean = false): Boolean { val programCode = this.programCode ?: return false currentPosition++ if (currentPosition >= programCode.length) { + if (eofThrow) { + throw InvalidTokenException("Unexpected EOF near `$currentChar' in line $lineNumber") + } return false } else { currentChar = programCode[currentPosition] @@ -68,7 +74,9 @@ class NeoLangLexer { ++lineNumber } // Skip white chars - moveToNextChar() + if (!moveToNextChar()) { + return NeoLangEOFToken() + } } if (currentPosition >= programCode.length) { @@ -78,11 +86,11 @@ class NeoLangLexer { val currentToken = NeoLangTokenValue.wrap(currentChar.toString()) val token: NeoLangToken = when (currentToken) { NeoLangTokenValue.COLON -> { - moveToNextChar() + moveToNextChar(eofThrow = true) NeoLangToken(NeoLangTokenType.COLON, currentToken) } NeoLangTokenValue.BRACKET_START -> { - moveToNextChar() + moveToNextChar(eofThrow = true) NeoLangToken(NeoLangTokenType.BRACKET_START, currentToken) } NeoLangTokenValue.BRACKET_END -> { @@ -90,14 +98,13 @@ class NeoLangLexer { NeoLangToken(NeoLangTokenType.BRACKET_END, currentToken) } NeoLangTokenValue.QUOTE -> { - moveToNextChar() - NeoLangToken(NeoLangTokenType.QUOTE, currentToken) + NeoLangToken(NeoLangTokenType.STRING, NeoLangTokenValue.wrap(getNextTokenAsString())) } else -> { - if (Character.isDigit(currentChar)) { + if (currentChar.isNumber()) { NeoLangToken(NeoLangTokenType.NUMBER, NeoLangTokenValue.wrap(getNextTokenAsNumber())) - } else if (Character.isLetterOrDigit(currentChar)) { - NeoLangToken(NeoLangTokenType.STRING, NeoLangTokenValue.wrap(getNextTokenAsString())) + } else if (isIdentifier(currentChar, true)) { + NeoLangToken(NeoLangTokenType.ID, NeoLangTokenValue.wrap(getNextTokenAsId())) } else { throw InvalidTokenException("Unexpected character: " + currentChar) } @@ -108,20 +115,118 @@ class NeoLangLexer { return token } - private fun getNextTokenAsNumber(): String { - return buildString { - while (Character.isDigit(currentChar)) { - append(currentChar) - if (!moveToNextChar()) { - break - } - } - } - } - private fun getNextTokenAsString(): String { + // Skip start quote + // and a single quote is now allowed + moveToNextChar(eofThrow = true) + val builder = StringBuilder() + + var loop = true + while (loop && currentChar != NeoLangTokenValue.QUOTE.value[0]) { + // Skip escaped char + if (currentChar == '\\') { + builder.append('\\') + moveToNextChar(eofThrow = true) + } + builder.append(currentChar) + loop = moveToNextChar() + } + + // Skip end quote + moveToNextChar() + return builder.toString() + } + + private fun getNextTokenAsNumber(): String { + var numberValue: Double = (currentChar.toInt() - '0'.toInt()).toDouble() + + // Four types of numbers are supported: + // Dec(123) Hex(0x123) Oct(017) Bin(0b11) + + // Dec + if (numberValue > 0) { + numberValue = getNextDecimalNumber(numberValue) + } else { + // is 0 + if (!moveToNextChar()) { + return numberValue.toString() + } + + // Hex + if (currentChar == 'x' || currentChar == 'X') { + numberValue = getNextHexNumber(numberValue) + } else if (currentChar == 'b' || currentChar == 'B') { + numberValue = getNextBinaryNumber(numberValue) + } else { + numberValue = getNextOctalNumber(numberValue) + } + } + + return numberValue.toString() + } + + private fun getNextOctalNumber(numberValue: Double): Double { + var value: Double = numberValue + var loop = true + while (loop && currentChar in ('0'..'7')) { + value = value * 8 + currentChar.toNumber() + loop = moveToNextChar() + } + return value + } + + private fun getNextBinaryNumber(numberValue: Double): Double { + var value = numberValue + var loop = moveToNextChar() // skip 'b' or 'B' + while (loop && currentChar in ('0'..'1')) { + value += value * 2 + currentChar.toNumber() + loop = moveToNextChar() + } + return value + } + + private fun getNextHexNumber(numberValue: Double): Double { + var value = numberValue + var loop = moveToNextChar() // skip 'x' or 'X' + while (loop && (currentChar.isHexNumber())) { + value *= 16 + + (currentChar.toInt().and(15)) + + if (currentChar >= 'A') 9 else 0 + loop = moveToNextChar() + } + return value + } + + private fun getNextDecimalNumber(numberValue: Double): Double { + var floatPointMeet = false + var floatPart: Double = 0.0 + var floatNumberCounter = 1 + var value = numberValue + + var loop = moveToNextChar() + while (loop) { + if (currentChar.isNumber()) { + if (floatPointMeet) { + floatPart = floatPart * 10 + currentChar.toNumber() + floatNumberCounter *= 10 + } else { + value = value * 10 + currentChar.toNumber() + } + loop = moveToNextChar() + + } else if (currentChar == '.') { + floatPointMeet = true + loop = moveToNextChar() + } else { + break + } + } + return value + floatPart / floatNumberCounter + } + + private fun getNextTokenAsId(): String { return buildString { - while (Character.isLetterOrDigit(currentChar)) { + while (isIdentifier(currentChar, false)) { append(currentChar) if (!moveToNextChar()) { break @@ -130,4 +235,26 @@ class NeoLangLexer { } } + private fun isIdentifier(tokenChar: Char, isFirstChar: Boolean): Boolean { + val isId = (tokenChar in 'a'..'z') + || (tokenChar in 'A'..'Z') + || ("_-#$".contains(tokenChar)) + return if (isFirstChar) isId else (isId || (tokenChar in '0'..'9')) + } + + private fun Char.toNumber(): Int { + return if (isNumber()) { + this.toInt() - '0'.toInt() + } else 0 + } + + private fun Char.isNumber(): Boolean { + return this in ('0'..'9') + } + + private fun Char.isHexNumber(): Boolean { + return this.isNumber() + || this in ('a'..'f') + || this in ('A'..'F') + } } diff --git a/NeoLang/src/main/java/io/neolang/parser/NeoLangParser.kt b/NeoLang/src/main/java/io/neolang/parser/NeoLangParser.kt index 86fc71b..c69e342 100755 --- a/NeoLang/src/main/java/io/neolang/parser/NeoLangParser.kt +++ b/NeoLang/src/main/java/io/neolang/parser/NeoLangParser.kt @@ -26,8 +26,11 @@ class NeoLangParser { private fun updateParserStatus(tokens: List) { if (tokens.isEmpty()) { - throw ParseException("Input tokens must be non-empty") + // Allow empty program + ast = NeoLangProgramNode.emptyNode() + return } + this.tokens.clear() this.tokens.addAll(tokens) currentPosition = 0 @@ -48,7 +51,7 @@ class NeoLangParser { return true } else if (errorThrow) { - throw InvalidTokenException("Unexpected token type " + + throw InvalidTokenException("Unexpected token `${currentToken.tokenValue}' typed " + "`${currentToken.tokenType}' near line ${currentToken.lineNumber}, " + "expected $tokenType") } @@ -98,7 +101,7 @@ class NeoLangParser { private fun attribute(): NeoLangAttributeNode? { val token = currentToken ?: throw InvalidTokenException("Unexpected token: null") - if (match(NeoLangTokenType.STRING)) { + if (match(NeoLangTokenType.ID)) { match(NeoLangTokenType.COLON, errorThrow = true) val block = block() ?: NeoLangBlockNode.emptyNode() return NeoLangAttributeNode(NeoLangStringNode(token), block) @@ -113,6 +116,10 @@ class NeoLangParser { match(NeoLangTokenType.NUMBER, errorThrow = true) return NeoLangBlockNode(NeoLangNumberNode(token)) } + NeoLangTokenType.ID -> { + match(NeoLangTokenType.ID, errorThrow = true) + return NeoLangBlockNode(NeoLangStringNode(token)) + } NeoLangTokenType.STRING -> { match(NeoLangTokenType.STRING, errorThrow = true) return NeoLangBlockNode(NeoLangStringNode(token)) @@ -127,7 +134,7 @@ class NeoLangParser { } else -> throw InvalidTokenException("Unexpected token `$token' for block, " + - "expected `${NeoLangTokenType.NUMBER}', `${NeoLangTokenType.STRING}' or `${NeoLangTokenType.BRACKET_START}'") + "expected `${NeoLangTokenType.NUMBER}', `${NeoLangTokenType.ID}' or `${NeoLangTokenType.BRACKET_START}'") } } diff --git a/app/build.gradle b/app/build.gradle index 9685f58..3dbc64a 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -56,5 +56,5 @@ dependencies { compile 'com.simplecityapps:recyclerview-fastscroll:1.0.16' // compile 'com.ramotion.cardslider:card-slider:0.1.0' // compile 'com.github.igalata:Bubble-Picker:v0.2.4' - implementation project(path: ':NeoLang') + compile project(path: ':NeoLang') } diff --git a/app/src/main/java/io/neoterm/customize/service/NeoServiceManager.kt b/app/src/main/java/io/neoterm/customize/service/NeoServiceManager.kt new file mode 100644 index 0000000..483ee53 --- /dev/null +++ b/app/src/main/java/io/neoterm/customize/service/NeoServiceManager.kt @@ -0,0 +1,10 @@ +package io.neoterm.customize.service + +import io.neolang.main.Main + +/** + * @author kiva + */ +object NeoServiceManager { + val main = Main() +} \ No newline at end of file diff --git a/app/src/test/java/io/neoterm/NeoLangTest.kt b/app/src/test/java/io/neoterm/NeoLangTest.kt index 8926545..613b50f 100644 --- a/app/src/test/java/io/neoterm/NeoLangTest.kt +++ b/app/src/test/java/io/neoterm/NeoLangTest.kt @@ -1,5 +1,6 @@ package io.neoterm +import io.neolang.main.Main import org.junit.Test /** @@ -10,10 +11,7 @@ import org.junit.Test class NeoLangTest { @Test fun testNeoLangParser() { - val parser = io.neolang.parser.NeoLangParser() - parser.setInputSource("app: { x: {} \n x: hello \n a: 1111 \n x: { x: 123 } }") - val ast = parser.parse() - println(ast) + Main.main(arrayOf("NeoLang/example/color-scheme.nl")) } }