diff --git a/data/evaluatto/src/main/java/io/github/sadellie/evaluatto/Expression.kt b/data/evaluatto/src/main/java/io/github/sadellie/evaluatto/Expression.kt index c86b0ce1..4b453c05 100644 --- a/data/evaluatto/src/main/java/io/github/sadellie/evaluatto/Expression.kt +++ b/data/evaluatto/src/main/java/io/github/sadellie/evaluatto/Expression.kt @@ -35,7 +35,7 @@ class Expression( private val radianMode: Boolean = true, private val roundingMode: RoundingMode = RoundingMode.HALF_EVEN ) { - private val tokens = Tokenizer(input).tokenize() + private val tokens = input.tokenize() private var cursorPosition = 0 /** diff --git a/data/evaluatto/src/main/java/io/github/sadellie/evaluatto/Tokenizer.kt b/data/evaluatto/src/main/java/io/github/sadellie/evaluatto/Tokenizer.kt index 129ba237..237f2355 100644 --- a/data/evaluatto/src/main/java/io/github/sadellie/evaluatto/Tokenizer.kt +++ b/data/evaluatto/src/main/java/io/github/sadellie/evaluatto/Tokenizer.kt @@ -26,188 +26,169 @@ sealed class TokenizerException(message: String) : Exception(message) { class BadScientificNotation : TokenizerException("Expected plus or minus symbol after \"E\"") } -class Tokenizer(private val streamOfTokens: String) { - // Don't create object at all? - fun tokenize(): List { - var cursor = 0 - val tokens: MutableList = mutableListOf() +fun String.tokenize(): List { + var cursor = 0 + val tokens: MutableList = mutableListOf() - while (cursor != streamOfTokens.length) { - val nextToken = peekTokenAfter(cursor) + while (cursor != this.length) { + val nextToken = peekTokenAfter(this, cursor) - if (nextToken != null) { - tokens.add(nextToken) - cursor += nextToken.length - } else { - // Didn't find any token, move left slowly (by 1 symbol) - cursor++ - } + if (nextToken != null) { + tokens.add(nextToken) + cursor += nextToken.length + } else { + // Didn't find any token, move left slowly (by 1 symbol) + cursor++ } - - return tokens.repairLexicon() } - private fun peekTokenAfter(cursor: Int): String? { - Token.expressionTokens.forEach { token -> - val subs = streamOfTokens - .substring( - cursor, - (cursor + token.length).coerceAtMost(streamOfTokens.length) - ) - if (subs == token) { - // Got a digit, see if there are other digits coming after - if (token in Token.Digit.allWithDot) { - val number = streamOfTokens - .substring(cursor) - .takeWhile { Token.Digit.allWithDot.contains(it.toString()) } + return tokens.repairLexicon() +} - if (number.count { it.toString() == Token.Digit.dot } > 1) { - throw TokenizerException.TooManyFractionSymbols() - } +private fun peekTokenAfter( + streamOfTokens: String, + cursor: Int +): String? { + Token.expressionTokens.forEach { token -> + val subs = streamOfTokens + .substring( + cursor, + (cursor + token.length).coerceAtMost(streamOfTokens.length) + ) + if (subs == token) { + // Got a digit, see if there are other digits coming after + if (token in Token.Digit.allWithDot) { + val number = streamOfTokens + .substring(cursor) + .takeWhile { Token.Digit.allWithDot.contains(it.toString()) } - return number - } - return token - } - } - return null - } - - private fun List.repairLexicon(): List { - return this - .missingClosingBrackets() - .unpackNotation() - .missingMultiply() - .unpackAllPercents() - // input like 80%80% should be treated as 80%*80%. - // After unpacking we get (80/100)(80/100), the multiply is missing (!!!) - // No, we can't unpack before fixing missing multiply. - // Ideally we we need to add missing multiply for 80%80% - // In that case unpackAllPercents gets input with all operators 80%*80% in this case - // Can't be done right now since missingMultiply checks for tokens in front only - .missingMultiply() - } - - private fun List.missingClosingBrackets(): List { - val leftBracket = this.count { it == Token.Operator.leftBracket } - val rightBrackets = this.count { it == Token.Operator.rightBracket } - val neededBrackets = leftBracket - rightBrackets - - if (neededBrackets <= 0) return this - - var fixed = this - repeat(neededBrackets) { - fixed = fixed + Token.Operator.rightBracket - } - return fixed - } - - private fun List.missingMultiply(): List { - val result = this.toMutableList() - val original = this - var offset = 0 - - fun addTokenAfter(index: Int) { - result.add(index + 1 + offset, Token.Operator.multiply) - offset += 1 - } - - original.forEachIndexed { index, token -> - when { - // This will not insert multiply between digits because they are grouped into a - // single token. It's not possible to get separate digit tokens near each other - // Things like ["123", "456"] are impossible, will be ["123456"] - token.isDigitToken() || - token in Token.Const.all || - token == Token.Operator.rightBracket -> { - val tokenInFront = original.tokenInFront(index) ?: return@forEachIndexed - - when { - tokenInFront == Token.Operator.leftBracket || - tokenInFront in Token.Func.all || - tokenInFront in Token.Const.all || - tokenInFront == Token.Operator.sqrt || - tokenInFront.isDigitToken() -> { - addTokenAfter(index) - } - } - } - } - } - - return result - } - - private fun List.unpackAllPercents(): List { - var result = this - while (result.contains(Token.Operator.percent)) { - val percIndex = result.indexOf(Token.Operator.percent) - result = result.unpackPercentAt(percIndex) - } - return result - } - - private fun List.unpackNotation(): List { - // Transform 1E+7 ==> 1*10^7 - // Transform 1E-7 ==> 1/10^7 - val result = this.toMutableList() - val listIterator = result.listIterator() - - while (listIterator.hasNext()) { - if (listIterator.next() == Token.DisplayOnly.engineeringE) { - listIterator.remove() - - val tokenAfterE = try { - listIterator.next() - } catch (e: Exception) { - throw TokenizerException.BadScientificNotation() + if (number.count { it.toString() == Token.Digit.dot } > 1) { + throw TokenizerException.TooManyFractionSymbols() } - listIterator.remove() + return number + } + return token + } + } + return null +} - when (tokenAfterE) { - Token.Operator.minus -> listIterator.add(Token.Operator.divide) - Token.Operator.plus -> listIterator.add(Token.Operator.multiply) - else -> throw TokenizerException.BadScientificNotation() - } +private fun MutableList.repairLexicon(): List { + return this + .missingClosingBrackets() + .unpackNotation() + .missingMultiply() + .unpackAllPercents() + // input like 80%80% should be treated as 80%*80%. + // After unpacking we get (80/100)(80/100), the multiply is missing (!!!) + // No, we can't unpack before fixing missing multiply. + // Ideally we we need to add missing multiply for 80%80% + // In that case unpackAllPercents gets input with all operators 80%*80% in this case + // Can't be done right now since missingMultiply checks for tokens in front only + .missingMultiply() +} - listIterator.add("10") - listIterator.add(Token.Operator.power) +private fun MutableList.missingClosingBrackets(): MutableList { + val leftBracket = this.count { it == Token.Operator.leftBracket } + val rightBrackets = this.count { it == Token.Operator.rightBracket } + val neededBrackets = leftBracket - rightBrackets + + if (neededBrackets <= 0) return this + + repeat(neededBrackets) { + this.add(Token.Operator.rightBracket) + } + return this +} + +private fun MutableList.missingMultiply(): MutableList { + val iterator = this.listIterator() + + while (iterator.hasNext()) { + val currentToken = iterator.next() + + // Need two token for checks + if (!iterator.hasNext()) break + + val isDigit = currentToken.isDigitToken() + val isConst = currentToken in Token.Const.all + val isRightBracket = currentToken == Token.Operator.rightBracket + + // may need a multiplication after + if (isDigit || isConst || isRightBracket) { + // Peek next, but then go back + val tokenAfter = iterator.next() + iterator.previous() + + if (tokenAfter == Token.Operator.leftBracket || + tokenAfter in Token.Func.all || + tokenAfter in Token.Const.all || + tokenAfter == Token.Operator.sqrt || + tokenAfter.isDigitToken()) { + + iterator.add(Token.Operator.multiply) } } - - return result } - private fun List.unpackPercentAt(percentIndex: Int): List { - var cursor = percentIndex + return this +} - // get whatever is the percentage - val percentage = this.getNumberOrExpressionBefore(percentIndex) - // Move cursor - cursor -= percentage.size +private fun MutableList.unpackNotation(): MutableList { + // Transform 1E+7 ==> 1*10^7 + // Transform 1E-7 ==> 1/10^7 + val iterator = this.listIterator() - // get the operator in front - cursor -= 1 - val operator = this.getOrNull(cursor) + while (iterator.hasNext()) { + if (iterator.next() == Token.DisplayOnly.engineeringE) { + iterator.remove() - // Don't go further - if ((operator == null) or (operator !in listOf(Token.Operator.plus, Token.Operator.minus))) { - val mutList = this.toMutableList() + val tokenAfterE = try { + iterator.next() + } catch (e: Exception) { + throw TokenizerException.BadScientificNotation() + } - // Remove percentage - mutList.removeAt(percentIndex) + iterator.remove() - //Add opening bracket before percentage - mutList.add(percentIndex - percentage.size, Token.Operator.leftBracket) + when (tokenAfterE) { + Token.Operator.minus -> iterator.add(Token.Operator.divide) + Token.Operator.plus -> iterator.add(Token.Operator.multiply) + else -> throw TokenizerException.BadScientificNotation() + } - // Add "/ 100" and closing bracket - mutList.addAll(percentIndex + 1, listOf(Token.Operator.divide, "100", Token.Operator.rightBracket)) - - return mutList + iterator.add("10") + iterator.add(Token.Operator.power) } - // Get the base - val base = this.getBaseBefore(cursor) + } + + return this +} + +private fun MutableList.unpackAllPercents(): MutableList { + var result = this + while (result.contains(Token.Operator.percent)) { + val percIndex = result.indexOf(Token.Operator.percent) + result = result.unpackPercentAt(percIndex) + } + return result +} + +private fun MutableList.unpackPercentAt(percentIndex: Int): MutableList { + var cursor = percentIndex + + // get whatever is the percentage + val percentage = this.getNumberOrExpressionBefore(percentIndex) + // Move cursor + cursor -= percentage.size + + // get the operator in front + cursor -= 1 + val operator = this.getOrNull(cursor) + + // Don't go further + if ((operator == null) or (operator !in listOf(Token.Operator.plus, Token.Operator.minus))) { val mutList = this.toMutableList() // Remove percentage @@ -216,71 +197,83 @@ class Tokenizer(private val streamOfTokens: String) { //Add opening bracket before percentage mutList.add(percentIndex - percentage.size, Token.Operator.leftBracket) - // Add "/ 100" and other stuff - mutList.addAll( - percentIndex + 1, - listOf( - Token.Operator.divide, - "100", - Token.Operator.multiply, - Token.Operator.leftBracket, - *base.toTypedArray(), - Token.Operator.rightBracket, - Token.Operator.rightBracket - ) - ) + // Add "/ 100" and closing bracket + mutList.addAll(percentIndex + 1, listOf(Token.Operator.divide, "100", Token.Operator.rightBracket)) return mutList } + // Get the base + val base = this.getBaseBefore(cursor) + val mutList = this.toMutableList() - private fun List.getNumberOrExpressionBefore(pos: Int): List { - val digits = Token.Digit.allWithDot.map { it[0] } + // Remove percentage + mutList.removeAt(percentIndex) - val tokenInFront = this[pos - 1] + //Add opening bracket before percentage + mutList.add(percentIndex - percentage.size, Token.Operator.leftBracket) - // Just number - if (tokenInFront.all { it in digits }) return listOf(tokenInFront) + // Add "/ 100" and other stuff + mutList.addAll( + percentIndex + 1, + listOf( + Token.Operator.divide, + "100", + Token.Operator.multiply, + Token.Operator.leftBracket, + *base.toTypedArray(), + Token.Operator.rightBracket, + Token.Operator.rightBracket + ) + ) - // For cases like "100+(2+5)|%". The check above won't pass, so the next expected thing is - // a number in brackets. Anything else is not expected. - if (tokenInFront != Token.Operator.rightBracket) throw TokenizerException.FailedToUnpackNumber() - - // Start walking left until we get balanced brackets - var cursor = pos - 1 - var leftBrackets = 0 - var rightBrackets = 1 // We set 1 because we start with closing bracket - - while (leftBrackets != rightBrackets) { - cursor-- - val currentToken = this[cursor] - if (currentToken == Token.Operator.leftBracket) leftBrackets++ - if (currentToken == Token.Operator.rightBracket) rightBrackets++ - } - - return this.subList(cursor, pos) - } - - private fun List.getBaseBefore(pos: Int): List { - var cursor = pos - var leftBrackets = 0 - var rightBrackets = 0 - - while ((--cursor >= 0)) { - val currentToken = this[cursor] - - if (currentToken == Token.Operator.leftBracket) leftBrackets++ - if (currentToken == Token.Operator.rightBracket) rightBrackets++ - - if (leftBrackets > rightBrackets) break - } - - // Return cursor back to last token - cursor += 1 - - return this.subList(cursor, pos) - } - - private fun String.isDigitToken(): Boolean = first().toString() in Token.Digit.allWithDot - - private fun List.tokenInFront(index: Int): String? = getOrNull(index + 1) + return mutList } + +private fun MutableList.getNumberOrExpressionBefore(pos: Int): List { + val digits = Token.Digit.allWithDot.map { it[0] } + + val tokenInFront = this[pos - 1] + + // Just number + if (tokenInFront.all { it in digits }) return listOf(tokenInFront) + + // For cases like "100+(2+5)|%". The check above won't pass, so the next expected thing is + // a number in brackets. Anything else is not expected. + if (tokenInFront != Token.Operator.rightBracket) throw TokenizerException.FailedToUnpackNumber() + + // Start walking left until we get balanced brackets + var cursor = pos - 1 + var leftBrackets = 0 + var rightBrackets = 1 // We set 1 because we start with closing bracket + + while (leftBrackets != rightBrackets) { + cursor-- + val currentToken = this[cursor] + if (currentToken == Token.Operator.leftBracket) leftBrackets++ + if (currentToken == Token.Operator.rightBracket) rightBrackets++ + } + + return this.subList(cursor, pos) +} + +private fun List.getBaseBefore(pos: Int): List { + var cursor = pos + var leftBrackets = 0 + var rightBrackets = 0 + + while ((--cursor >= 0)) { + val currentToken = this[cursor] + + if (currentToken == Token.Operator.leftBracket) leftBrackets++ + if (currentToken == Token.Operator.rightBracket) rightBrackets++ + + if (leftBrackets > rightBrackets) break + } + + // Return cursor back to last token + cursor += 1 + + return this.subList(cursor, pos) +} + +private fun String.isDigitToken(): Boolean = first().toString() in Token.Digit.allWithDot diff --git a/data/evaluatto/src/test/java/io/github/sadellie/evaluatto/Helpers.kt b/data/evaluatto/src/test/java/io/github/sadellie/evaluatto/Helpers.kt index 55861c01..2b1c9e31 100644 --- a/data/evaluatto/src/test/java/io/github/sadellie/evaluatto/Helpers.kt +++ b/data/evaluatto/src/test/java/io/github/sadellie/evaluatto/Helpers.kt @@ -40,7 +40,7 @@ fun assertExprFail( } fun assertLex(expected: List, actual: String) = - assertEquals(expected, Tokenizer(actual).tokenize()) + assertEquals(expected, actual.tokenize()) fun assertLex(expected: String, actual: String) = - assertEquals(expected, Tokenizer(actual).tokenize().joinToString("")) + assertEquals(expected, actual.tokenize().joinToString(""))