Lexicon fixer improvements

data/evaluatto/src/main/java/io/github/sadellie/evaluatto/Tokenizer.kt

@@ -76,8 +76,8 @@ class Tokenizer(private val streamOfTokens: String) {
             .missingClosingBrackets()
             .missingMultiply()
             .unpackAlPercents()
-            // input like 80%80% should be treated as 80%-80%.
+            // input like 80%80% should be treated as 80%*80%.
-            // After unpacking we get (80/100)(80/100), the multiply is missing
+            // After unpacking we get (80/100)(80/100), the multiply is missing (!!!)
             // No, we can't unpack before fixing missing multiply.
             // Ideally we we need to add missing multiply for 80%80%
             // In that case unpackAlPercents gets input with all operators 80%*80% in this case
@@ -100,37 +100,39 @@ class Tokenizer(private val streamOfTokens: String) {
     }
 
     private fun List<String>.missingMultiply(): List<String> {
-        val results = this.toMutableList()
+        val result = this.toMutableList()
-        val insertIndexes = mutableListOf<Int>()
+        val original = this
+        var offset = 0
 
-        // Records the index if it needs a multiply symbol
+        fun addTokenAfter(index: Int) {
-        fun needsMultiply(index: Int) {
+            result.add(index + 1 + offset, Token.Operator.multiply)
-            val tokenInFront = results.getOrNull(index - 1) ?: return
+            offset += 1
+        }
 
+        original.forEachIndexed { index, token ->
             when {
-                tokenInFront.first().toString() in Token.Digit.allWithDot ||
+                // This will not insert multiply between digits because they are grouped into a
-                tokenInFront == Token.Operator.rightBracket ||
+                // single token. It's not possible to get separate digit tokens near each other
-                tokenInFront in Token.Const.all -> {
+                // Things like ["123", "456"] are impossible, will be ["123456"]
-                    // Can't add token now, it will modify tokens list (we are looping over it)
+                token.isDigitToken() ||
-                    insertIndexes.add(index + insertIndexes.size)
+                token in Token.Const.all ||
+                token == Token.Operator.rightBracket -> {
+                    val tokenInFront = original.tokenInFront(index) ?: return@forEachIndexed
+
+                    when {
+                        tokenInFront == Token.Operator.leftBracket ||
+                        tokenInFront in Token.Func.all ||
+                        tokenInFront in Token.Const.all ||
+                        tokenInFront == Token.Operator.sqrt ||
+                        tokenInFront.isDigitToken() -> {
+                            addTokenAfter(index)
+                        }
+                    }
                 }
             }
         }
 
-        results.forEachIndexed { index, s ->
+        return result
-            when (s) {
-                Token.Operator.leftBracket,
-                Token.Operator.sqrt,
-                in Token.Const.all,
-                in Token.Func.all -> needsMultiply(index)
-            }
-        }
-
-        insertIndexes.forEach {
-            results.add(it, Token.Operator.multiply)
-        }
-
-        return results
     }
 
     private fun List<String>.unpackAlPercents(): List<String> {
@@ -241,4 +243,8 @@ class Tokenizer(private val streamOfTokens: String) {
 
         return this.subList(cursor, pos)
     }
+
+    private fun String.isDigitToken(): Boolean = first().toString() in Token.Digit.allWithDot
+
+    private fun List<String>.tokenInFront(index: Int): String? = getOrNull(index + 1)
 }

data/evaluatto/src/test/java/io/github/sadellie/evaluatto/FixLexiconTest.kt

@@ -48,6 +48,10 @@ class FixLexiconTest {
         assertLex(
             "e×e+π", "ee+π"
         )
+
+        assertLex(
+            "(69)×420", "(69)420"
+        )
     }
 
     @Test
@@ -61,7 +65,7 @@ class FixLexiconTest {
         )
 
         assertLex(
-            "123)))12+4", "123)))12+4"
+            "123)))×12+4", "123)))12+4"
         )
 
         assertLex(