diff --git a/YesNt.CodeEditor/SyntaxHighlighter.cs b/YesNt.CodeEditor/SyntaxHighlighter.cs index 8db25f7..6444c7d 100644 --- a/YesNt.CodeEditor/SyntaxHighlighter.cs +++ b/YesNt.CodeEditor/SyntaxHighlighter.cs @@ -129,7 +129,7 @@ internal partial class SyntaxHighlighter(ReadOnlyCollection= 0 && colorIndex < 16) { - messagePart = Base64Decode(messagePart.Replace($"\v{stringColor}\v", string.Empty)); + messagePart = Base64Decode(messagePart.Replace("\x01" + stringColor + "\x01", string.Empty)); consoleColor = (ConsoleColor)colorIndex; } else @@ -155,9 +155,9 @@ internal partial class SyntaxHighlighter(ReadOnlyCollection originalString.ReplaceFirstOccurrence(value, $"\0\v{(int)color}\v{base64Value}\0" + new string(' ', spacesAtEnd)), - SearchMode.EndOfLine => originalString.ReplaceLastOccurrence(value, $"\0\v{(int)color}\v{base64Value}\0" + new string(' ', spacesAtEnd)), - _ => originalString.Replace(value, $"\0\v{(int)color}\v{base64Value}\0" + new string(' ', spacesAtEnd)) + SearchMode.StartOfLine => originalString.ReplaceFirstOccurrence(value, $"\0\x01{(int)color}\x01{base64Value}\0" + new string(' ', spacesAtEnd)), + SearchMode.EndOfLine => originalString.ReplaceLastOccurrence(value, $"\0\x01{(int)color}\x01{base64Value}\0" + new string(' ', spacesAtEnd)), + _ => originalString.Replace(value, $"\0\x01{(int)color}\x01{base64Value}\0" + new string(' ', spacesAtEnd)) }; return result; } @@ -170,7 +170,7 @@ internal partial class SyntaxHighlighter(ReadOnlyCollection lines = [ "list items new", - "list items add hello~spcworld", + "list items add \"hello world\"", "list items get 0", "var result = %out eval", "${result}" diff --git a/YesNt.Interpreter.Tests/ProcessingStatementsTests.cs b/YesNt.Interpreter.Tests/ProcessingStatementsTests.cs index 7bd07a3..4d20081 100644 --- a/YesNt.Interpreter.Tests/ProcessingStatementsTests.cs +++ b/YesNt.Interpreter.Tests/ProcessingStatementsTests.cs @@ -46,9 +46,21 @@ public class ProcessingStatementsTests } [TestMethod] - public void EvalDecodesSafeStringTest() + public void EvalRawTildeSequenceIsLiteralTest() { - YesNtAssert.IsLineEqual("hello~nliworld eval", "hello\nworld"); + YesNtAssert.IsLineEqual("hello~nliworld eval", "hello~nliworld"); + } + + [TestMethod] + public void EvalDecodesStringLiteralEscapesTest() + { + List lines = + [ + "var x = \"hello\\nworld\"", + "${x} eval" + ]; + + YesNtAssert.IsLastLineEqual(lines, "hello\nworld"); } [TestMethod] diff --git a/YesNt.Interpreter/Statements/StringLiteralStatements.cs b/YesNt.Interpreter/Statements/StringLiteralStatements.cs index e502a30..6b56c69 100644 --- a/YesNt.Interpreter/Statements/StringLiteralStatements.cs +++ b/YesNt.Interpreter/Statements/StringLiteralStatements.cs @@ -70,6 +70,10 @@ internal class StringLiteralStatements : StatementRuntimeInformation 'n' => '\n', 'r' => '\r', 't' => '\t', + 'b' => '\b', + 'f' => '\f', + 'a' => '\a', + 'v' => '\v', '"' => '"', '\\' => '\\', _ => escapeChar diff --git a/YesNt.Interpreter/Utilities/StringExtentions.cs b/YesNt.Interpreter/Utilities/StringExtentions.cs index 71d13f7..b5ff31b 100644 --- a/YesNt.Interpreter/Utilities/StringExtentions.cs +++ b/YesNt.Interpreter/Utilities/StringExtentions.cs @@ -13,10 +13,14 @@ namespace YesNt.Interpreter.Utilities; /// /// YesNt uses a "safe string" encoding to pass values through the interpreter pipeline without /// accidentally triggering keyword matching. Special characters (spaces, operators, punctuation, -/// control characters) are replaced with tilde-prefixed three-letter codes -/// (e.g. space → ~spc, newline → ~nli). The mapping is defined in -/// . Use to encode and -/// to decode. +/// control characters) are replaced with SOH-delimited three-letter codes +/// (e.g. space → \x01spc\x01, newline → \x01nli\x01). These codes use the +/// non-printable SOH character (U+0001) as a sentinel. Written via string concatenation +/// ("\x01" + "spc" + "\x01") to avoid C#'s greedy \x hex escape absorbing +/// following hex-digit letters. U+0001 cannot appear in normal user source, preventing raw +/// source text from being accidentally decoded. +/// The mapping is defined in . Use +/// to encode and to decode. /// /// public static class StringExtensions @@ -29,21 +33,21 @@ public static class StringExtensions /// public static Dictionary ReplacementRules { get; } = new() { - {"~", "~til" }, - {" ", "~spc" }, - {"%", "~per" }, - {"<", "~let" }, - {">", "~grt" }, - {",", "~com" }, - {"!", "~exm" }, - {"|", "~pip" }, - {"\n","~nli" }, - {"\r","~ret" }, - {"\t","~tab" }, - {"\b","~bac" }, - {"\f","~for" }, - {"\a","~ale" }, - {"", "~emp" }, + {"~", "\x01" + "til" + "\x01" }, + {" ", "\x01" + "spc" + "\x01" }, + {"%", "\x01" + "per" + "\x01" }, + {"<", "\x01" + "let" + "\x01" }, + {">", "\x01" + "grt" + "\x01" }, + {",", "\x01" + "com" + "\x01" }, + {"!", "\x01" + "exm" + "\x01" }, + {"|", "\x01" + "pip" + "\x01" }, + {"\n", "\x01" + "nli" + "\x01" }, + {"\r", "\x01" + "ret" + "\x01" }, + {"\t", "\x01" + "tab" + "\x01" }, + {"\b", "\x01" + "bac" + "\x01" }, + {"\f", "\x01" + "for" + "\x01" }, + {"\a", "\x01" + "ale" + "\x01" }, + {"", "\x01" + "emp" + "\x01" }, }; static StringExtensions() @@ -130,15 +134,15 @@ public static class StringExtensions private static string ReplaceOnce(string input, Dictionary replacementRules) { - // ~emp/string.Empty is a special case, it is used to represent empty strings and won't work with the normal rules because an empty string always matches and causes an infinite loop 3 letter abbreviation - IEnumerable> matches = replacementRules.Where(rule => rule.Key != string.Empty && input.Contains(rule.Key)); + // \x01emp\x01/string.Empty is a special case, it is used to represent empty strings and won't work with the normal rules because an empty string always matches and causes an infinite loop 3 letter abbreviation + IEnumerable> matches = replacementRules.Where(rule => rule.Key != string.Empty && input.Contains(rule.Key, StringComparison.Ordinal)); if (!matches.Any()) { return input; } KeyValuePair match = matches.First(); - int startIndex = input.IndexOf(match.Key); + int startIndex = input.IndexOf(match.Key, StringComparison.Ordinal); int endIndex = startIndex + match.Key.Length; string before = ReplaceOnce(input[..startIndex], replacementRules); diff --git a/docs/language-reference.md b/docs/language-reference.md index 834e207..ad845b9 100644 --- a/docs/language-reference.md +++ b/docs/language-reference.md @@ -30,7 +30,7 @@ Execution proceeds top-to-bottom unless a control-flow statement changes the lin - Blank lines are silently skipped. - Lines starting with `#` are comments and are silently skipped. - Variable interpolation uses `${name}` and is evaluated before the statement runs. -- Special characters inside string literals are encoded internally and decoded on output — this is transparent to scripts. +- Special characters inside string literals are encoded internally and decoded on output - this is transparent to scripts. --- @@ -38,7 +38,7 @@ Execution proceeds top-to-bottom unless a control-flow statement changes the lin ```ynt # This is a comment. -print_line Hello # inline comments are NOT supported — everything after print_line is the argument +print_line Hello # inline comments are NOT supported - everything after print_line is the argument ``` Only whole-line comments (lines whose first non-whitespace character is `#`) are supported. @@ -60,10 +60,14 @@ print_line "She said \"hi\"" | `\n` | Newline | | `\r` | Carriage return | | `\t` | Horizontal tab | +| `\b` | Backspace | +| `\f` | Form feed | +| `\a` | Alert (bell) | +| `\v` | Vertical tab | | `\"` | Literal double-quote | | `\\` | Literal backslash | -Variable interpolation (`${name}`) is **not** evaluated inside string literals — the braces +Variable interpolation (`${name}`) is **not** evaluated inside string literals - the braces and content are passed through verbatim. ```ynt @@ -76,7 +80,7 @@ print_line "hello " ${x} # prints: hello world (interpolation outside the lite ## Variables -### Local variables — `var` +### Local variables - `var` ``` var = @@ -92,7 +96,7 @@ var greeting = Hello, world! Variable names may only contain letters and digits (`[a-zA-Z0-9]`). -### Global variables — `global` +### Global variables - `global` ``` global = @@ -104,7 +108,7 @@ Defines or updates a variable that is visible across all function scopes and bac global total = 100 ``` -### Reading a variable — `${name}` +### Reading a variable - `${name}` `${name}` is an inline token that is replaced with the variable's value before the statement executes. It can appear anywhere in a line and multiple occurrences are replaced left to right. @@ -116,7 +120,7 @@ var b = 10 print_line ${a} plus ${b} ``` -### Deleting a variable — `delete` +### Deleting a variable - `delete` ``` delete @@ -134,7 +138,7 @@ delete temp ## Console I/O -### Print with newline — `print_line` +### Print with newline - `print_line` ``` print_line @@ -149,7 +153,7 @@ print_line print_line Done. ``` -### Print without newline — `print` +### Print without newline - `print` ``` print @@ -163,7 +167,7 @@ var name = %read_line print_line Hello, ${name}! ``` -### Read a line of input — `%read_line` +### Read a line of input - `%read_line` `%read_line` is an inline token that is replaced with one line of text read from standard input. @@ -172,7 +176,7 @@ var answer = %read_line print_line You typed: ${answer} ``` -### Read a single key — `%read_key` +### Read a single key - `%read_key` `%read_key` is an inline token that is replaced with the single character pressed by the user (no Enter required). @@ -182,7 +186,7 @@ var key = %read_key print_line You pressed: ${key} ``` -### Clear the console — `clear` +### Clear the console - `clear` ``` clear @@ -348,7 +352,7 @@ if ${debug} == True call dump_state ## Functions -### Declaring a function — `func` +### Declaring a function - `func` ``` func : @@ -367,7 +371,7 @@ func add: return ``` -### Calling a function — `call` +### Calling a function - `call` ``` call @@ -384,13 +388,13 @@ call add with 3, 7 ### Input arguments -#### Push an input argument — `push_in` +#### Push an input argument - `push_in` ``` push_in ``` -Pushes a value onto the input argument stack. Values are consumed in the order they were pushed — +Pushes a value onto the input argument stack. Values are consumed in the order they were pushed - the first `push_in` call is the first value consumed by `%in` inside the function. ```ynt @@ -398,7 +402,7 @@ push_in Alice call greet ``` -#### Pop the next input argument — `%in` +#### Pop the next input argument - `%in` `%in` is an inline token (only valid inside a function) that is replaced with the next value popped from the argument stack. @@ -410,7 +414,7 @@ func greet: return ``` -#### Check if input argument exists — `%has_in` +#### Check if input argument exists - `%has_in` `%has_in` is replaced with `True` or `False` depending on whether the input stack is non-empty. Only valid inside a function. @@ -423,7 +427,7 @@ return ### Output values -#### Push an output value — `push_out` +#### Push an output value - `push_out` ``` push_out @@ -431,7 +435,7 @@ push_out Only valid inside a function. Pushes a return value onto the output stack. -#### Consume an output value — `%out` +#### Consume an output value - `%out` `%out` is an inline token that pops and inserts the top value from the output stack. Valid anywhere after a function call or list/processing statement that pushes to the output stack. @@ -442,7 +446,7 @@ var total = %out print_line ${total} ``` -#### Check if output value exists — `%has_out` +#### Check if output value exists - `%has_out` `%has_out` is replaced with `True` or `False` depending on whether the output stack is non-empty. @@ -451,7 +455,7 @@ call maybe_produce if %has_out == True call consume_result ``` -### Clear the call stack — `clear_call_stack` +### Clear the call stack - `clear_call_stack` ``` clear_call_stack @@ -465,7 +469,7 @@ Discards all frames on the function call stack. Useful for error recovery. Lists are ordered, mutable sequences of strings. All list operations start with the keyword `list`. -### Create or reset — `list … new` +### Create or reset - `list … new` ``` list new @@ -473,7 +477,7 @@ list new Creates an empty list. If the list already exists it is cleared. -### Add an item — `list … add` +### Add an item - `list … add` ``` list add @@ -481,7 +485,7 @@ list add Appends `` to the end of the list. -### Get an item — `list … get` +### Get an item - `list … get` ``` list get @@ -498,7 +502,7 @@ var first = %out print_line ${first} ``` -### Set an item — `list … set` +### Set an item - `list … set` ``` list set @@ -506,7 +510,7 @@ list set Replaces the item at `` with ``. -### Remove an item — `list … remove` +### Remove an item - `list … remove` ``` list remove @@ -514,7 +518,7 @@ list remove Removes the item at ``. Subsequent items shift down. -### Insert an item — `list … insert` +### Insert an item - `list … insert` ``` list insert @@ -522,7 +526,7 @@ list insert Inserts `` before position ``. -### Get the length — `list … length` +### Get the length - `list … length` ``` list length @@ -536,7 +540,7 @@ var n = %out print_line ${n} items ``` -### Clear all items — `list … clear` +### Clear all items - `list … clear` ``` list clear @@ -544,7 +548,7 @@ list clear Removes all items but keeps the list alive. -### Delete a list — `list … delete` +### Delete a list - `list … delete` ``` list delete @@ -556,11 +560,11 @@ Removes the list entirely. ## Processing -### Arithmetic — `calc` +### Arithmetic - `calc` See [Arithmetic](#arithmetic). -### Decode a safe string — `eval` +### Decode a safe string - `eval` ``` eval @@ -569,7 +573,7 @@ See [Arithmetic](#arithmetic). Decodes any internally-encoded characters in the current line back to their plain-text form. Useful after producing text from string literal operations that you want to re-use as plain text. -### Run from the current line in a background task — `task` +### Run from the current line in a background task - `task` ``` task @@ -608,7 +612,7 @@ func background_job: This works well for task-only worker flows. In non-task/shared flows, prefer `return` if you want to return to the caller instead of terminating that execution flow with `exit`. -### Sleep — `sleep` +### Sleep - `sleep` ``` sleep @@ -622,7 +626,7 @@ sleep 1000 print_line One second later. ``` -### Get string length — `length` +### Get string length - `length` ``` length @@ -636,7 +640,7 @@ var len = %out print_line ${len} ``` -### Import another script — `import` +### Import another script - `import` ``` import @@ -655,7 +659,7 @@ import lib/math.ynt ## System -### Execute a program — `exec` +### Execute a program - `exec` ``` exec @@ -706,7 +710,7 @@ var roll = %rand ## Termination -### Normal exit — `exit` +### Normal exit - `exit` ``` exit @@ -714,7 +718,7 @@ exit Terminates the script gracefully. Background tasks that are still running are not cancelled. -### Exit and cancel all tasks — `abort_all` +### Exit and cancel all tasks - `abort_all` ``` abort_all @@ -722,7 +726,7 @@ abort_all Terminates the script and signals all background tasks to stop. -### Throw an error — `throw` +### Throw an error - `throw` ``` throw @@ -740,7 +744,7 @@ func validate_fail: return ``` -### Non-fatal error — `error` +### Non-fatal error - `error` ``` error