Stone_Red/YesNt-Interpreter
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/YesNt-Interpreter.git synced 2026-09-04 00:56:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

Switch safe string encoding from ~tilde codes to SOH-delimited codes throughout

Browse Source
This commit is contained in:
Stone_Red
2026-03-05 00:24:20 +01:00
parent 7eea8756bc
commit 071d9a937e
6 changed files with 97 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
YesNt.CodeEditor/SyntaxHighlighter.cs
+6 -6
View File
@@ -129,7 +129,7 @@ internal partial class SyntaxHighlighter(ReadOnlyCollection<StatementInformation
bool succ = int.TryParse(stringColor, out int colorIndex); bool succ = int.TryParse(stringColor, out int colorIndex);
if (succ && colorIndex >= 0 && colorIndex < 16) if (succ && colorIndex >= 0 && colorIndex < 16)
{ {
messagePart = Base64Decode(messagePart.Replace($"\v{stringColor}\v", string.Empty)); messagePart = Base64Decode(messagePart.Replace("\x01" + stringColor + "\x01", string.Empty));
consoleColor = (ConsoleColor)colorIndex; consoleColor = (ConsoleColor)colorIndex;
} }
else else
@@ -155,9 +155,9 @@ internal partial class SyntaxHighlighter(ReadOnlyCollection<StatementInformation
string result = searchMode switch string result = searchMode switch
{ {
SearchMode.StartOfLine => originalString.ReplaceFirstOccurrence(value, $"\0\v{(int)color}\v{base64Value}\0" + new string(' ', spacesAtEnd)), SearchMode.StartOfLine => originalString.ReplaceFirstOccurrence(value, $"\0\x01{(int)color}\x01{base64Value}\0" + new string(' ', spacesAtEnd)),
SearchMode.EndOfLine => originalString.ReplaceLastOccurrence(value, $"\0\v{(int)color}\v{base64Value}\0" + new string(' ', spacesAtEnd)), SearchMode.EndOfLine => originalString.ReplaceLastOccurrence(value, $"\0\x01{(int)color}\x01{base64Value}\0" + new string(' ', spacesAtEnd)),
_ => originalString.Replace(value, $"\0\v{(int)color}\v{base64Value}\0" + new string(' ', spacesAtEnd)) _ => originalString.Replace(value, $"\0\x01{(int)color}\x01{base64Value}\0" + new string(' ', spacesAtEnd))
}; };
return result; return result;
} }
@@ -170,7 +170,7 @@ internal partial class SyntaxHighlighter(ReadOnlyCollection<StatementInformation
[GeneratedRegex(@"(?<!\\)(?:\\\\{2})*""(?:\\.|[^""\\])*""")] [GeneratedRegex(@"(?<!\\)(?:\\\\{2})*""(?:\\.|[^""\\])*""")]
private static partial Regex StringRegex(); private static partial Regex StringRegex();
// This regex matches the color information embedded in the string, which is in the format \v{colorIndex}\v{base64EncodedValue}\v. // This regex matches the color information embedded in the string, which is in the format \x01{colorIndex}\x01{base64EncodedValue}.
[GeneratedRegex("(?<=(\\v))(.*)(?=\\v)")] [GeneratedRegex(@"(?<=(\x01))(.*)(?=\x01)")]
private static partial Regex StringColorRegex(); private static partial Regex StringColorRegex();
} }
YesNt.Interpreter.Tests/ListStatementsTests.cs
+1 -1
View File
@@ -150,7 +150,7 @@ public class ListStatementsTests
List<string> lines = List<string> lines =
[ [
"list items new", "list items new",
"list items add hello~spcworld", "list items add \"hello world\"",
"list items get 0", "list items get 0",
"var result = %out eval", "var result = %out eval",
"${result}" "${result}"
YesNt.Interpreter.Tests/ProcessingStatementsTests.cs
+14 -2
View File
@@ -46,9 +46,21 @@ public class ProcessingStatementsTests
} }
[TestMethod] [TestMethod]
public void EvalDecodesSafeStringTest() public void EvalRawTildeSequenceIsLiteralTest()
{ {
YesNtAssert.IsLineEqual("hello~nliworld eval", "hello\nworld"); YesNtAssert.IsLineEqual("hello~nliworld eval", "hello~nliworld");
}
[TestMethod]
public void EvalDecodesStringLiteralEscapesTest()
{
List<string> lines =
[
"var x = \"hello\\nworld\"",
"${x} eval"
];
YesNtAssert.IsLastLineEqual(lines, "hello\nworld");
} }
[TestMethod] [TestMethod]
YesNt.Interpreter/Statements/StringLiteralStatements.cs
+4
View File
@@ -70,6 +70,10 @@ internal class StringLiteralStatements : StatementRuntimeInformation
'n' => '\n', 'n' => '\n',
'r' => '\r', 'r' => '\r',
't' => '\t', 't' => '\t',
'b' => '\b',
'f' => '\f',
'a' => '\a',
'v' => '\v',
'"' => '"', '"' => '"',
'\\' => '\\', '\\' => '\\',
_ => escapeChar _ => escapeChar
YesNt.Interpreter/Utilities/StringExtentions.cs
+26 -22
View File
@@ -13,10 +13,14 @@ namespace YesNt.Interpreter.Utilities;
/// <para> /// <para>
/// YesNt uses a "safe string" encoding to pass values through the interpreter pipeline without /// YesNt uses a "safe string" encoding to pass values through the interpreter pipeline without
/// accidentally triggering keyword matching. Special characters (spaces, operators, punctuation, /// accidentally triggering keyword matching. Special characters (spaces, operators, punctuation,
/// control characters) are replaced with tilde-prefixed three-letter codes /// control characters) are replaced with SOH-delimited three-letter codes
/// (e.g. space → <c>~spc</c>, newline → <c>~nli</c>). The mapping is defined in /// (e.g. space → <c>\x01spc\x01</c>, newline → <c>\x01nli\x01</c>). These codes use the
/// <see cref="ReplacementRules"/>. Use <see cref="ToSafeString"/> to encode and /// non-printable SOH character (U+0001) as a sentinel. Written via string concatenation
/// <see cref="FromSafeString"/> to decode. /// (<c>"\x01" + "spc" + "\x01"</c>) to avoid C#'s greedy <c>\x</c> hex escape absorbing
/// following hex-digit letters. U+0001 cannot appear in normal user source, preventing raw
/// source text from being accidentally decoded.
/// The mapping is defined in <see cref="ReplacementRules"/>. Use <see cref="ToSafeString"/>
/// to encode and <see cref="FromSafeString"/> to decode.
/// </para> /// </para>
/// </remarks> /// </remarks>
public static class StringExtensions public static class StringExtensions
@@ -29,21 +33,21 @@ public static class StringExtensions
/// </summary> /// </summary>
public static Dictionary<string, string> ReplacementRules { get; } = new() public static Dictionary<string, string> ReplacementRules { get; } = new()
{ {
{"~", "~til" }, {"~", "\x01" + "til" + "\x01" },
{" ", "~spc" }, {" ", "\x01" + "spc" + "\x01" },
{"%", "~per" }, {"%", "\x01" + "per" + "\x01" },
{"<", "~let" }, {"<", "\x01" + "let" + "\x01" },
{">", "~grt" }, {">", "\x01" + "grt" + "\x01" },
{",", "~com" }, {",", "\x01" + "com" + "\x01" },
{"!", "~exm" }, {"!", "\x01" + "exm" + "\x01" },
{"|", "~pip" }, {"|", "\x01" + "pip" + "\x01" },
{"\n","~nli" }, {"\n", "\x01" + "nli" + "\x01" },
{"\r","~ret" }, {"\r", "\x01" + "ret" + "\x01" },
{"\t","~tab" }, {"\t", "\x01" + "tab" + "\x01" },
{"\b","~bac" }, {"\b", "\x01" + "bac" + "\x01" },
{"\f","~for" }, {"\f", "\x01" + "for" + "\x01" },
{"\a","~ale" }, {"\a", "\x01" + "ale" + "\x01" },
{"", "~emp" }, {"", "\x01" + "emp" + "\x01" },
}; };
static StringExtensions() static StringExtensions()
@@ -130,15 +134,15 @@ public static class StringExtensions
private static string ReplaceOnce(string input, Dictionary<string, string> replacementRules) private static string ReplaceOnce(string input, Dictionary<string, string> replacementRules)
{ {
// ~emp/string.Empty is a special case, it is used to represent empty strings and won't work with the normal rules because an empty string always matches and causes an infinite loop 3 letter abbreviation // \x01emp\x01/string.Empty is a special case, it is used to represent empty strings and won't work with the normal rules because an empty string always matches and causes an infinite loop 3 letter abbreviation
IEnumerable<KeyValuePair<string, string>> matches = replacementRules.Where(rule => rule.Key != string.Empty && input.Contains(rule.Key)); IEnumerable<KeyValuePair<string, string>> matches = replacementRules.Where(rule => rule.Key != string.Empty && input.Contains(rule.Key, StringComparison.Ordinal));
if (!matches.Any()) if (!matches.Any())
{ {
return input; return input;
} }
KeyValuePair<string, string> match = matches.First(); KeyValuePair<string, string> match = matches.First();
int startIndex = input.IndexOf(match.Key); int startIndex = input.IndexOf(match.Key, StringComparison.Ordinal);
int endIndex = startIndex + match.Key.Length; int endIndex = startIndex + match.Key.Length;
string before = ReplaceOnce(input[..startIndex], replacementRules); string before = ReplaceOnce(input[..startIndex], replacementRules);
docs/language-reference.md
+46 -42
View File
@@ -30,7 +30,7 @@ Execution proceeds top-to-bottom unless a control-flow statement changes the lin
- Blank lines are silently skipped. - Blank lines are silently skipped.
- Lines starting with `#` are comments and are silently skipped. - Lines starting with `#` are comments and are silently skipped.
- Variable interpolation uses `${name}` and is evaluated before the statement runs. - Variable interpolation uses `${name}` and is evaluated before the statement runs.
- Special characters inside string literals are encoded internally and decoded on output — this is transparent to scripts. - Special characters inside string literals are encoded internally and decoded on output - this is transparent to scripts.
--- ---
@@ -38,7 +38,7 @@ Execution proceeds top-to-bottom unless a control-flow statement changes the lin
```ynt ```ynt
# This is a comment. # This is a comment.
print_line Hello # inline comments are NOT supported — everything after print_line is the argument print_line Hello # inline comments are NOT supported - everything after print_line is the argument
``` ```
Only whole-line comments (lines whose first non-whitespace character is `#`) are supported. Only whole-line comments (lines whose first non-whitespace character is `#`) are supported.
@@ -60,10 +60,14 @@ print_line "She said \"hi\""
| `\n` | Newline | | `\n` | Newline |
| `\r` | Carriage return | | `\r` | Carriage return |
| `\t` | Horizontal tab | | `\t` | Horizontal tab |
| `\b` | Backspace |
| `\f` | Form feed |
| `\a` | Alert (bell) |
| `\v` | Vertical tab |
| `\"` | Literal double-quote | | `\"` | Literal double-quote |
| `\\` | Literal backslash | | `\\` | Literal backslash |
Variable interpolation (`${name}`) is **not** evaluated inside string literals — the braces Variable interpolation (`${name}`) is **not** evaluated inside string literals - the braces
and content are passed through verbatim. and content are passed through verbatim.
```ynt ```ynt
@@ -76,7 +80,7 @@ print_line "hello " ${x} # prints: hello world (interpolation outside the lite
## Variables ## Variables
### Local variables — `var` ### Local variables - `var`
``` ```
var <name> = <value> var <name> = <value>
@@ -92,7 +96,7 @@ var greeting = Hello, world!
Variable names may only contain letters and digits (`[a-zA-Z0-9]`). Variable names may only contain letters and digits (`[a-zA-Z0-9]`).
### Global variables — `global` ### Global variables - `global`
``` ```
global <name> = <value> global <name> = <value>
@@ -104,7 +108,7 @@ Defines or updates a variable that is visible across all function scopes and bac
global total = 100 global total = 100
``` ```
### Reading a variable — `${name}` ### Reading a variable - `${name}`
`${name}` is an inline token that is replaced with the variable's value before the statement executes. `${name}` is an inline token that is replaced with the variable's value before the statement executes.
It can appear anywhere in a line and multiple occurrences are replaced left to right. It can appear anywhere in a line and multiple occurrences are replaced left to right.
@@ -116,7 +120,7 @@ var b = 10
print_line ${a} plus ${b} print_line ${a} plus ${b}
``` ```
### Deleting a variable — `delete` ### Deleting a variable - `delete`
``` ```
delete <name> delete <name>
@@ -134,7 +138,7 @@ delete temp
## Console I/O ## Console I/O
### Print with newline — `print_line` ### Print with newline - `print_line`
``` ```
print_line <text> print_line <text>
@@ -149,7 +153,7 @@ print_line
print_line Done. print_line Done.
``` ```
### Print without newline — `print` ### Print without newline - `print`
``` ```
print <text> print <text>
@@ -163,7 +167,7 @@ var name = %read_line
print_line Hello, ${name}! print_line Hello, ${name}!
``` ```
### Read a line of input — `%read_line` ### Read a line of input - `%read_line`
`%read_line` is an inline token that is replaced with one line of text read from standard input. `%read_line` is an inline token that is replaced with one line of text read from standard input.
@@ -172,7 +176,7 @@ var answer = %read_line
print_line You typed: ${answer} print_line You typed: ${answer}
``` ```
### Read a single key — `%read_key` ### Read a single key - `%read_key`
`%read_key` is an inline token that is replaced with the single character pressed by the user (no Enter required). `%read_key` is an inline token that is replaced with the single character pressed by the user (no Enter required).
@@ -182,7 +186,7 @@ var key = %read_key
print_line You pressed: ${key} print_line You pressed: ${key}
``` ```
### Clear the console — `clear` ### Clear the console - `clear`
``` ```
clear clear
@@ -348,7 +352,7 @@ if ${debug} == True call dump_state
## Functions ## Functions
### Declaring a function — `func` ### Declaring a function - `func`
``` ```
func <name>: func <name>:
@@ -367,7 +371,7 @@ func add:
return return
``` ```
### Calling a function — `call` ### Calling a function - `call`
``` ```
call <name> call <name>
@@ -384,13 +388,13 @@ call add with 3, 7
### Input arguments ### Input arguments
#### Push an input argument — `push_in` #### Push an input argument - `push_in`
``` ```
push_in <value> push_in <value>
``` ```
Pushes a value onto the input argument stack. Values are consumed in the order they were pushed — Pushes a value onto the input argument stack. Values are consumed in the order they were pushed -
the first `push_in` call is the first value consumed by `%in` inside the function. the first `push_in` call is the first value consumed by `%in` inside the function.
```ynt ```ynt
@@ -398,7 +402,7 @@ push_in Alice
call greet call greet
``` ```
#### Pop the next input argument — `%in` #### Pop the next input argument - `%in`
`%in` is an inline token (only valid inside a function) that is replaced with the next value `%in` is an inline token (only valid inside a function) that is replaced with the next value
popped from the argument stack. popped from the argument stack.
@@ -410,7 +414,7 @@ func greet:
return return
``` ```
#### Check if input argument exists — `%has_in` #### Check if input argument exists - `%has_in`
`%has_in` is replaced with `True` or `False` depending on whether the input stack is non-empty. `%has_in` is replaced with `True` or `False` depending on whether the input stack is non-empty.
Only valid inside a function. Only valid inside a function.
@@ -423,7 +427,7 @@ return
### Output values ### Output values
#### Push an output value — `push_out` #### Push an output value - `push_out`
``` ```
push_out <value> push_out <value>
@@ -431,7 +435,7 @@ push_out <value>
Only valid inside a function. Pushes a return value onto the output stack. Only valid inside a function. Pushes a return value onto the output stack.
#### Consume an output value — `%out` #### Consume an output value - `%out`
`%out` is an inline token that pops and inserts the top value from the output stack. `%out` is an inline token that pops and inserts the top value from the output stack.
Valid anywhere after a function call or list/processing statement that pushes to the output stack. Valid anywhere after a function call or list/processing statement that pushes to the output stack.
@@ -442,7 +446,7 @@ var total = %out
print_line ${total} print_line ${total}
``` ```
#### Check if output value exists — `%has_out` #### Check if output value exists - `%has_out`
`%has_out` is replaced with `True` or `False` depending on whether the output stack is non-empty. `%has_out` is replaced with `True` or `False` depending on whether the output stack is non-empty.
@@ -451,7 +455,7 @@ call maybe_produce
if %has_out == True call consume_result if %has_out == True call consume_result
``` ```
### Clear the call stack — `clear_call_stack` ### Clear the call stack - `clear_call_stack`
``` ```
clear_call_stack clear_call_stack
@@ -465,7 +469,7 @@ Discards all frames on the function call stack. Useful for error recovery.
Lists are ordered, mutable sequences of strings. All list operations start with the keyword `list`. Lists are ordered, mutable sequences of strings. All list operations start with the keyword `list`.
### Create or reset — `list … new` ### Create or reset - `list … new`
``` ```
list <name> new list <name> new
@@ -473,7 +477,7 @@ list <name> new
Creates an empty list. If the list already exists it is cleared. Creates an empty list. If the list already exists it is cleared.
### Add an item — `list … add` ### Add an item - `list … add`
``` ```
list <name> add <value> list <name> add <value>
@@ -481,7 +485,7 @@ list <name> add <value>
Appends `<value>` to the end of the list. Appends `<value>` to the end of the list.
### Get an item — `list … get` ### Get an item - `list … get`
``` ```
list <name> get <index> list <name> get <index>
@@ -498,7 +502,7 @@ var first = %out
print_line ${first} print_line ${first}
``` ```
### Set an item — `list … set` ### Set an item - `list … set`
``` ```
list <name> set <index> <value> list <name> set <index> <value>
@@ -506,7 +510,7 @@ list <name> set <index> <value>
Replaces the item at `<index>` with `<value>`. Replaces the item at `<index>` with `<value>`.
### Remove an item — `list … remove` ### Remove an item - `list … remove`
``` ```
list <name> remove <index> list <name> remove <index>
@@ -514,7 +518,7 @@ list <name> remove <index>
Removes the item at `<index>`. Subsequent items shift down. Removes the item at `<index>`. Subsequent items shift down.
### Insert an item — `list … insert` ### Insert an item - `list … insert`
``` ```
list <name> insert <index> <value> list <name> insert <index> <value>
@@ -522,7 +526,7 @@ list <name> insert <index> <value>
Inserts `<value>` before position `<index>`. Inserts `<value>` before position `<index>`.
### Get the length — `list … length` ### Get the length - `list … length`
``` ```
list <name> length list <name> length
@@ -536,7 +540,7 @@ var n = %out
print_line ${n} items print_line ${n} items
``` ```
### Clear all items — `list … clear` ### Clear all items - `list … clear`
``` ```
list <name> clear list <name> clear
@@ -544,7 +548,7 @@ list <name> clear
Removes all items but keeps the list alive. Removes all items but keeps the list alive.
### Delete a list — `list … delete` ### Delete a list - `list … delete`
``` ```
list <name> delete list <name> delete
@@ -556,11 +560,11 @@ Removes the list entirely.
## Processing ## Processing
### Arithmetic — `calc` ### Arithmetic - `calc`
See [Arithmetic](#arithmetic). See [Arithmetic](#arithmetic).
### Decode a safe string — `eval` ### Decode a safe string - `eval`
``` ```
<expression> eval <expression> eval
@@ -569,7 +573,7 @@ See [Arithmetic](#arithmetic).
Decodes any internally-encoded characters in the current line back to their plain-text form. Decodes any internally-encoded characters in the current line back to their plain-text form.
Useful after producing text from string literal operations that you want to re-use as plain text. Useful after producing text from string literal operations that you want to re-use as plain text.
### Run from the current line in a background task — `task` ### Run from the current line in a background task - `task`
``` ```
<line> task <line> task
@@ -608,7 +612,7 @@ func background_job:
This works well for task-only worker flows. In non-task/shared flows, prefer `return` if you want This works well for task-only worker flows. In non-task/shared flows, prefer `return` if you want
to return to the caller instead of terminating that execution flow with `exit`. to return to the caller instead of terminating that execution flow with `exit`.
### Sleep — `sleep` ### Sleep - `sleep`
``` ```
sleep <milliseconds> sleep <milliseconds>
@@ -622,7 +626,7 @@ sleep 1000
print_line One second later. print_line One second later.
``` ```
### Get string length — `length` ### Get string length - `length`
``` ```
length <text> length <text>
@@ -636,7 +640,7 @@ var len = %out
print_line ${len} print_line ${len}
``` ```
### Import another script — `import` ### Import another script - `import`
``` ```
import <path> import <path>
@@ -655,7 +659,7 @@ import lib/math.ynt
## System ## System
### Execute a program — `exec` ### Execute a program - `exec`
``` ```
exec <program> exec <program>
@@ -706,7 +710,7 @@ var roll = %rand
## Termination ## Termination
### Normal exit — `exit` ### Normal exit - `exit`
``` ```
exit exit
@@ -714,7 +718,7 @@ exit
Terminates the script gracefully. Background tasks that are still running are not cancelled. Terminates the script gracefully. Background tasks that are still running are not cancelled.
### Exit and cancel all tasks — `abort_all` ### Exit and cancel all tasks - `abort_all`
``` ```
abort_all abort_all
@@ -722,7 +726,7 @@ abort_all
Terminates the script and signals all background tasks to stop. Terminates the script and signals all background tasks to stop.
### Throw an error — `throw` ### Throw an error - `throw`
``` ```
throw <message> throw <message>
@@ -740,7 +744,7 @@ func validate_fail:
return return
``` ```
### Non-fatal error — `error` ### Non-fatal error - `error`
``` ```
error <message> error <message>