llama : add token matching support to llama-grammar (#17816)

* llama : add token support to llama-grammar * fix inverse token comment * refactor trigger_patterns to replay tokens instead of the entire string * add token documentation * fix test-llama-grammar * improve test cases for tokens
2025-12-09 00:32:57 -06:00
parent 1d2a1ab73d
commit e39502e74b
6 changed files with 400 additions and 38 deletions
@@ -515,5 +515,19 @@ int main()
        {LLAMA_GRETYPE_END, 0},
    });

+    // <[1000]> = "<think>"
+    // <[1001]> = "</think>"
+    verify_parsing(R"""(
+        root  ::= <[1000]> !<[1001]> <[1001]>
+    )""", {
+        {"root", 0}
+    }, {
+        // root (index 0)
+        {LLAMA_GRETYPE_TOKEN, 1000},
+        {LLAMA_GRETYPE_TOKEN_NOT, 1001},
+        {LLAMA_GRETYPE_TOKEN, 1001},
+        {LLAMA_GRETYPE_END, 0},
+    });
+
    return 0;
 }