feat(source): stage typed declaration syntax on shared raw tokens

crates/oa-core/src/language.rs

@@ -2,6 +2,7 @@
 #[path = "language/semantic_source.rs"]
 mod semantic_source;
 mod source3_bridge;
+mod raw_tokens;
 // Staged syntax foundation only. Public source8 admission waits for complete
 // declaration lowering, source updates and the root-owned resource gate.
 #[cfg(test)]
@@ -152,86 +153,7 @@ pub struct Analysis {
 /// Tolerant highlighting tokens cover the whole input, including incomplete
 /// strings and unexpected characters. No model is created or modified.
 pub fn tokenize(source: &str) -> Vec<TokenSpan> {
-    let mut tokens = Vec::new();
-    let mut cursor = 0;
-    while cursor < source.len() {
-        let start = cursor;
-        let ch = source[cursor..].chars().next().expect("in bounds");
-        let class = if matches!(ch, ' ' | '\t' | '\r' | '\n') {
-            cursor += 1;
-            while source
-                .as_bytes()
-                .get(cursor)
-                .is_some_and(|next| matches!(*next, b' ' | b'\t' | b'\r' | b'\n'))
-            {
-                cursor += 1;
-            }
-            TokenClass::Whitespace
-        } else if source[cursor..].starts_with("//") {
-            cursor += source[cursor..].find('\n').unwrap_or(source.len() - cursor);
-            TokenClass::Comment
-        } else if ch == '"' {
-            cursor += 1;
-            let mut escaped = false;
-            let mut closed = false;
-            while cursor < source.len() {
-                let next = source[cursor..].chars().next().expect("in bounds");
-                cursor += next.len_utf8();
-                if escaped {
-                    escaped = false;
-                } else if next == '\\' {
-                    escaped = true;
-                } else if next == '"' {
-                    closed = true;
-                    break;
-                }
-            }
-            if closed && serde_json::from_str::<String>(&source[start..cursor]).is_ok() {
-                TokenClass::String
-            } else {
-                TokenClass::Error
-            }
-        } else if ch.is_ascii_alphabetic() || ch == '_' {
-            cursor += 1;
-            while source
-                .as_bytes()
-                .get(cursor)
-                .is_some_and(|next| next.is_ascii_alphanumeric() || *next == b'_')
-            {
-                cursor += 1;
-            }
-            if reserved(&source[start..cursor])
-                || matches!(&source[start..cursor], "true" | "false" | "none")
-            {
-                TokenClass::Keyword
-            } else {
-                TokenClass::Identifier
-            }
-        } else if ch.is_ascii_digit() || matches!(ch, '-' | '+' | '.') {
-            cursor += 1;
-            while source.as_bytes().get(cursor).is_some_and(|next| {
-                next.is_ascii_alphanumeric() || matches!(*next, b'.' | b'+' | b'-' | b'_')
-            }) {
-                cursor += 1;
-            }
-            if serde_json::from_str::<serde_json::Number>(&source[start..cursor]).is_ok() {
-                TokenClass::Number
-            } else {
-                TokenClass::Error
-            }
-        } else if "{}();,[]:=@".contains(ch) {
-            cursor += ch.len_utf8();
-            TokenClass::Punctuation
-        } else {
-            cursor += ch.len_utf8();
-            TokenClass::Error
-        };
-        tokens.push(TokenSpan {
-            class,
-            span: Span { start, end: cursor },
-        });
-    }
-    tokens
+    raw_tokens::legacy(source)
 }
 
 pub fn lint(source: &str) -> Vec<Diagnostic> {